From 9192239bfdbf9d28214c946a26771f3fd6f22467 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 17 Nov 2026 04:17:24 +0000 Subject: [PATCH] Keep progress writes outside model deadlines --- cmd/audita/main_integration_test.go | 42 +++++++++++++++++++++++++++++ docs/internal/llm-runtime.md | 2 ++ docs/roadmap/remediation.md | 5 ++++ internal/cli/process_pipeline.go | 9 ++----- internal/cli/process_progress.go | 19 ++++++++----- internal/framework/llm/adapter.go | 11 ++++++++ 6 files changed, 75 insertions(+), 13 deletions(-) diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 6812714..e8af3b2 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -1561,3 +1561,45 @@ func TestProgressDeliveryFailureKeepsCompletedProcessing(t *testing.T) { t.Fatalf("presentation changed report status: %+v", doc) } } + +type delayedProposalProgress struct { + delay time.Duration + delayed bool +} + +func (w *delayedProposalProgress) Write(p []byte) (int, error) { + if !w.delayed && bytes.Contains(p, []byte("proposal started")) { + w.delayed = true + time.Sleep(w.delay) + } + return len(p), nil +} + +func TestProgressDeliveryDoesNotSpendModelDeadline(t *testing.T) { + const timeout = 500 * time.Millisecond + for _, showProgress := range []bool{false, true} { + f := newProcessFixture(t, "normal", "[grammar]") + t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret) + config, err := os.ReadFile(f.config) + if err != nil { + t.Fatal(err) + } + config = bytes.Replace(config, []byte("llm:\n"), []byte("llm:\n operation_timeout: "+timeout.String()+"\n"), 1) + if err := os.WriteFile(f.config, config, 0600); err != nil { + t.Fatal(err) + } + args := []string{"process", f.transcript, "--config", f.config, "--output", f.output} + diagnostics := &delayedProposalProgress{delay: timeout + 250*time.Millisecond} + if showProgress { + args = append(args, "--progress") + } + code := cli.Run(context.Background(), args, io.Discard, diagnostics) + output, err := os.ReadFile(f.output) + if code != 0 || err != nil || !bytes.Contains(output, []byte("Hello world")) || f.provider.callCount() != 3 { + t.Fatalf("progress=%v, exit=%d, requests=%d, output=%q, read error=%v", showProgress, code, f.provider.callCount(), output, err) + } + if diagnostics.delayed != showProgress { + t.Fatalf("progress=%v, delayed=%v", showProgress, diagnostics.delayed) + } + } +} diff --git a/docs/internal/llm-runtime.md b/docs/internal/llm-runtime.md index 5e86211..13a8333 100644 --- a/docs/internal/llm-runtime.md +++ b/docs/internal/llm-runtime.md @@ -34,6 +34,8 @@ An opaque exchange in the structured-output contract keeps provider state outsid modules and validators. The adapter shares request construction, notifications, observers, and once-only completion between ordinary and coordinated calls. Completion attaches terminal counts when available and records one logical finish. +Progress uses the adapter's logical call boundaries. Writes to the diagnostic +stream occur outside PromptKit's operation deadline. Prepared requests and terminal output travel in the returned response for an optional interaction artifact. Logical call notifications carry metadata. diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 06a76b3..dd2f46c 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -197,6 +197,11 @@ reflects the evidence modes, and the CLI paragraph renders normally. ### R6 — Keep Progress Writes Outside Model Operation Deadlines (P2) +**Complete.** Progress is emitted before and after each timed model call, +including coordinated repair turns. An integration regression delays stderr +beyond the model deadline and verifies the same provider calls and correction. +Live progress, reporting modes, and presentation failure tests also passed. + **Affected plan:** Stages 12 and 19. **Evidence:** [Pipeline wiring](../../internal/cli/process_pipeline.go#L30) diff --git a/internal/cli/process_pipeline.go b/internal/cli/process_pipeline.go index 586808c..dae6a63 100644 --- a/internal/cli/process_pipeline.go +++ b/internal/cli/process_pipeline.go @@ -13,7 +13,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/modules" - pk "go.maximumdirect.net/promptkit" ) func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) { @@ -28,12 +27,8 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa hooks = observer.Hooks() } if progress != nil { - observe := hooks.OnEventWithCall - hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) { - if observe != nil { - observe(callID, coordinates, event) - } - reportModelProgress(progress, coordinates, event) + hooks.OnProgress = func(coordinates structuredoutput.DiagnosticContext, counts *outcomes.ExecutionCounts, err error) { + reportModelProgress(progress, coordinates, counts, err) } } for i := range specs { diff --git a/internal/cli/process_progress.go b/internal/cli/process_progress.go index d8ee14c..b0c8aa4 100644 --- a/internal/cli/process_progress.go +++ b/internal/cli/process_progress.go @@ -5,9 +5,9 @@ import ( "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" - pk "go.maximumdirect.net/promptkit" ) func runnerProgress(progress func(string)) func(runner.Progress) { @@ -31,8 +31,8 @@ func runnerProgress(progress func(string)) func(runner.Progress) { } } -func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) { - if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) { +func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, counts *outcomes.ExecutionCounts, err error) { + if coordinates.StageIndex == nil { return } location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex) @@ -49,11 +49,18 @@ func reportModelProgress(progress func(string), coordinates structuredoutput.Dia } operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt) } - if event.Kind == pk.ExecutionStarted { + if counts == nil { progress(fmt.Sprintf("%s %s started", location, operation)) - } else { - progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts)) + return } + outcome := "completed" + if err != nil { + outcome = "failed" + } + if counts.Available { + outcome += fmt.Sprintf(", %d provider attempts", counts.ProviderAttempts) + } + progress(fmt.Sprintf("%s %s finished (%s)", location, operation, outcome)) } func progressLocation(index int, module string, id *string, section *int) string { diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index c00a8c3..b86fe3e 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -22,6 +22,9 @@ type CallHooks struct { OnCall func(structuredoutput.CallRecord) OnEvent pk.ExecutionObserver OnEventWithCall func(uint64, structuredoutput.DiagnosticContext, pk.ExecutionEvent) + // OnProgress runs before and after the engine's timed operation. A nil + // count marks the start. A completion may have unknown execution counts. + OnProgress func(structuredoutput.DiagnosticContext, *outcomes.ExecutionCounts, error) } type RoleClient struct { @@ -66,6 +69,11 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re var prepared *structuredoutput.PreparedSnapshot var prompt structuredoutput.PromptMetadata var record structuredoutput.CallRecord + progressContext := req.Context + progressContext.Role = c.role + if progress := c.hooks.OnProgress; progress != nil { + progress(progressContext, nil, nil) + } if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil { record.CallID = c.runtime.nextCallID.Add(1) record.RequestedPromptID = req.PromptID @@ -115,6 +123,9 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re } onCall(record) } + if progress := c.hooks.OnProgress; progress != nil { + progress(progressContext, &response.Execution, err) + } return response, err }} fail := func(err error) (*pendingCall, error) {