Keep progress writes outside model deadlines

This commit is contained in:
2026-11-17 04:17:24 +00:00
parent 1bc54a405c
commit 9192239bfd
6 changed files with 75 additions and 13 deletions

View File

@@ -1561,3 +1561,45 @@ func TestProgressDeliveryFailureKeepsCompletedProcessing(t *testing.T) {
t.Fatalf("presentation changed report status: %+v", doc)
}
}
type delayedProposalProgress struct {
delay time.Duration
delayed bool
}
func (w *delayedProposalProgress) Write(p []byte) (int, error) {
if !w.delayed && bytes.Contains(p, []byte("proposal started")) {
w.delayed = true
time.Sleep(w.delay)
}
return len(p), nil
}
func TestProgressDeliveryDoesNotSpendModelDeadline(t *testing.T) {
const timeout = 500 * time.Millisecond
for _, showProgress := range []bool{false, true} {
f := newProcessFixture(t, "normal", "[grammar]")
t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret)
config, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
config = bytes.Replace(config, []byte("llm:\n"), []byte("llm:\n operation_timeout: "+timeout.String()+"\n"), 1)
if err := os.WriteFile(f.config, config, 0600); err != nil {
t.Fatal(err)
}
args := []string{"process", f.transcript, "--config", f.config, "--output", f.output}
diagnostics := &delayedProposalProgress{delay: timeout + 250*time.Millisecond}
if showProgress {
args = append(args, "--progress")
}
code := cli.Run(context.Background(), args, io.Discard, diagnostics)
output, err := os.ReadFile(f.output)
if code != 0 || err != nil || !bytes.Contains(output, []byte("Hello world")) || f.provider.callCount() != 3 {
t.Fatalf("progress=%v, exit=%d, requests=%d, output=%q, read error=%v", showProgress, code, f.provider.callCount(), output, err)
}
if diagnostics.delayed != showProgress {
t.Fatalf("progress=%v, delayed=%v", showProgress, diagnostics.delayed)
}
}
}

View File

@@ -34,6 +34,8 @@ An opaque exchange in the structured-output contract keeps provider state outsid
modules and validators. The adapter shares request construction, notifications,
observers, and once-only completion between ordinary and coordinated calls.
Completion attaches terminal counts when available and records one logical finish.
Progress uses the adapter's logical call boundaries. Writes to the diagnostic
stream occur outside PromptKit's operation deadline.
Prepared requests and terminal output travel in the returned response for an
optional interaction artifact. Logical call notifications carry metadata.

View File

@@ -197,6 +197,11 @@ reflects the evidence modes, and the CLI paragraph renders normally.
### R6 — Keep Progress Writes Outside Model Operation Deadlines (P2)
**Complete.** Progress is emitted before and after each timed model call,
including coordinated repair turns. An integration regression delays stderr
beyond the model deadline and verifies the same provider calls and correction.
Live progress, reporting modes, and presentation failure tests also passed.
**Affected plan:** Stages 12 and 19.
**Evidence:** [Pipeline wiring](../../internal/cli/process_pipeline.go#L30)

View File

@@ -13,7 +13,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules"
pk "go.maximumdirect.net/promptkit"
)
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) {
@@ -28,12 +27,8 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
hooks = observer.Hooks()
}
if progress != nil {
observe := hooks.OnEventWithCall
hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
if observe != nil {
observe(callID, coordinates, event)
}
reportModelProgress(progress, coordinates, event)
hooks.OnProgress = func(coordinates structuredoutput.DiagnosticContext, counts *outcomes.ExecutionCounts, err error) {
reportModelProgress(progress, coordinates, counts, err)
}
}
for i := range specs {

View File

@@ -5,9 +5,9 @@ import (
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "go.maximumdirect.net/promptkit"
)
func runnerProgress(progress func(string)) func(runner.Progress) {
@@ -31,8 +31,8 @@ func runnerProgress(progress func(string)) func(runner.Progress) {
}
}
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) {
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, counts *outcomes.ExecutionCounts, err error) {
if coordinates.StageIndex == nil {
return
}
location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex)
@@ -49,11 +49,18 @@ func reportModelProgress(progress func(string), coordinates structuredoutput.Dia
}
operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt)
}
if event.Kind == pk.ExecutionStarted {
if counts == nil {
progress(fmt.Sprintf("%s %s started", location, operation))
} else {
progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts))
return
}
outcome := "completed"
if err != nil {
outcome = "failed"
}
if counts.Available {
outcome += fmt.Sprintf(", %d provider attempts", counts.ProviderAttempts)
}
progress(fmt.Sprintf("%s %s finished (%s)", location, operation, outcome))
}
func progressLocation(index int, module string, id *string, section *int) string {

View File

@@ -22,6 +22,9 @@ type CallHooks struct {
OnCall func(structuredoutput.CallRecord)
OnEvent pk.ExecutionObserver
OnEventWithCall func(uint64, structuredoutput.DiagnosticContext, pk.ExecutionEvent)
// OnProgress runs before and after the engine's timed operation. A nil
// count marks the start. A completion may have unknown execution counts.
OnProgress func(structuredoutput.DiagnosticContext, *outcomes.ExecutionCounts, error)
}
type RoleClient struct {
@@ -66,6 +69,11 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
var prepared *structuredoutput.PreparedSnapshot
var prompt structuredoutput.PromptMetadata
var record structuredoutput.CallRecord
progressContext := req.Context
progressContext.Role = c.role
if progress := c.hooks.OnProgress; progress != nil {
progress(progressContext, nil, nil)
}
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1)
record.RequestedPromptID = req.PromptID
@@ -115,6 +123,9 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
}
onCall(record)
}
if progress := c.hooks.OnProgress; progress != nil {
progress(progressContext, &response.Execution, err)
}
return response, err
}}
fail := func(err error) (*pendingCall, error) {