Keep progress writes outside model deadlines
This commit is contained in:
@@ -1561,3 +1561,45 @@ func TestProgressDeliveryFailureKeepsCompletedProcessing(t *testing.T) {
|
||||
t.Fatalf("presentation changed report status: %+v", doc)
|
||||
}
|
||||
}
|
||||
|
||||
type delayedProposalProgress struct {
|
||||
delay time.Duration
|
||||
delayed bool
|
||||
}
|
||||
|
||||
func (w *delayedProposalProgress) Write(p []byte) (int, error) {
|
||||
if !w.delayed && bytes.Contains(p, []byte("proposal started")) {
|
||||
w.delayed = true
|
||||
time.Sleep(w.delay)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestProgressDeliveryDoesNotSpendModelDeadline(t *testing.T) {
|
||||
const timeout = 500 * time.Millisecond
|
||||
for _, showProgress := range []bool{false, true} {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret)
|
||||
config, err := os.ReadFile(f.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config = bytes.Replace(config, []byte("llm:\n"), []byte("llm:\n operation_timeout: "+timeout.String()+"\n"), 1)
|
||||
if err := os.WriteFile(f.config, config, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := []string{"process", f.transcript, "--config", f.config, "--output", f.output}
|
||||
diagnostics := &delayedProposalProgress{delay: timeout + 250*time.Millisecond}
|
||||
if showProgress {
|
||||
args = append(args, "--progress")
|
||||
}
|
||||
code := cli.Run(context.Background(), args, io.Discard, diagnostics)
|
||||
output, err := os.ReadFile(f.output)
|
||||
if code != 0 || err != nil || !bytes.Contains(output, []byte("Hello world")) || f.provider.callCount() != 3 {
|
||||
t.Fatalf("progress=%v, exit=%d, requests=%d, output=%q, read error=%v", showProgress, code, f.provider.callCount(), output, err)
|
||||
}
|
||||
if diagnostics.delayed != showProgress {
|
||||
t.Fatalf("progress=%v, delayed=%v", showProgress, diagnostics.delayed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,8 @@ An opaque exchange in the structured-output contract keeps provider state outsid
|
||||
modules and validators. The adapter shares request construction, notifications,
|
||||
observers, and once-only completion between ordinary and coordinated calls.
|
||||
Completion attaches terminal counts when available and records one logical finish.
|
||||
Progress uses the adapter's logical call boundaries. Writes to the diagnostic
|
||||
stream occur outside PromptKit's operation deadline.
|
||||
Prepared requests and terminal output travel in the returned response for an
|
||||
optional interaction artifact. Logical call notifications carry metadata.
|
||||
|
||||
|
||||
@@ -197,6 +197,11 @@ reflects the evidence modes, and the CLI paragraph renders normally.
|
||||
|
||||
### R6 — Keep Progress Writes Outside Model Operation Deadlines (P2)
|
||||
|
||||
**Complete.** Progress is emitted before and after each timed model call,
|
||||
including coordinated repair turns. An integration regression delays stderr
|
||||
beyond the model deadline and verifies the same provider calls and correction.
|
||||
Live progress, reporting modes, and presentation failure tests also passed.
|
||||
|
||||
**Affected plan:** Stages 12 and 19.
|
||||
|
||||
**Evidence:** [Pipeline wiring](../../internal/cli/process_pipeline.go#L30)
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error), progress func(string)) (processResult, error) {
|
||||
@@ -28,12 +27,8 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
hooks = observer.Hooks()
|
||||
}
|
||||
if progress != nil {
|
||||
observe := hooks.OnEventWithCall
|
||||
hooks.OnEventWithCall = func(callID uint64, coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
|
||||
if observe != nil {
|
||||
observe(callID, coordinates, event)
|
||||
}
|
||||
reportModelProgress(progress, coordinates, event)
|
||||
hooks.OnProgress = func(coordinates structuredoutput.DiagnosticContext, counts *outcomes.ExecutionCounts, err error) {
|
||||
reportModelProgress(progress, coordinates, counts, err)
|
||||
}
|
||||
}
|
||||
for i := range specs {
|
||||
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func runnerProgress(progress func(string)) func(runner.Progress) {
|
||||
@@ -31,8 +31,8 @@ func runnerProgress(progress func(string)) func(runner.Progress) {
|
||||
}
|
||||
}
|
||||
|
||||
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, event pk.ExecutionEvent) {
|
||||
if coordinates.StageIndex == nil || (event.Kind != pk.ExecutionStarted && event.Kind != pk.ExecutionFinished) {
|
||||
func reportModelProgress(progress func(string), coordinates structuredoutput.DiagnosticContext, counts *outcomes.ExecutionCounts, err error) {
|
||||
if coordinates.StageIndex == nil {
|
||||
return
|
||||
}
|
||||
location := progressLocation(*coordinates.StageIndex, coordinates.ModuleKey, coordinates.StageID, coordinates.SectionIndex)
|
||||
@@ -49,11 +49,18 @@ func reportModelProgress(progress func(string), coordinates structuredoutput.Dia
|
||||
}
|
||||
operation += fmt.Sprintf(" turn %d", *coordinates.RepairAttempt)
|
||||
}
|
||||
if event.Kind == pk.ExecutionStarted {
|
||||
if counts == nil {
|
||||
progress(fmt.Sprintf("%s %s started", location, operation))
|
||||
} else {
|
||||
progress(fmt.Sprintf("%s %s finished (%s, %d provider attempts)", location, operation, event.Outcome, event.Attempts))
|
||||
return
|
||||
}
|
||||
outcome := "completed"
|
||||
if err != nil {
|
||||
outcome = "failed"
|
||||
}
|
||||
if counts.Available {
|
||||
outcome += fmt.Sprintf(", %d provider attempts", counts.ProviderAttempts)
|
||||
}
|
||||
progress(fmt.Sprintf("%s %s finished (%s)", location, operation, outcome))
|
||||
}
|
||||
|
||||
func progressLocation(index int, module string, id *string, section *int) string {
|
||||
|
||||
@@ -22,6 +22,9 @@ type CallHooks struct {
|
||||
OnCall func(structuredoutput.CallRecord)
|
||||
OnEvent pk.ExecutionObserver
|
||||
OnEventWithCall func(uint64, structuredoutput.DiagnosticContext, pk.ExecutionEvent)
|
||||
// OnProgress runs before and after the engine's timed operation. A nil
|
||||
// count marks the start. A completion may have unknown execution counts.
|
||||
OnProgress func(structuredoutput.DiagnosticContext, *outcomes.ExecutionCounts, error)
|
||||
}
|
||||
|
||||
type RoleClient struct {
|
||||
@@ -66,6 +69,11 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
var prepared *structuredoutput.PreparedSnapshot
|
||||
var prompt structuredoutput.PromptMetadata
|
||||
var record structuredoutput.CallRecord
|
||||
progressContext := req.Context
|
||||
progressContext.Role = c.role
|
||||
if progress := c.hooks.OnProgress; progress != nil {
|
||||
progress(progressContext, nil, nil)
|
||||
}
|
||||
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
|
||||
record.CallID = c.runtime.nextCallID.Add(1)
|
||||
record.RequestedPromptID = req.PromptID
|
||||
@@ -115,6 +123,9 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
}
|
||||
onCall(record)
|
||||
}
|
||||
if progress := c.hooks.OnProgress; progress != nil {
|
||||
progress(progressContext, &response.Execution, err)
|
||||
}
|
||||
return response, err
|
||||
}}
|
||||
fail := func(err error) (*pendingCall, error) {
|
||||
|
||||
Reference in New Issue
Block a user