From 891d4aac2c4e43c8754444667a2713ffdeb65bbc Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 15:08:02 +0000 Subject: [PATCH] Retain processing evidence only for selected outputs --- docs/internal/diagnostics-reporting.md | 6 + docs/roadmap/implementation.md | 16 + internal/cli/process_pipeline.go | 5 +- internal/framework/llm/adapter.go | 34 +- internal/framework/llm/adapter_test.go | 2 +- internal/framework/llm/conversation_test.go | 6 +- internal/framework/llm/coordinate_test.go | 6 +- internal/framework/llm/native_adapter_test.go | 2 + internal/framework/outcomes/outcomes.go | 54 +++ .../processreport/partial_validation_test.go | 2 +- internal/framework/processreport/report.go | 67 ++-- .../framework/proposal_generation/generate.go | 7 +- .../framework/runner/cancellation_test.go | 2 +- internal/framework/runner/evidence_test.go | 94 ++++++ internal/framework/runner/repair.go | 2 +- internal/framework/runner/runner.go | 317 ++++++++++-------- internal/framework/runner/runner_test.go | 8 +- internal/framework/runner/window_test.go | 2 +- internal/framework/runner/work_totals.go | 78 +++++ internal/framework/structuredoutput/call.go | 11 +- .../framework/validators/llm_validators.go | 2 +- 21 files changed, 494 insertions(+), 229 deletions(-) create mode 100644 internal/framework/runner/evidence_test.go create mode 100644 internal/framework/runner/work_totals.go diff --git a/docs/internal/diagnostics-reporting.md b/docs/internal/diagnostics-reporting.md index 43d815f..16f0a7d 100644 --- a/docs/internal/diagnostics-reporting.md +++ b/docs/internal/diagnostics-reporting.md @@ -79,3 +79,9 @@ own phase and available operation coordinates, and peer interruptions and later file failures preserve their own attribution. The correction ledger joins one candidate row by module instance and proposal index, preserving repair lineage and the recorded validator execution class. + +The invocation selects retained evidence once. Summary-only processing +accumulates counters as completed sections leave the work window. Ordinary +reports retain the full work matrix and operation facts. Debug retains +candidate and decision history, repair lineage, and native observations. +Without debug, the runtime skips prepared message snapshots. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5d3ee58..d7bce46 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -658,6 +658,22 @@ Audit #32 and supporting work for #30, #33, and #44. Resource measurements support reduction of waste without setting a constant-memory or disk-use requirement. +**Complete.** The CLI selects ordinary report and debug evidence before +processing. Section outcomes feed shared totals as they drain. The runner +retains the full work grid only for reports, and retains candidate and +validator history only for debug in the CLI. Duplicate module operations and +unused proposal totals were removed. Prepared messages are captured only for +interaction diagnostics. Mode comparison tests and the full offline suite +passed. + +A synthetic 4,000-section run with recoverable failures, proposal recoveries, +and apply-time filtering retained about 0.46 MB with summary only, 2.73 MB +with ordinary reports, 1.56 MB with debug evidence, and 3.83 MB with both +after garbage collection. Allocations per run were approximately 20.9, +22.0, 24.0, and 25.0 MB respectively. These runner measurements exclude +provider calls and disk artifacts. All four modes agreed on the transcript +and counters. Temporary measurement instrumentation was removed. + ## Stage 19 — Emit native processing progress and update operator guidance Audit #5, #57, and the operator-facing parts of #8, #24 and #25. diff --git a/internal/cli/process_pipeline.go b/internal/cli/process_pipeline.go index b46a8b7..c8ac138 100644 --- a/internal/cli/process_pipeline.go +++ b/internal/cli/process_pipeline.go @@ -19,6 +19,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa cfg := &loaded.Configuration.Config specs := prepared.Prepared.Specs debug := prepared.Prepared.Debug + evidence := runner.EvidenceSelection{Reports: cfg.Reporting.Enabled, Debug: debug != nil} var observer *llm.ObservationCollector var hooks llm.CallHooks if debug != nil { @@ -57,7 +58,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa } output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{ Config: cfg, Transcript: normalized, ModuleSpecs: specs, - Interactions: interactions, CaptureCandidates: debug != nil, + Interactions: interactions, Evidence: &evidence, }) if debug != nil { for _, warning := range writer.Warnings() { @@ -68,6 +69,6 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output}))) } snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, - ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !cfg.Reporting.Enabled}) + ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !evidence.Reports}) return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr } diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index 357cb17..c00a8c3 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -64,6 +64,7 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re var terminal pk.ExecutionEvent var terminalAvailable bool var prepared *structuredoutput.PreparedSnapshot + var prompt structuredoutput.PromptMetadata var record structuredoutput.CallRecord if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil { record.CallID = c.runtime.nextCallID.Add(1) @@ -90,19 +91,19 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re response.Execution.HTTPStatus = &status } } - if prepared != nil { - response.Prepared = prepared - response.Prompt = prepared.Prompt - response.Model = prepared.Prompt.Model - } + response.Prepared = prepared + response.Prompt = prompt + response.Model = prompt.Model mu.Unlock() if result != nil { - response.FinalContent = result.RawOutput + if req.CaptureDiagnostics { + response.FinalContent = result.RawOutput + } response.Model = result.ModelName } err := c.finishResponse(parent, &response, result, runErr, out) if onCall := c.hooks.OnCall; onCall != nil { - if response.Prepared != nil { + if response.Prompt.PromptID != "" { record.Prompt = response.Prompt } record.Finished = time.Now().UTC() @@ -146,12 +147,18 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re c.hooks.OnEventWithCall(record.CallID, record.Context, event) } }), - OnPrepared: func(_ context.Context, details pk.PreparedRun) { - snapshot := preparedSnapshot(details) + } + if req.CaptureDiagnostics || c.hooks.OnCall != nil { + call.request.OnPrepared = func(_ context.Context, details pk.PreparedRun) { + metadata := promptMetadata(details) + var snapshot *structuredoutput.PreparedSnapshot + if req.CaptureDiagnostics { + snapshot = &structuredoutput.PreparedSnapshot{Messages: convertMessages(details.Messages), Prompt: metadata} + } mu.Lock() - prepared = snapshot + prepared, prompt = snapshot, metadata mu.Unlock() - }, + } } return call, nil } @@ -290,11 +297,6 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata { } } -func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot { - messages := convertMessages(details.Messages) - return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)} -} - func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message { result := make([]structuredoutput.Message, len(messages)) for i, message := range messages { diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go index 81f39c7..a5b5609 100644 --- a/internal/framework/llm/adapter_test.go +++ b/internal/framework/llm/adapter_test.go @@ -89,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request { id = prompts.PromptIDValidatorEditorialReview vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"} } - return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}} + return structuredoutput.Request{CaptureDiagnostics: true, PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}} } func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { diff --git a/internal/framework/llm/conversation_test.go b/internal/framework/llm/conversation_test.go index cf3328a..575ad3a 100644 --- a/internal/framework/llm/conversation_test.go +++ b/internal/framework/llm/conversation_test.go @@ -86,7 +86,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) { if turns == 2 { return nil, nil } - return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil + return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil }) if err != nil || exhausted || turns != 2 { t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err) @@ -111,7 +111,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) { if turn.Response != nil { return nil, turn.Err } - return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil + return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil } _, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent) if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") { @@ -172,7 +172,7 @@ func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T) result = turn.Response return nil, turn.Err } - return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil + return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil }) if err != nil || exhausted || result == nil { t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err) diff --git a/internal/framework/llm/coordinate_test.go b/internal/framework/llm/coordinate_test.go index 0df55a7..ef437ba 100644 --- a/internal/framework/llm/coordinate_test.go +++ b/internal/framework/llm/coordinate_test.go @@ -55,7 +55,7 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) { return nil, nil } } - return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil + return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil }) if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 { t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err) @@ -102,7 +102,7 @@ func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) { } return nil, turn.Err } - return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil + return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil }) if !errors.Is(err, context.Canceled) || deliveries != 1 { t.Fatalf("deliveries=%d err=%v", deliveries, err) @@ -132,7 +132,7 @@ func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) { _, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{}) return nil, err } - return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil + return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil }) summary := Summarize(collector.Snapshot()) if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete { diff --git a/internal/framework/llm/native_adapter_test.go b/internal/framework/llm/native_adapter_test.go index 0d5ca09..6cb03d7 100644 --- a/internal/framework/llm/native_adapter_test.go +++ b/internal/framework/llm/native_adapter_test.go @@ -109,6 +109,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) { var result map[string]any capturing := request capturing.CaptureExchange = true + capturing.CaptureDiagnostics = true response, err := client.CompleteStructured(context.Background(), capturing, &result) if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) { t.Fatalf("captured final response: %+v", response.Exchange) @@ -261,6 +262,7 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) { if err != nil { t.Fatal(err) } + req.CaptureDiagnostics = true response, err := client.CompleteStructured(context.Background(), req, &map[string]any{}) if err != nil { t.Fatal(err) diff --git a/internal/framework/outcomes/outcomes.go b/internal/framework/outcomes/outcomes.go index f824411..4edfbe7 100644 --- a/internal/framework/outcomes/outcomes.go +++ b/internal/framework/outcomes/outcomes.go @@ -121,3 +121,57 @@ func ExecutionRecoveries(counts ExecutionCounts) []RecoveryReason { } return reasons } + +// WorkTotals retains the facts shared by the CLI summary and ordinary reports. +type WorkTotals struct { + Total int + Started int + Completed int + WithErrors int + Corrections int + ModelOperations int + ProposalResponses int + Recoveries int + FilteringResponses int +} + +func (t *WorkTotals) Record(state WorkState, operations []Operation) { + if state != WorkNotStarted { + t.Started++ + } + if state == WorkCompleted { + t.Completed++ + } + hasError := false + for _, operation := range operations { + if operation.Reference.Kind != OperationValidationInput { + t.ModelOperations++ + } + if operation.Succeeded { + if operation.Reference.Kind == OperationProposal { + t.ProposalResponses++ + if len(operation.Filtered) > 0 { + t.FilteringResponses++ + } + } + if len(operation.RecoveryReasons) > 0 { + t.Recoveries++ + } + } + hasError = hasError || operation.FailureClass == FailureRecoverable + } + if hasError { + t.WithErrors++ + } +} + +func (t *WorkTotals) Add(module WorkTotals) { + t.Started += module.Started + t.Completed += module.Completed + t.WithErrors += module.WithErrors + t.Corrections += module.Corrections + t.ModelOperations += module.ModelOperations + t.ProposalResponses += module.ProposalResponses + t.Recoveries += module.Recoveries + t.FilteringResponses += module.FilteringResponses +} diff --git a/internal/framework/processreport/partial_validation_test.go b/internal/framework/processreport/partial_validation_test.go index 00fdd0b..9344665 100644 --- a/internal/framework/processreport/partial_validation_test.go +++ b/internal/framework/processreport/partial_validation_test.go @@ -74,7 +74,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) { cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}} result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, Evidence: &runner.EvidenceSelection{Reports: true, Debug: true}, }) if !errors.Is(err, origin) || client.calls != 3 { t.Fatalf("fatal cause/calls: %v %d", err, client.calls) diff --git a/internal/framework/processreport/report.go b/internal/framework/processreport/report.go index e9c88ff..cc37b5d 100644 --- a/internal/framework/processreport/report.go +++ b/internal/framework/processreport/report.go @@ -66,59 +66,35 @@ func Build(input BuildInput) reporting.Snapshot { return units[i].Section.Index < units[j].Section.Index }) } - total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0 - corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0 + var measured outcomes.WorkTotals + if input.RunOutput.Totals != nil { + measured = *input.RunOutput.Totals + } else { + measured.Total = len(units) + for _, unit := range units { + measured.Record(unit.State, unit.Operations) + measured.Corrections += unit.CorrectionsApplied + } + } for _, unit := range units { - if unit.State == outcomes.WorkNotStarted { - notStarted++ - } else { - started++ - } - if unit.State == outcomes.WorkCompleted { - completed++ - } - if unit.State == outcomes.WorkInterrupted { - interrupted++ - } - corrections += unit.CorrectionsApplied - operations := unit.Operations - if details { - operations = append([]outcomes.Operation(nil), operations...) - sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) }) + if !details { + break } + operations := append([]outcomes.Operation(nil), unit.Operations...) + sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) }) var failures []reporting.Failure unitHasError := false unitRecoveries := 0 for _, operation := range operations { - if operation.Reference.Kind != outcomes.OperationValidationInput { - llmOperations++ - } - if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { - proposalResponses++ - } if operation.Succeeded && len(operation.RecoveryReasons) > 0 { - recoveryCount++ unitRecoveries++ - if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal { - filteringResponses++ - } - if details { - snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation)) - } + snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation)) } if operation.FailureClass == outcomes.FailureRecoverable { unitHasError = true - if details { - failures = append(failures, mapFailure(operation)) - } + failures = append(failures, mapFailure(operation)) } } - if unitHasError { - withErrors++ - } - if !details { - continue - } row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID, State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries} @@ -153,12 +129,13 @@ func Build(input BuildInput) reporting.Snapshot { return a.ProposalIndex < b.ProposalIndex }) runErrors := len(failures) + interrupted, notStarted := measured.Started-measured.Completed, measured.Total-measured.Started snapshot.Summary = reporting.Summary{ - WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed, - WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &withErrors, - RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations, - ProposalResponses: &proposalResponses, Recoveries: &recoveryCount, - ProposalResponsesWithFiltering: &filteringResponses, + WorkUnitsTotal: &measured.Total, WorkUnitsStarted: &measured.Started, WorkUnitsCompleted: &measured.Completed, + WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &measured.WithErrors, + RunErrors: &runErrors, CorrectionsApplied: &measured.Corrections, LLMOperations: &measured.ModelOperations, + ProposalResponses: &measured.ProposalResponses, Recoveries: &measured.Recoveries, + ProposalResponsesWithFiltering: &measured.FilteringResponses, } } else if len(failures) > 0 { runErrors := len(failures) diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index a8613b7..ec4fab4 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -82,9 +82,10 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { ) coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)} completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - CaptureExchange: req.CaptureExchange, - PromptID: req.PromptID, - Variables: req.Variables, Schema: responseschema.CorrectionSetKey, + CaptureExchange: req.CaptureExchange, + CaptureDiagnostics: req.Interactions != nil, + PromptID: req.PromptID, + Variables: req.Variables, Schema: responseschema.CorrectionSetKey, Context: coordinates, }, &response) sectionIndex := 0 diff --git a/internal/framework/runner/cancellation_test.go b/internal/framework/runner/cancellation_test.go index e1c810a..b6b2980 100644 --- a/internal/framework/runner/cancellation_test.go +++ b/internal/framework/runner/cancellation_test.go @@ -207,7 +207,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ - Config: &cfg, CaptureCandidates: true, + Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true}, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, }) diff --git a/internal/framework/runner/evidence_test.go b/internal/framework/runner/evidence_test.go new file mode 100644 index 0000000..061b877 --- /dev/null +++ b/internal/framework/runner/evidence_test.go @@ -0,0 +1,94 @@ +package runner + +import ( + "context" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" +) + +func evidenceFixture(size int) (*Runner, RunInput) { + cfg := runnerTestConfig() + cfg.Chunking.MaxSectionTokens = 1 + cfg.Chunking.MinSectionTokens = 0 + transcript := &schema.Transcript{Segments: make([]schema.Segment, size)} + for i := range transcript.Segments { + transcript.Segments[i] = schema.Segment{ID: i + 1, Text: "word sample"} + } + validator := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) { + var decisions []validators.Decision + for _, candidate := range req.CandidateProposal { + decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}) + } + return validators.Result{Decisions: decisions}, nil + }} + module := outcomeModule{ + fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}}, + proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { + operation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}} + category := req.Section.Index % 4 + if category == 0 { + operation.FailureClass, operation.FailureReason = outcomes.FailureRecoverable, outcomes.ReasonTransportFailed + return contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, nil + } + operation.Succeeded = true + original := "word" + if category == 1 { + original = "absent" + } + if category == 2 { + operation.RecoveryReasons = []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered} + } + return contracts.ProposalResult{ + Operations: []outcomes.Operation{operation}, + Proposals: []proposals.CorrectionProposal{{TargetSegmentID: req.Section.StartSegmentID, OriginalText: original, CorrectedText: "Word"}}, + }, nil + }, + } + return New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}), RunInput{ + Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, + } +} + +func TestEvidenceSelectionKeepsTotalsAndRequestedRecords(t *testing.T) { + r, input := evidenceFixture(40) + want := outcomes.WorkTotals{Total: 40, Started: 40, Completed: 40, WithErrors: 10, Corrections: 20, + ModelOperations: 40, ProposalResponses: 30, Recoveries: 20, FilteringResponses: 10} + var baseline *schema.Transcript + for _, tc := range []struct { + name string + selection EvidenceSelection + units int + candidates int + }{ + {name: "summary"}, + {name: "reports", selection: EvidenceSelection{Reports: true}, units: 40}, + {name: "debug", selection: EvidenceSelection{Debug: true}, candidates: 30}, + {name: "reports and debug", selection: EvidenceSelection{Reports: true, Debug: true}, units: 40, candidates: 30}, + } { + t.Run(tc.name, func(t *testing.T) { + input.Evidence = &tc.selection + out, err := r.Run(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if out.Totals == nil || *out.Totals != want { + t.Fatalf("totals: got %+v, want %+v", out.Totals, want) + } + if len(out.WorkUnits) != tc.units || len(out.Candidates) != tc.candidates || + len(out.ModuleResults[0].ValidatorDecisions) != tc.candidates { + t.Fatalf("retained units=%d candidates=%d decisions=%d", len(out.WorkUnits), len(out.Candidates), len(out.ModuleResults[0].ValidatorDecisions)) + } + if baseline == nil { + baseline = out.FinalTranscript + } else if !reflect.DeepEqual(baseline, out.FinalTranscript) { + t.Fatal("output selection changed final corrections") + } + }) + } +} diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index 9278b76..ce05ced 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -161,7 +161,7 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair } proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1 - return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil + return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, CaptureDiagnostics: debug, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil } exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) { if turn.Response == nil { diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 4bed95b..d5fb2e3 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -45,8 +45,6 @@ type ModuleResult struct { ModuleInstance string `json:"module_instance"` ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` Status string `json:"status"` - ProposalCount int `json:"proposal_count"` - Operations []outcomes.Operation `json:"operations,omitempty"` ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"` ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"` AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` @@ -95,13 +93,32 @@ type ValidatorRejectedChange struct { Message string `json:"message"` } +// EvidenceSelection selects the detailed views consumed by the invocation. +type EvidenceSelection struct { + Reports bool + Debug bool +} + +type evidenceNeeds struct { + reports bool + decisions bool + candidates bool +} + +func selectEvidence(selection *EvidenceSelection) evidenceNeeds { + if selection == nil { + return evidenceNeeds{reports: true, decisions: true} + } + return evidenceNeeds{reports: selection.Reports, decisions: selection.Debug, candidates: selection.Debug} +} + // RunInput is the deterministic runner input. type RunInput struct { - Config *config.Config - Transcript *schema.Transcript - ModuleSpecs []contracts.ModuleRunSpec - Interactions structuredoutput.InteractionSink - CaptureCandidates bool + Config *config.Config + Transcript *schema.Transcript + ModuleSpecs []contracts.ModuleRunSpec + Interactions structuredoutput.InteractionSink + Evidence *EvidenceSelection } // RunOutput is the deterministic runner output. @@ -109,6 +126,7 @@ type RunOutput struct { FinalTranscript *schema.Transcript `json:"-"` ModuleResults []ModuleResult `json:"module_results"` Sections []contracts.SectionMetadata `json:"sections"` + Totals *outcomes.WorkTotals `json:"-"` InventoryAvailable bool `json:"-"` WorkUnits []WorkUnitOutcome `json:"work_units"` Candidates []proposals.EnrichedCorrectionProposal `json:"-"` @@ -132,54 +150,70 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { if err != nil { return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)} } - inventory := make([]contracts.SectionMetadata, len(sections)) - workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections)) + evidence := selectEvidence(input.Evidence) + totals := outcomes.WorkTotals{Total: len(input.ModuleSpecs) * len(sections)} + var inventory []contracts.SectionMetadata + var workUnits []WorkUnitOutcome var candidates []proposals.EnrichedCorrectionProposal - for i, section := range sections { - inventory[i] = contracts.SectionMetadataFromSection(section) - } - for moduleIndex, spec := range input.ModuleSpecs { - for _, section := range inventory { - workUnits = append(workUnits, WorkUnitOutcome{ - ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, - ModuleInstance: spec.InstanceName, Section: section, - State: outcomes.WorkNotStarted, - }) + if evidence.reports { + inventory = make([]contracts.SectionMetadata, len(sections)) + for i, section := range sections { + inventory[i] = contracts.SectionMetadataFromSection(section) + } + workUnits = make([]WorkUnitOutcome, 0, totals.Total) + for moduleIndex, spec := range input.ModuleSpecs { + for _, section := range inventory { + workUnits = append(workUnits, newWorkUnit(spec, section, moduleIndex)) + } } } snapshot := func() RunOutput { return RunOutput{ FinalTranscript: transcriptFromSections(sections), ModuleResults: results, - Sections: inventory, InventoryAvailable: true, WorkUnits: workUnits, Candidates: candidates, + Sections: inventory, InventoryAvailable: true, Totals: &totals, WorkUnits: workUnits, Candidates: candidates, } } for moduleIndex, spec := range input.ModuleSpecs { - execution, err := r.executeModule(ctx, spec, sections, input.Interactions) - units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] - copy(units, execution.pipeline.WorkUnits) - if err == nil && ctx.Err() != nil { - err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil) + var reportUnits []WorkUnitOutcome + if evidence.reports { + reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] } + execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits) + units := execution.pipeline.WorkUnits if err == nil { - previous := sections - sections = execution.updated - attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units) + filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy) + if canceled := ctx.Err(); canceled != nil { + err = locateFailure(canceled, "application", failureReference(spec, nil, ""), false, nil) + } else { + execution.pipeline.recordApplyFiltering(filters, evidence.reports) + sections = execution.updated + execution.pipeline.Totals.Started = len(sections) + execution.pipeline.Totals.Completed = len(sections) + execution.pipeline.Totals.Corrections = len(execution.applied) + } } - var failure *ExecutionFailure applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application" - appliedIndices := make(map[int]struct{}, len(execution.applied)) - for _, change := range execution.applied { - appliedIndices[change.ProposalIndex] = struct{}{} + if applicationInterrupted { + execution.pipeline.Totals.Completed = 0 + } + totals.Add(execution.pipeline.Totals) + + appliedIndices := make(map[int]struct{}) + if evidence.reports || evidence.candidates { + for _, change := range execution.applied { + appliedIndices[change.ProposalIndex] = struct{}{} + } } - var operations []outcomes.Operation for i := range units { unit := &units[i] unit.ModuleIndex = moduleIndex if err == nil { unit.State = outcomes.WorkCompleted - unit.CorrectionsApplied = execution.appliedBySection[i] + if evidence.reports { + unit.CorrectionsApplied = execution.appliedBySection[unit.Section.Index] + } } else if applicationInterrupted && unit.State != outcomes.WorkNotStarted { unit.State = outcomes.WorkInterrupted } @@ -196,16 +230,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { branch.Outcome = "approved_skipped" } } - operations = append(operations, unit.Operations...) } - - if input.CaptureCandidates { - candidates = append(candidates, execution.pipeline.Candidates...) + if !evidence.reports && evidence.candidates { + workUnits = append(workUnits, units...) } + candidates = append(candidates, execution.pipeline.Candidates...) result := ModuleResult{ StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy, - ProposalCount: execution.pipeline.ProposalCount, Operations: operations, ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected, } if err != nil { @@ -213,8 +245,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { result.ErrorMessage = err.Error() } else { result.Status = ModuleStatusSuccess - result.AppliedChanges = execution.applied - result.SkippedChanges = execution.skipped + if evidence.decisions { + result.AppliedChanges = execution.applied + result.SkippedChanges = execution.skipped + } } results = append(results, result) if err != nil { @@ -237,7 +271,7 @@ type moduleExecution struct { appliedBySection []int } -func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) { +func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) { var execution moduleExecution origin := failureReference(spec, nil, "") if err := ctx.Err(); err != nil { @@ -253,6 +287,8 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec } execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{ Module: module, + Evidence: &evidence, + ReportUnits: reportUnits, Spec: spec, Policy: execution.policy, Sections: sections, @@ -323,6 +359,8 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric } type collectSectionProposalsInput struct { + Evidence *evidenceNeeds + ReportUnits []WorkUnitOutcome Interactions structuredoutput.InteractionSink Module contracts.TranscriptModule Spec contracts.ModuleRunSpec @@ -356,16 +394,18 @@ type sectionValidationResult struct { } type sectionWork struct { + unit WorkUnitOutcome candidates []proposals.EnrichedCorrectionProposal validation *sectionValidationResult finished bool } type modulePipelineResult struct { - ProposalCount int + Totals outcomes.WorkTotals + proposalFacts map[int]proposalRecovery + revised map[int]struct{} Candidates []proposals.EnrichedCorrectionProposal Approved []proposals.EnrichedCorrectionProposal - Operations []outcomes.Operation WorkUnits []WorkUnitOutcome ValidatorDecisions []ValidatorDecisionRecord ValidatorRejected []ValidatorRejectedChange @@ -410,17 +450,16 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec } func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) { - out := modulePipelineResult{ - Approved: make([]proposals.EnrichedCorrectionProposal, 0), - Operations: make([]outcomes.Operation, 0), - WorkUnits: make([]WorkUnitOutcome, len(input.Sections)), - ValidatorDecisions: make([]ValidatorDecisionRecord, 0), - ValidatorRejected: make([]ValidatorRejectedChange, 0), + var out modulePipelineResult + evidence := selectEvidence(nil) + if input.Evidence != nil { + evidence = *input.Evidence } - for i, section := range input.Sections { - out.WorkUnits[i] = WorkUnitOutcome{ - ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, StageIndex: input.Spec.Index, StageID: input.Spec.ID, - Section: contracts.SectionMetadataFromSection(section), State: outcomes.WorkNotStarted, + out.WorkUnits = input.ReportUnits + if evidence.reports && out.WorkUnits == nil { + out.WorkUnits = make([]WorkUnitOutcome, len(input.Sections)) + for i, section := range input.Sections { + out.WorkUnits[i] = newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(section), 0) } } @@ -487,17 +526,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) return sectionEnriched } - recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) { - unit := &out.WorkUnits[sectionPos] + recordCandidates := func(unit *WorkUnitOutcome, enriched []proposals.EnrichedCorrectionProposal) { unit.ProposalIndexOffset = nextProposalIndex - for _, candidate := range enriched { - candidate.ProposalIndex += nextProposalIndex - out.Candidates = append(out.Candidates, candidate) + if evidence.candidates { + for _, candidate := range enriched { + candidate.ProposalIndex += nextProposalIndex + out.Candidates = append(out.Candidates, candidate) + } } - out.ProposalCount += len(enriched) for operationIndex := range unit.Operations { operation := &unit.Operations[operationIndex] - if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { + if evidence.reports && operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { for _, proposal := range enriched { operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex) } @@ -516,16 +555,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) interactions = capture } validationInput := validateSectionCandidatesInput{ - Spec: input.Spec, - Policy: input.Policy, - WorkingTranscript: transcriptFromSection(sectionResult.section), - SectionIndex: §ionMetadata.Index, - Validators: validatorsOrdered, - SectionEnriched: enriched, - Exchange: sectionResult.proposal.Exchange, - ValidationLLMClient: input.ValidationClient, - GlossaryView: input.GlossaryView, - Interactions: interactions, + Spec: input.Spec, + SuppressDecisionHistory: !evidence.decisions, + Policy: input.Policy, + WorkingTranscript: transcriptFromSection(sectionResult.section), + SectionIndex: §ionMetadata.Index, + Validators: validatorsOrdered, + SectionEnriched: enriched, + Exchange: sectionResult.proposal.Exchange, + ValidationLLMClient: input.ValidationClient, + GlossaryView: input.GlossaryView, + Interactions: interactions, } if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 { validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript) @@ -555,8 +595,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) failure: failure, } } - recordValidation := func(res sectionValidationResult) { - unit := &out.WorkUnits[res.sectionPos] + recordValidation := func(unit *WorkUnitOutcome, res sectionValidationResult) { unit.Operations = append(unit.Operations, res.operations...) unit.Repairs = res.repairs mergeFilteredProposals(unit, res.filtered) @@ -567,18 +606,30 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) unit.State = outcomes.WorkInterrupted } out.Approved = append(out.Approved, res.approved...) - out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...) - out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...) + if evidence.decisions { + out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...) + out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...) + } + if len(res.approved) > 0 { + out.rememberProposalRecovery(unit) + } } nextAdmission, active := 0, 0 for { for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] { offset := nextProposalIndex - recordCandidates(nextSectionToRecord, work.candidates) + recordCandidates(&work.unit, work.candidates) if work.validation != nil { work.validation.assignProposalIndices(offset) - recordValidation(*work.validation) + recordValidation(&work.unit, *work.validation) + } + out.Totals.Record(work.unit.State, work.unit.Operations) + if evidence.reports { + out.WorkUnits[nextSectionToRecord] = work.unit + } else if evidence.candidates && len(work.unit.Repairs) > 0 { + work.unit.Operations = nil + out.WorkUnits = append(out.WorkUnits, work.unit) } delete(pending, nextSectionToRecord) nextSectionToRecord++ @@ -586,7 +637,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil { sectionPos := nextAdmission nextAdmission++ - pending[sectionPos] = §ionWork{} + pending[sectionPos] = §ionWork{unit: newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(input.Sections[sectionPos]), 0)} active++ workers.Go(func() { proposed := proposeSection(runCtx, input, sectionPos) @@ -600,13 +651,13 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) select { case result := <-proposalResults: active-- - unit := &out.WorkUnits[result.sectionPos] + work := pending[result.sectionPos] + unit := &work.unit unit.Operations = append(unit.Operations, result.proposal.Operations...) if result.started { unit.State = outcomes.WorkInterrupted } unit.Failure = result.failure - work := pending[result.sectionPos] if result.started && result.failure == nil { work.candidates = makeCandidates(result) if runCtx.Err() == nil { @@ -633,10 +684,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex }) - for _, unit := range out.WorkUnits { - out.Operations = append(out.Operations, unit.Operations...) - } - sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool { if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex { return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex @@ -668,19 +715,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } type validateSectionCandidatesInput struct { - Section *validators.SectionLookup - RepairProposalIndex *int - RepairAttempt *int - Exchange *structuredoutput.Exchange - Spec contracts.ModuleRunSpec - Policy proposals.ReplacementPolicy - WorkingTranscript *schema.Transcript - SectionIndex *int - Validators []validators.Validator - SectionEnriched []proposals.EnrichedCorrectionProposal - ValidationLLMClient contracts.StructuredLLMClient - Interactions structuredoutput.InteractionSink - GlossaryView *validators.GlossaryView + SuppressDecisionHistory bool + Section *validators.SectionLookup + RepairProposalIndex *int + RepairAttempt *int + Exchange *structuredoutput.Exchange + Spec contracts.ModuleRunSpec + Policy proposals.ReplacementPolicy + WorkingTranscript *schema.Transcript + SectionIndex *int + Validators []validators.Validator + SectionEnriched []proposals.EnrichedCorrectionProposal + ValidationLLMClient contracts.StructuredLLMClient + Interactions structuredoutput.InteractionSink + GlossaryView *validators.GlossaryView } type validateSectionCandidatesResult struct { @@ -781,16 +829,18 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida noEffect[d.ProposalIndex] = true } } - decisions = append(decisions, ValidatorDecisionRecord{ - ValidatorName: validator.Name(), - ExecutionClass: class, - ProposalIndex: d.ProposalIndex, - Approved: approved, - Disposition: disposition, - ReasonCode: d.ReasonCode, - Message: d.Message, - DiagnosticArtifactPath: d.DiagnosticArtifactPath, - }) + if !input.SuppressDecisionHistory { + decisions = append(decisions, ValidatorDecisionRecord{ + ValidatorName: validator.Name(), + ExecutionClass: class, + ProposalIndex: d.ProposalIndex, + Approved: approved, + Disposition: disposition, + ReasonCode: d.ReasonCode, + Message: d.Message, + DiagnosticArtifactPath: d.DiagnosticArtifactPath, + }) + } if approved { nextEligible = append(nextEligible, byIndex[d.ProposalIndex]) continue @@ -800,19 +850,21 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1}) } } - p := byIndex[d.ProposalIndex] - rejected = append(rejected, ValidatorRejectedChange{ - ValidatorName: validator.Name(), - ProposalIndex: p.ProposalIndex, - ModuleKey: p.ModuleKey, - ModuleInstance: p.ModuleInstance, - TargetSegmentID: p.TargetSegmentID, - OriginalText: p.OriginalText, - CorrectedText: p.CorrectedText, - Disposition: disposition, - ReasonCode: d.ReasonCode, - Message: d.Message, - }) + if feedback || !input.SuppressDecisionHistory { + p := byIndex[d.ProposalIndex] + rejected = append(rejected, ValidatorRejectedChange{ + ValidatorName: validator.Name(), + ProposalIndex: p.ProposalIndex, + ModuleKey: p.ModuleKey, + ModuleInstance: p.ModuleInstance, + TargetSegmentID: p.TargetSegmentID, + OriginalText: p.OriginalText, + CorrectedText: p.CorrectedText, + Disposition: disposition, + ReasonCode: d.ReasonCode, + Message: d.Message, + }) + } } if err != nil { @@ -877,9 +929,9 @@ func mergeOperationFiltering(operation *outcomes.Operation, filtered []outcomes. } } -func attributeApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy, units []WorkUnitOutcome) { +func findApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy) map[int][]outcomes.FilteredProposal { if len(skipped) == 0 { - return + return nil } byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved)) sectionPosition := make(map[int]int, len(sections)) @@ -916,26 +968,7 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E } filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1}) } - for pos, items := range filtered { - unit := &units[pos] - revisions := make(map[int]*outcomes.Operation) - for i := range unit.Operations { - op := &unit.Operations[i] - if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Succeeded && op.Reference.ProposalIndex != nil && - slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) { - revisions[*op.Reference.ProposalIndex] = op - } - } - original := items[:0] - for _, item := range items { - if op := revisions[item.ProposalIndex]; op != nil { - mergeOperationFiltering(op, []outcomes.FilteredProposal{item}) - } else { - original = append(original, item) - } - } - mergeFilteredProposals(unit, original) - } + return filtered } func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) { diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 7cae0cf..1c41b90 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -173,7 +173,7 @@ func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) { if len(without.Candidates) != 0 { t.Fatalf("debug-off candidate capture: %+v", without.Candidates) } - input.CaptureCandidates = true + input.Evidence = &EvidenceSelection{Reports: true, Debug: true} with, err := r.Run(context.Background(), input) if err != nil { t.Fatal(err) @@ -844,7 +844,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) { }, }, }}) - out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, CaptureCandidates: true}) + out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, Evidence: &EvidenceSelection{Reports: true, Debug: true}}) if err != nil { t.Fatal(err) } @@ -973,8 +973,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress( if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 { t.Fatalf("expected second module rejection, got %+v", out.ModuleResults) } - if len(out.ModuleResults[1].Operations) != 1 || out.ModuleResults[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid { - t.Fatalf("expected invalid decision-set operation, got %+v", out.ModuleResults[1].Operations) + if len(out.WorkUnits[1].Operations) != 1 || out.WorkUnits[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid { + t.Fatalf("expected invalid decision-set operation, got %+v", out.WorkUnits[1].Operations) } } diff --git a/internal/framework/runner/window_test.go b/internal/framework/runner/window_test.go index 4cedfd3..1d95fc9 100644 --- a/internal/framework/runner/window_test.go +++ b/internal/framework/runner/window_test.go @@ -218,7 +218,7 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) { cfg.Chunking.MaxSectionTokens = 1 cfg.Chunking.MinSectionTokens = 0 out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ - Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true, + Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true}, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation, diff --git a/internal/framework/runner/work_totals.go b/internal/framework/runner/work_totals.go new file mode 100644 index 0000000..3fd0d84 --- /dev/null +++ b/internal/framework/runner/work_totals.go @@ -0,0 +1,78 @@ +package runner + +import ( + "slices" + + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" +) + +func newWorkUnit(spec contracts.ModuleRunSpec, section contracts.SectionMetadata, moduleIndex int) WorkUnitOutcome { + return WorkUnitOutcome{ + ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, + ModuleInstance: spec.InstanceName, Section: section, State: outcomes.WorkNotStarted, + } +} + +type proposalRecovery struct { + recovered bool + filtered bool +} + +// Application can discover additional defects. Keep only the proposal operation +// flags needed to update totals and the identities of approved revisions. +func (p *modulePipelineResult) rememberProposalRecovery(unit *WorkUnitOutcome) { + for _, operation := range unit.Operations { + if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { + if p.proposalFacts == nil { + p.proposalFacts = make(map[int]proposalRecovery) + } + p.proposalFacts[unit.Section.Index] = proposalRecovery{ + recovered: len(operation.RecoveryReasons) > 0, filtered: len(operation.Filtered) > 0, + } + break + } + } + for _, repair := range unit.Repairs { + if repair.Outcome == "approved" { + if p.revised == nil { + p.revised = make(map[int]struct{}) + } + p.revised[repair.ProposalIndex] = struct{}{} + } + } +} + +func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.FilteredProposal, reports bool) { + for pos, items := range filters { + original := items[:0] + for _, item := range items { + if _, repaired := p.revised[item.ProposalIndex]; !repaired { + original = append(original, item) + continue + } + if reports { + operations := p.WorkUnits[pos].Operations + for i := range operations { + op := &operations[i] + if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil && + *op.Reference.ProposalIndex == item.ProposalIndex && slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) { + mergeOperationFiltering(op, []outcomes.FilteredProposal{item}) + break + } + } + } + } + if facts, ok := p.proposalFacts[pos]; ok && len(original) != 0 { + if !facts.recovered { + p.Totals.Recoveries++ + } + if !facts.filtered { + p.Totals.FilteringResponses++ + } + if reports { + mergeFilteredProposals(&p.WorkUnits[pos], original) + } + } + } +} diff --git a/internal/framework/structuredoutput/call.go b/internal/framework/structuredoutput/call.go index b48ecdf..da5eed0 100644 --- a/internal/framework/structuredoutput/call.go +++ b/internal/framework/structuredoutput/call.go @@ -26,11 +26,12 @@ type DiagnosticContext struct { // Request names an embedded prompt and response schema. Profile selection and // credentials are supplied by the role-bound runtime client. type Request struct { - PromptID string `json:"prompt_id"` - Variables map[string]string `json:"variables"` - Schema responseschema.Key `json:"schema"` - Context DiagnosticContext `json:"context"` - CaptureExchange bool `json:"-"` + PromptID string `json:"prompt_id"` + Variables map[string]string `json:"variables"` + Schema responseschema.Key `json:"schema"` + Context DiagnosticContext `json:"context"` + CaptureExchange bool `json:"-"` + CaptureDiagnostics bool `json:"-"` } // PromptMetadata is the execution snapshot returned by the runtime when available. diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 86b3bc0..fe61444 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -130,7 +130,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{ PromptID: promptMetadata.PromptID, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, - Context: coordinates, + Context: coordinates, CaptureDiagnostics: req.Interactions != nil, }, &response) batchIndex := batch.BatchIndex operation := outcomes.Operation{