From 71a9b1f560587699743bd27f465788172cb5a5fe Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 10 Nov 2026 20:48:32 +0000 Subject: [PATCH] Skip report and debug work for disabled outputs --- cmd/audita/main_integration_test.go | 4 +- docs/roadmap/remediation.md | 2 + internal/cli/process_inputs.go | 39 ++++++----- .../cli/process_ordered_integration_test.go | 5 +- internal/cli/process_output.go | 34 ++++++---- internal/cli/process_pipeline.go | 17 +++-- internal/framework/llm/adapter.go | 39 ++++++----- internal/framework/processreport/report.go | 66 +++++++++++++------ .../framework/processreport/report_test.go | 10 ++- .../framework/proposal_generation/generate.go | 23 ++++--- .../framework/validators/llm_validators.go | 22 ++++--- 11 files changed, 161 insertions(+), 100 deletions(-) diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 931fa91..f304b77 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -871,7 +871,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) { for _, selection := range []struct{ reporting, debug bool }{ {false, false}, {false, true}, {true, false}, {true, true}, } { - f := newProcessFixture(t, "normal", "[grammar]") + f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]") reportDir := filepath.Join(f.dir, "missing") extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir} @@ -886,7 +886,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) { extra = append(extra, "--debug") } code, stdout, stderr := f.run(t, true, extra...) - if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") { + if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") { t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr) } output, err := os.ReadFile(f.output) diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index ff1006e..8753a4c 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -640,6 +640,8 @@ numbering and punctuation or escaped text at encoding boundaries. ## R18 — P3: Skip Evidence Work for Disabled Outputs +**Status:** Completed. Disabled reporting skips report payloads and detailed rows while retaining completion counts. Detailed observation hooks require an active debug writer. Subprocess coverage compares output and nonzero correction, error, and recovery counts across all four settings. + **Category** Unused report rendering and event collection in Stages 8 and 13. diff --git a/internal/cli/process_inputs.go b/internal/cli/process_inputs.go index bc52f29..65cdbf3 100644 --- a/internal/cli/process_inputs.go +++ b/internal/cli/process_inputs.go @@ -87,24 +87,31 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu return nil, err } started := time.Now().UTC() - snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value} - for index, item := range cfg.Pipeline.Stages { - snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID)) + var secrets []string + if cfg.Reporting.Enabled || input.Debug.Value { + secrets = loaded.Runtime.ConfiguredSecrets(ctx) } - secrets := loaded.Runtime.ConfiguredSecrets(ctx) - initial, err := renderReportBundle(snapshot, secrets...) - if err != nil { - return nil, err - } - failed := func(primary error, failures []diagnostics.FileFailure) diagnostics.ReportBundle { - failedSnapshot := snapshot - completed := time.Now().UTC() - failedSnapshot.CompletedAt = &completed - if primary != nil { - failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}} + var initial diagnostics.ReportBundle + var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle + if cfg.Reporting.Enabled { + snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value} + for index, item := range cfg.Pipeline.Stages { + snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID)) + } + initial, err = renderReportBundle(snapshot, secrets...) + if err != nil { + return nil, err + } + failed = func(primary error, failures []diagnostics.FileFailure) diagnostics.ReportBundle { + failedSnapshot := snapshot + completed := time.Now().UTC() + failedSnapshot.CompletedAt = &completed + if primary != nil { + failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}} + } + bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...) + return bundle } - bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...) - return bundle } var debug *preflight.DebugInput var warnings []error diff --git a/internal/cli/process_ordered_integration_test.go b/internal/cli/process_ordered_integration_test.go index 0b7cf39..9c32354 100644 --- a/internal/cli/process_ordered_integration_test.go +++ b/internal/cli/process_ordered_integration_test.go @@ -73,7 +73,6 @@ func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) { dir := t.TempDir() files := map[string]string{ "config.yml": `version: 3 -reporting: {enabled: false} glossary_collections: party: entries: [{id: alice, name: Alice, category: person}] @@ -117,6 +116,7 @@ pipeline: } inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source} loaded.Config.Diagnostics.WorkDir = dir + loaded.Config.Reporting.StdoutDir = dir prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json"), Debug: commandkit.Supplied[bool]{Value: true}}, inputs, &bytes.Buffer{}) if err != nil { t.Fatal(err) @@ -159,7 +159,7 @@ pipeline: func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) { dir := t.TempDir() files := map[string]string{ - "config.yml": "version: 3\nreporting: {enabled: false}\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n", + "config.yml": "version: 3\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n", "prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n", "transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`, } @@ -184,6 +184,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) { t.Fatal(err) } inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source} + loaded.Config.Reporting.StdoutDir = dir prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{}) if err != nil { t.Fatal(err) diff --git a/internal/cli/process_output.go b/internal/cli/process_output.go index aa548a0..d932c34 100644 --- a/internal/cli/process_output.go +++ b/internal/cli/process_output.go @@ -31,31 +31,39 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces if primary != nil { snapshot = failedProcessingSnapshot(snapshot, primary, inv.Context.Err()) } - reports, err := renderReportBundle(snapshot, secrets...) - if err != nil { - if primary == nil { - primary = err + reportPaths := prepared.Prepared.Output.ReportPaths() + var reports diagnostics.ReportBundle + var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle + if len(reportPaths) > 0 { + var err error + reports, err = renderReportBundle(snapshot, secrets...) + if err != nil { + if primary == nil { + primary = err + } + snapshot.Status = reporting.StatusFailed } - snapshot.Status = reporting.StatusFailed - } - failed := func(failures []diagnostics.FileFailure) diagnostics.ReportBundle { - result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...) - if bundleErr != nil { - return reports + failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle { + result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...) + if bundleErr != nil { + return reports + } + return result } - return result } if canceled := inv.Context.Err(); primary == nil && canceled != nil { primary = canceled snapshot = failedProcessingSnapshot(snapshot, canceled, canceled) - reports = failed(nil) + if failed != nil { + reports = failed(nil) + } } result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed) if result != nil { return commandkit.Failure(safe.String(result.Error())).WithCause(result) } _ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot)) - for _, path := range prepared.Prepared.Output.ReportPaths() { + for _, path := range reportPaths { _ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path))) } if prepared.Prepared.Debug != nil { diff --git a/internal/cli/process_pipeline.go b/internal/cli/process_pipeline.go index 266ce54..8ea35ae 100644 --- a/internal/cli/process_pipeline.go +++ b/internal/cli/process_pipeline.go @@ -20,14 +20,20 @@ import ( func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) { cfg := &loaded.Configuration.Config specs := contracts.ResolveModuleRunSpecs(cfg) - observer := llm.NewObservationCollector() + debug := prepared.Prepared.Debug + var observer *llm.ObservationCollector + var hooks llm.CallHooks + if debug != nil { + observer = llm.NewObservationCollector() + hooks = observer.Hooks() + } for i := range specs { item := &specs[i] - proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, observer.Hooks()) + proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, hooks) if err != nil { return processResult{}, fmt.Errorf("%s proposal client: %w", item.InstanceName, err) } - validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, observer.Hooks()) + validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, hooks) if err != nil { return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err) } @@ -39,7 +45,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath, CaptureCandidates: debugPath != "", ReportDiagnosticError: warn, }) - if debug := prepared.Prepared.Debug; debug != nil { + if debug != nil { warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot()))) warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output}))) } @@ -61,6 +67,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa } failures = append(failures, failure) } - snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, ModuleSpecs: specs, RunOutput: &output, RunFailures: failures}) + snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, + ModuleSpecs: specs, RunOutput: &output, RunFailures: failures, SummaryOnly: !cfg.Reporting.Enabled}) return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr } diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index 20edfbf..df350e8 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -69,26 +69,29 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou } terminalMu.Unlock() }() - record := structuredoutput.CallRecord{CallID: c.runtime.nextCallID.Add(1), Context: copyContext(req.Context), Started: time.Now().UTC()} - record.Context.Role = c.role - record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID} - if c.hooks.OnCall != nil { - c.hooks.OnCall(record) + var record structuredoutput.CallRecord + if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil { + record.CallID = c.runtime.nextCallID.Add(1) + record.Context = copyContext(req.Context) + record.Context.Role = c.role } - defer func() { - record.Finished = time.Now().UTC() - if err != nil { - var failure *structuredoutput.Failure - if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted { - record.Failure = structuredoutput.FailureMalformedOutput - } else { - record.Failure = structuredoutput.FailureRuntime + if onCall := c.hooks.OnCall; onCall != nil { + record.Started = time.Now().UTC() + record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID} + onCall(record) + defer func() { + record.Finished = time.Now().UTC() + if err != nil { + var failure *structuredoutput.Failure + if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted { + record.Failure = structuredoutput.FailureMalformedOutput + } else { + record.Failure = structuredoutput.FailureRuntime + } } - } - if c.hooks.OnCall != nil { - c.hooks.OnCall(record) - } - }() + onCall(record) + }() + } if parent.Err() != nil { return response, classifiedFailure(parent, ctx, parent.Err(), nil) } diff --git a/internal/framework/processreport/report.go b/internal/framework/processreport/report.go index 26a9ff7..aa02fb3 100644 --- a/internal/framework/processreport/report.go +++ b/internal/framework/processreport/report.go @@ -29,6 +29,7 @@ type BuildInput struct { ModuleSpecs []contracts.ModuleRunSpec RunOutput *runner.RunOutput RunFailures []RunFailure + SummaryOnly bool } // Build maps processing facts once into an immutable presentation snapshot. @@ -36,20 +37,28 @@ func Build(input BuildInput) reporting.Snapshot { snapshot := reporting.Snapshot{ RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt, TranscriptPath: input.TranscriptPath, OutputPath: input.OutputPath, - StageNames: make([]string, len(input.ModuleSpecs)), Errors: make([]reporting.ErrorRecord, 0), - Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0), } - for i, spec := range input.ModuleSpecs { - snapshot.StageNames[i] = reporting.StageLabel(spec.Index, spec.ModuleKey, spec.ID) + details := !input.SummaryOnly + if details { + snapshot.StageNames = make([]string, len(input.ModuleSpecs)) + snapshot.Errors = make([]reporting.ErrorRecord, 0) + snapshot.Recoveries = make([]reporting.RecoveryRecord, 0) + snapshot.WorkUnits = make([]reporting.WorkUnit, 0) + for i, spec := range input.ModuleSpecs { + snapshot.StageNames[i] = reporting.StageLabel(spec.Index, spec.ModuleKey, spec.ID) + } } if input.RunOutput != nil && input.RunOutput.InventoryAvailable { - units := append([]runner.WorkUnitOutcome(nil), input.RunOutput.WorkUnits...) - sort.SliceStable(units, func(i, j int) bool { - if units[i].ModuleIndex != units[j].ModuleIndex { - return units[i].ModuleIndex < units[j].ModuleIndex - } - return units[i].Section.Index < units[j].Section.Index - }) + units := input.RunOutput.WorkUnits + if details { + units = append([]runner.WorkUnitOutcome(nil), units...) + sort.SliceStable(units, func(i, j int) bool { + if units[i].ModuleIndex != units[j].ModuleIndex { + return units[i].ModuleIndex < units[j].ModuleIndex + } + return units[i].Section.Index < units[j].Section.Index + }) + } total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0 corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0 for _, unit := range units { @@ -65,12 +74,14 @@ func Build(input BuildInput) reporting.Snapshot { interrupted++ } corrections += unit.CorrectionsApplied - row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, - ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID, - State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied} - operations := append([]outcomes.Operation(nil), unit.Operations...) - sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) }) - failures := make([]reporting.Failure, 0) + operations := unit.Operations + if details { + operations = append([]outcomes.Operation(nil), operations...) + sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) }) + } + var failures []reporting.Failure + unitHasError := false + unitRecoveries := 0 for _, operation := range operations { if operation.Reference.Kind != outcomes.OperationValidationInput { llmOperations++ @@ -80,18 +91,31 @@ func Build(input BuildInput) reporting.Snapshot { } if operation.Succeeded && len(operation.RecoveryReasons) > 0 { recoveryCount++ - row.RecoveryCount++ + unitRecoveries++ if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal { filteringResponses++ } - snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation)) + if details { + snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation)) + } } if operation.FailureReason != "" && isRecoverableReason(operation.FailureReason) { - failures = append(failures, mapFailure(operation)) + unitHasError = true + if details { + failures = append(failures, mapFailure(operation)) + } } } - if len(failures) > 0 { + if unitHasError { withErrors++ + } + if !details { + continue + } + row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, + ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID, + State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries} + if unitHasError { row.ErrorCount = 1 stage, chunk, start, end, applied := unit.ModuleIndex, unit.Section.Index, unit.Section.StartSegmentID, unit.Section.EndSegmentID, unit.CorrectionsApplied snapshot.Errors = append(snapshot.Errors, reporting.ErrorRecord{Scope: "work_unit", StageIndex: &stage, StageID: unit.StageID, diff --git a/internal/framework/processreport/report_test.go b/internal/framework/processreport/report_test.go index a8f2cbc..333ac9c 100644 --- a/internal/framework/processreport/report_test.go +++ b/internal/framework/processreport/report_test.go @@ -27,8 +27,14 @@ func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) { {ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted}, {ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}}, }} - snapshot := Build(BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output}) + input := BuildInput{RunID: "run-1", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar_1"}, {ModuleKey: "grammar", InstanceName: "grammar_2"}}, RunOutput: output} + snapshot := Build(input) + input.SummaryOnly = true + summary := Build(input) + if !reflect.DeepEqual(snapshot.Summary, summary.Summary) || len(summary.Errors)+len(summary.Recoveries)+len(summary.WorkUnits) != 0 { + t.Fatalf("summary-only snapshot changed counts or retained details: %+v", summary) + } if snapshot.Summary.WorkUnitsTotal == nil || *snapshot.Summary.WorkUnitsTotal != 2 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 1 || *snapshot.Summary.LLMOperations != 2 { t.Fatalf("summary: %+v", snapshot.Summary) } diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 9ed1ecb..a3c0923 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -123,18 +123,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { Counts: completion.Execution, } - requestMetadata := map[string]any{ - "stage_index": req.StageIndex, "stage_id": req.StageID, - "module_key": req.ModuleKey, - "module_instance": req.ModuleInstance, - "replacement_policy": req.ReplacementPolicy, - "section": req.Section, - "start_index": req.StartIndex, - "segment_ids": segmentIDs(req.WorkingTranscript), - "prompt": completion.Prompt, - } - requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap() - var writer InteractionDiagnosticsWriter if req.DiagnosticsWriter != nil { writer = req.DiagnosticsWriter @@ -154,6 +142,17 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { } if writer != nil { + requestMetadata := map[string]any{ + "stage_index": req.StageIndex, "stage_id": req.StageID, + "module_key": req.ModuleKey, + "module_instance": req.ModuleInstance, + "replacement_policy": req.ReplacementPolicy, + "section": req.Section, + "start_index": req.StartIndex, + "segment_ids": segmentIDs(req.WorkingTranscript), + "prompt": completion.Prompt, + } + requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap() var diagnosticErr error artifacts, diagnosticErr = writer.WriteInteraction( stage, diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index bf2a2d4..baba17e 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -124,13 +124,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, var response LLMValidationResponse var completion StructuredCompletionResponse - responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) promptMetadata := validatorPromptMetadata(v.validatorType) - stageInstance := req.ModuleInstance - if req.SectionIndex != nil { - stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex) - } - stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex) completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{ PromptID: promptMetadata.PromptID, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, @@ -142,10 +136,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, Counts: completion.Execution, } - writeInteraction := func(responsePayload any, callErr error) InteractionArtifacts { + writeInteraction := func(reconciled *validationReconciliation, callErr error) InteractionArtifacts { if req.DiagnosticsWriter == nil { return InteractionArtifacts{} } + stageInstance := req.ModuleInstance + if req.SectionIndex != nil { + stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex) + } + stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex) + responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) + responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model} + if reconciled != nil { + responsePayload["reconciliation"] = *reconciled + } var diagnosticErr error artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction( stage, @@ -168,7 +172,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, return artifacts } if err != nil { - writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, err) + writeInteraction(nil, err) var failure *structuredoutput.Failure if errors.As(err, &failure) { operation.FailureReason = failure.Reason @@ -186,7 +190,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } reconciled := reconcileLLMResponse(batch.Items, response) - artifacts := writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model, "reconciliation": reconciled}, nil) + artifacts := writeInteraction(&reconciled, nil) if len(reconciled.UnavailableIndices) > 0 { operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid operation.ProposalIndices = reconciled.UnavailableIndices