From b98d463958ad66d03c0e2256dd7d8b0ec81cbc2f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 14:53:14 +0000 Subject: [PATCH] Bound concurrent section work and finalize proposal identities in order --- docs/internal/pipeline.md | 11 +- docs/roadmap/implementation.md | 15 + internal/cli/process_pipeline.go | 3 +- internal/framework/processreport/llm.go | 28 ++ .../framework/runner/cancellation_test.go | 19 +- internal/framework/runner/repair.go | 43 +- internal/framework/runner/runner.go | 387 +++++++++--------- internal/framework/runner/section_evidence.go | 111 +++++ internal/framework/runner/window_test.go | 275 +++++++++++++ .../framework/structuredoutput/diagnostics.go | 21 + .../framework/validators/llm_validators.go | 66 +-- 11 files changed, 739 insertions(+), 240 deletions(-) create mode 100644 internal/framework/runner/section_evidence.go create mode 100644 internal/framework/runner/window_test.go diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 09b5fca..6c50528 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -15,8 +15,11 @@ formats; [configuration](../config.md) owns settings; the estimates stay fixed while text evolves. 4. Module instances execute in configured order. Each section hands its decoded proposals and original local segment references to the runner along with - operation facts and any captured exchange. Sections may propose concurrently. - The runner assigns indices in section order and response order. + operation facts and any captured exchange. A bounded window admits sections + in order and lets validation and repair proceed as proposals become ready. + The runner assigns final proposal indices in section and response order + when it records completed work. It then writes diagnostics that use those + indices. PromptKit controls provider capacity across all three roles. 5. Deterministic validators precede LLM-backed validators. Every submitted candidate receives one decision per validator on a normal return. Fatal returns may preserve a checked subset of completed decisions as diagnostic evidence. @@ -43,8 +46,8 @@ complete with model errors and applied corrections. Recoverable model failures never trigger shared cancellation. Fatal failures and parent cancellation stop further dispatch and join active workers. The initiating error carries its processing phase and available origin. Work-unit evidence preserves each peer's -interruption or error. Usable proposal results from later sections are indexed -in order after joining, even when the runner could not validate them. The +interruption or error. The runner drains active outcomes and assigns indices +to usable proposals from later sections even when it could not validate them. The working transcript advances only at the module's successful commit boundary. Earlier committed text remains available to an interrupted run, and transcript delivery requires successful processing. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 17c5801..b8c6831 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -589,6 +589,21 @@ pending, and identical index ordering for the same usable results under different completion orders. Report representative pending-work measurements without pinning a timing or worker-count threshold in the default suite. +**Complete.** The runner admits sections through a bounded rolling window, +drains active outcomes on cancellation, and lets validation and repair proceed +while earlier proposals are pending. It assigns final indices in section order +and publishes identity-sensitive artifacts through the interaction writer. +Tests checked native contention and overlap, stable ordering, and repair and +revalidation evidence after completion and cancellation. The full offline Go +suite and targeted race checks passed. + +In three runs of a 36-section fixture with two shared provider slots and the +first proposal held for 250 ms, ordered dispatch completed seven later proposals +and no validations during the hold. With section-local identities, seven later +validations also reached the provider before release, with eight sections +charged to the window in both variants. Both variants completed 36 proposals +and 36 validations overall. The window of eight was retained for this workload. + ## Stage 17 — Project typed report and ledger facts Audit #22, #27 (ledger), #39, #41 (projection). diff --git a/internal/cli/process_pipeline.go b/internal/cli/process_pipeline.go index a768bb3..03e2096 100644 --- a/internal/cli/process_pipeline.go +++ b/internal/cli/process_pipeline.go @@ -59,7 +59,8 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa for _, warning := range writer.Warnings() { warn(warning) } - warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot()))) + observations := processreport.IndexLLMObservations(observer.Snapshot(), output.WorkUnits) + warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observations))) warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output}))) } var failures []processreport.RunFailure diff --git a/internal/framework/processreport/llm.go b/internal/framework/processreport/llm.go index 40661bb..18de981 100644 --- a/internal/framework/processreport/llm.go +++ b/internal/framework/processreport/llm.go @@ -5,6 +5,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) @@ -94,3 +95,30 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any { Events []executionEvent `json:"events"` }{llm.Summarize(snapshot), calls, events} } + +// IndexLLMObservations assigns the same proposal identities used by the ledger. +func IndexLLMObservations(snapshot llm.ObservationSnapshot, units []runner.WorkUnitOutcome) llm.ObservationSnapshot { + type sectionKey struct { + module string + section int + } + offsets := make(map[sectionKey]int) + for _, unit := range units { + if unit.ProposalIndexOffset != 0 { + offsets[sectionKey{unit.ModuleInstance, unit.Section.Index}] = unit.ProposalIndexOffset + } + } + index := func(context *structuredoutput.DiagnosticContext) { + if context.SectionIndex != nil && context.ProposalIndex != nil { + offset := offsets[sectionKey{context.ModuleInstance, *context.SectionIndex}] + context.ProposalIndex = structuredoutput.OffsetProposalIndex(context.ProposalIndex, offset) + } + } + for i := range snapshot.Calls { + index(&snapshot.Calls[i].Context) + } + for i := range snapshot.Events { + index(&snapshot.Events[i].Context) + } + return snapshot +} diff --git a/internal/framework/runner/cancellation_test.go b/internal/framework/runner/cancellation_test.go index d391804..e1c810a 100644 --- a/internal/framework/runner/cancellation_test.go +++ b/internal/framework/runner/cancellation_test.go @@ -176,27 +176,26 @@ func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) { func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) { origin := errors.New("earlier proposal failed") - laterCompleted := make(chan struct{}) + laterValidationStarted := make(chan struct{}) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - var validations atomic.Int32 module := outcomeModule{ fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{ - contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) { - validations.Add(1) - return validators.Result{}, errors.New("validation should not be submitted") + contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) { + close(laterValidationStarted) + <-ctx.Done() + return validators.Result{}, ctx.Err() }}, }}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { if req.Section.Index == 0 { select { - case <-laterCompleted: + case <-laterValidationStarted: return contracts.ProposalResult{}, origin case <-ctx.Done(): return contracts.ProposalResult{}, ctx.Err() } } - close(laterCompleted) return contracts.ProposalResult{ Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}}, SegmentReferences: []int{1}, @@ -212,8 +211,8 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, }) - if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 { - t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err) + if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 { + t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err) } var failure *ExecutionFailure if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection || @@ -222,7 +221,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) } candidate := out.Candidates[0] later := out.WorkUnits[1] - if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || + if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) || len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex || out.FinalTranscript.Segments[1].Text != "other text" { t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later) diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index d5ab68f..9278b76 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -190,18 +190,37 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair snapshot := *revision record.Revision = &snapshot } - name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber) - var failure any - if err != nil { - failure = map[string]string{"error": err.Error()} - } - record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{ - Name: name, - Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}, - RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, - RequestPayload: completion.Prepared, - ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, - ErrorPayload: failure, + feedback := findings + record.Artifacts, _ = structuredoutput.WriteIndexedInteraction(input.Interactions, func(offset int, resolvePath func(string) string) structuredoutput.Interaction { + index := proposalIndex + offset + name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, index, attemptNumber) + var failure any + if err != nil { + failure = map[string]string{"error": err.Error()} + } + var correction *proposals.EnrichedCorrectionProposal + if record.Revision != nil { + snapshot := *record.Revision + snapshot.ProposalIndex = index + correction = &snapshot + } + decisions := slices.Clone(validated.decisions) + for i := range decisions { + decisions[i].ProposalIndex += offset + decisions[i].DiagnosticArtifactPath = resolvePath(decisions[i].DiagnosticArtifactPath) + } + rejections := slices.Clone(validated.rejected) + for i := range rejections { + rejections[i].ProposalIndex += offset + } + return structuredoutput.Interaction{ + Name: name, + Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &index, RepairAttempt: &attemptNumber}, + RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": index, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": feedback}, + RequestPayload: completion.Prepared, + ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": correction, "decisions": decisions, "rejections": rejections}, + ErrorPayload: failure, + } }) branch.Attempts = append(branch.Attempts, record) } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 9421d47..8b749fc 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -56,17 +56,18 @@ type ModuleResult struct { // WorkUnitOutcome keeps initial section identity while text evolves between // module instances. Detailed candidate and decision records are debug evidence. type WorkUnitOutcome struct { - StageIndex int `json:"stage_index"` - StageID *string `json:"stage_id,omitempty"` - ModuleIndex int `json:"module_index"` - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - Section contracts.SectionMetadata `json:"section"` - State outcomes.WorkState `json:"state"` - Failure *ExecutionFailure `json:"-"` - CorrectionsApplied int `json:"corrections_applied"` - Repairs []RepairOutcome `json:"-"` - Operations []outcomes.Operation `json:"operations,omitempty"` + ProposalIndexOffset int `json:"-"` + StageIndex int `json:"stage_index"` + StageID *string `json:"stage_id,omitempty"` + ModuleIndex int `json:"module_index"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + Section contracts.SectionMetadata `json:"section"` + State outcomes.WorkState `json:"state"` + Failure *ExecutionFailure `json:"-"` + CorrectionsApplied int `json:"corrections_applied"` + Repairs []RepairOutcome `json:"-"` + Operations []outcomes.Operation `json:"operations,omitempty"` } type ValidatorDecisionRecord struct { @@ -342,15 +343,21 @@ type sectionProposalResult struct { } type sectionValidationResult struct { - repairs []RepairOutcome - sectionPos int - enriched []proposals.EnrichedCorrectionProposal - approved []proposals.EnrichedCorrectionProposal - decisions []ValidatorDecisionRecord - rejected []ValidatorRejectedChange - operations []outcomes.Operation - filtered []outcomes.FilteredProposal - failure *ExecutionFailure + interactions *sectionInteractions + repairs []RepairOutcome + sectionPos int + approved []proposals.EnrichedCorrectionProposal + decisions []ValidatorDecisionRecord + rejected []ValidatorRejectedChange + operations []outcomes.Operation + filtered []outcomes.FilteredProposal + failure *ExecutionFailure +} + +type sectionWork struct { + candidates []proposals.EnrichedCorrectionProposal + validation *sectionValidationResult + finished bool } type modulePipelineResult struct { @@ -363,60 +370,42 @@ type modulePipelineResult struct { ValidatorRejected []ValidatorRejectedChange } -func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult { - results := make(chan sectionProposalResult, len(input.Sections)) +// moduleWorkWindow bounds active and downstream work across all roles in +// one module. PromptKit still owns admission to each provider backend. +const moduleWorkWindow = 8 - go func() { - defer close(results) - var wg sync.WaitGroup - for sectionPos, section := range input.Sections { - sectionPos := sectionPos - section := section - wg.Add(1) - go func() { - defer wg.Done() - if ctx.Err() != nil { - results <- sectionProposalResult{sectionPos: sectionPos, section: section} - return - } - meta := contracts.SectionMetadataFromSection(section) - proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{ - ExecutionContext: contracts.ExecutionContext{ - StageIndex: &input.Spec.Index, StageID: input.Spec.ID, - Settings: input.Spec.Settings, - WorkingTranscript: transcriptFromSection(section), - Glossary: input.Spec.Glossary, - ProtectedTerms: input.Spec.ProtectedTerms, - GlossaryVariables: input.GlossaryVariables, - Section: &meta, - Interactions: input.Interactions, - }, - RunSpec: contracts.ModuleRunSpec{ - ModuleKey: input.Spec.ModuleKey, - InstanceName: input.Spec.InstanceName, - ReplacementPolicy: input.Policy, - CaptureProposalExchange: input.Spec.CaptureProposalExchange, - }, - LLMClient: input.ProposalClient, - }) - var failure *ExecutionFailure - if err != nil { - failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations) - fail(failure) - } - results <- sectionProposalResult{ - sectionPos: sectionPos, - section: section, - proposal: proposalResult, - started: true, - failure: failure, - } - }() - } - wg.Wait() - }() - - return results +func proposeSection(ctx context.Context, input collectSectionProposalsInput, sectionPos int) sectionProposalResult { + section := input.Sections[sectionPos] + result := sectionProposalResult{sectionPos: sectionPos, section: section} + if ctx.Err() != nil { + return result + } + meta := contracts.SectionMetadataFromSection(section) + proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{ + ExecutionContext: contracts.ExecutionContext{ + StageIndex: &input.Spec.Index, StageID: input.Spec.ID, + Settings: input.Spec.Settings, + WorkingTranscript: transcriptFromSection(section), + Glossary: input.Spec.Glossary, + ProtectedTerms: input.Spec.ProtectedTerms, + GlossaryVariables: input.GlossaryVariables, + Section: &meta, + Interactions: input.Interactions, + }, + RunSpec: contracts.ModuleRunSpec{ + ModuleKey: input.Spec.ModuleKey, + InstanceName: input.Spec.InstanceName, + ReplacementPolicy: input.Policy, + CaptureProposalExchange: input.Spec.CaptureProposalExchange, + }, + LLMClient: input.ProposalClient, + }) + result.started = true + result.proposal = proposalResult + if err != nil { + result.failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations) + } + return result } func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) { @@ -453,23 +442,26 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) input.Spec.CaptureProposalExchange = true } - pending := make(map[int]sectionProposalResult, len(input.Sections)) - validationResults := make(chan sectionValidationResult, len(input.Sections)) - validationBySection := make(map[int]sectionValidationResult, len(input.Sections)) + pending := make(map[int]*sectionWork, moduleWorkWindow) + proposalResults := make(chan sectionProposalResult) + validationResults := make(chan sectionValidationResult) validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators()) - nextSectionToProcess := 0 + nextSectionToRecord := 0 nextProposalIndex := 0 var ( firstFailure *ExecutionFailure errMu sync.Mutex - vwg sync.WaitGroup + workers sync.WaitGroup ) runCtx, cancel := context.WithCancel(ctx) defer cancel() setFailure := func(failure *ExecutionFailure) { + if failure == nil { + return + } errMu.Lock() defer errMu.Unlock() if firstFailure == nil { @@ -477,133 +469,93 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) cancel() } } - proposalResults := collectSectionProposals(runCtx, input, setFailure) - - collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal { + makeCandidates := func(sectionResult sectionProposalResult) []proposals.EnrichedCorrectionProposal { sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals)) for i, corrected := range sectionResult.proposal.Proposals { - sectionIndex := index + sectionIndex := sectionResult.section.Index sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{ CorrectionProposal: corrected, ProposalMetadata: proposals.ProposalMetadata{ - ProposalIndex: nextProposalIndex + i, + ProposalIndex: i, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: §ionIndex, }, }) } - nextProposalIndex += len(sectionEnriched) - out.ProposalCount += len(sectionEnriched) - out.Candidates = append(out.Candidates, sectionEnriched...) - for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations { - operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex] - if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { - for _, proposal := range sectionEnriched { - operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex) - } - } - } return sectionEnriched } - for result := range proposalResults { - unit := &out.WorkUnits[result.sectionPos] - if result.started { - unit.State = outcomes.WorkInterrupted + recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) { + unit := &out.WorkUnits[sectionPos] + unit.ProposalIndexOffset = nextProposalIndex + for _, candidate := range enriched { + candidate.ProposalIndex += nextProposalIndex + out.Candidates = append(out.Candidates, candidate) } - unit.Operations = append(unit.Operations, result.proposal.Operations...) - if result.failure != nil { - unit.Failure = result.failure - continue - } - if !result.started { - continue - } - pending[result.sectionPos] = result - - for { - if runCtx.Err() != nil { - break + out.ProposalCount += len(enriched) + for operationIndex := range unit.Operations { + operation := &unit.Operations[operationIndex] + if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { + for _, proposal := range enriched { + operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex) + } } - sectionResult, ok := pending[nextSectionToProcess] - if !ok { - break + } + nextProposalIndex += len(enriched) + } + + validate := func(sectionResult sectionProposalResult, enriched []proposals.EnrichedCorrectionProposal) sectionValidationResult { + sectionPos := sectionResult.sectionPos + sectionMetadata := contracts.SectionMetadataFromSection(sectionResult.section) + var interactions structuredoutput.InteractionSink + var capture *sectionInteractions + if input.Interactions != nil { + capture = §ionInteractions{InteractionSink: input.Interactions} + interactions = capture + } + validationInput := validateSectionCandidatesInput{ + Spec: input.Spec, + Policy: input.Policy, + WorkingTranscript: transcriptFromSection(sectionResult.section), + SectionIndex: §ionMetadata.Index, + Validators: validatorsOrdered, + SectionEnriched: enriched, + Exchange: sectionResult.proposal.Exchange, + ValidationLLMClient: input.ValidationClient, + GlossaryView: input.GlossaryView, + Interactions: interactions, + } + if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 { + validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript) + } + validated, err := validateSectionCandidates(runCtx, validationInput) + var repairs []RepairOutcome + phase, kind := "validation", outcomes.OperationValidation + if err == nil && repairPromptID != "" { + phase, kind = "correction_repair", outcomes.OperationCorrectionRepair + validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID) + } + var failure *ExecutionFailure + if err != nil { + if !errors.As(err, &failure) { + failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations) } - delete(pending, nextSectionToProcess) - sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section) - sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index) - - vwg.Add(1) - go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) { - defer vwg.Done() - validationInput := validateSectionCandidatesInput{ - Spec: input.Spec, - Policy: input.Policy, - WorkingTranscript: transcriptFromSection(sectionResult.section), - SectionIndex: §ionMetadata.Index, - Validators: validatorsOrdered, - SectionEnriched: enriched, - Exchange: sectionResult.proposal.Exchange, - ValidationLLMClient: input.ValidationClient, - GlossaryView: input.GlossaryView, - Interactions: input.Interactions, - } - if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 { - validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript) - } - validated, err := validateSectionCandidates(runCtx, validationInput) - var repairs []RepairOutcome - phase, kind := "validation", outcomes.OperationValidation - if err == nil && repairPromptID != "" { - phase, kind = "correction_repair", outcomes.OperationCorrectionRepair - validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID) - } - var failure *ExecutionFailure - if err != nil { - if !errors.As(err, &failure) { - failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations) - } - setFailure(failure) - } - validationResults <- sectionValidationResult{ - repairs: repairs, - sectionPos: sectionPos, - enriched: enriched, - approved: validated.approved, - decisions: validated.decisions, - rejected: validated.rejected, - operations: validated.operations, - filtered: validated.filtered, - failure: failure, - } - }(nextSectionToProcess, sectionEnriched, sectionMeta) - nextSectionToProcess++ + } + return sectionValidationResult{ + interactions: capture, + repairs: repairs, + sectionPos: sectionPos, + approved: validated.approved, + decisions: validated.decisions, + rejected: validated.rejected, + operations: validated.operations, + filtered: validated.filtered, + failure: failure, } } - - vwg.Wait() - close(validationResults) - - // A failed or canceled module can still have usable proposals from later - // sections. Assign their indices in order after preserving the dispatched - // prefix, without submitting new validation work. - for pos := nextSectionToProcess; pos < len(input.Sections); pos++ { - if result, ok := pending[pos]; ok { - collectCandidates(result, result.section.Index) - } - } - - for v := range validationResults { - validationBySection[v.sectionPos] = v - } - - for i := range input.Sections { - res, ok := validationBySection[i] - if !ok { - continue - } - unit := &out.WorkUnits[i] + recordValidation := func(res sectionValidationResult) { + unit := &out.WorkUnits[res.sectionPos] unit.Operations = append(unit.Operations, res.operations...) unit.Repairs = res.repairs mergeFilteredProposals(unit, res.filtered) @@ -617,6 +569,69 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...) out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...) } + + nextAdmission, active := 0, 0 + for { + for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] { + offset := nextProposalIndex + recordCandidates(nextSectionToRecord, work.candidates) + if work.validation != nil { + work.validation.assignProposalIndices(offset) + recordValidation(*work.validation) + } + delete(pending, nextSectionToRecord) + nextSectionToRecord++ + } + for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil { + sectionPos := nextAdmission + nextAdmission++ + pending[sectionPos] = §ionWork{} + active++ + workers.Go(func() { + proposed := proposeSection(runCtx, input, sectionPos) + setFailure(proposed.failure) + proposalResults <- proposed + }) + } + if active == 0 { + break + } + select { + case result := <-proposalResults: + active-- + unit := &out.WorkUnits[result.sectionPos] + unit.Operations = append(unit.Operations, result.proposal.Operations...) + if result.started { + unit.State = outcomes.WorkInterrupted + } + unit.Failure = result.failure + work := pending[result.sectionPos] + if result.started && result.failure == nil { + work.candidates = makeCandidates(result) + if runCtx.Err() == nil { + enriched := work.candidates + active++ + workers.Go(func() { + validated := validate(result, enriched) + setFailure(validated.failure) + validationResults <- validated + }) + continue + } + } + work.finished = true + case validated := <-validationResults: + active-- + work := pending[validated.sectionPos] + work.validation = &validated + work.finished = true + } + } + workers.Wait() + sort.SliceStable(out.Approved, func(i, j int) bool { + return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex + }) + for _, unit := range out.WorkUnits { out.Operations = append(out.Operations, unit.Operations...) } diff --git a/internal/framework/runner/section_evidence.go b/internal/framework/runner/section_evidence.go new file mode 100644 index 0000000..5f68ed8 --- /dev/null +++ b/internal/framework/runner/section_evidence.go @@ -0,0 +1,111 @@ +package runner + +import ( + "slices" + "strconv" + "strings" + "sync" + + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" +) + +// Section validation can finish before preceding proposal counts are known. +type sectionInteractions struct { + structuredoutput.InteractionSink + mu sync.Mutex + builders []structuredoutput.IndexedInteraction + written []structuredoutput.InteractionArtifacts +} + +const pendingInteractionPrefix = "\x00interaction:" + +func (s *sectionInteractions) WriteIndexedInteraction(build structuredoutput.IndexedInteraction) (structuredoutput.InteractionArtifacts, error) { + s.mu.Lock() + defer s.mu.Unlock() + receipt := pendingInteractionPrefix + strconv.Itoa(len(s.builders)) + s.builders = append(s.builders, build) + return structuredoutput.InteractionArtifacts{ResponsePayloadPath: receipt}, nil +} + +func (s *sectionInteractions) write(offset int) { + for _, build := range s.builders { + paths, _ := s.InteractionSink.WriteInteraction(build(offset, s.responsePath)) + s.written = append(s.written, paths) + } + s.builders = nil +} + +func (s *sectionInteractions) artifacts(paths structuredoutput.InteractionArtifacts) structuredoutput.InteractionArtifacts { + value, pending := strings.CutPrefix(paths.ResponsePayloadPath, pendingInteractionPrefix) + if !pending { + return paths + } + index, err := strconv.Atoi(value) + if err != nil || index < 0 || index >= len(s.written) { + return structuredoutput.InteractionArtifacts{} + } + return s.written[index] +} + +func (s *sectionInteractions) responsePath(path string) string { + return s.artifacts(structuredoutput.InteractionArtifacts{ResponsePayloadPath: path}).ResponsePayloadPath +} + +func (r *sectionValidationResult) assignProposalIndices(offset int) { + capture := r.interactions != nil + if capture { + r.interactions.write(offset) + } + for i := range r.approved { + r.approved[i].ProposalIndex += offset + } + for i := range r.rejected { + r.rejected[i].ProposalIndex += offset + } + assignDecisions := func(decisions []ValidatorDecisionRecord) { + for i := range decisions { + decisions[i].ProposalIndex += offset + if capture { + decisions[i].DiagnosticArtifactPath = r.interactions.responsePath(decisions[i].DiagnosticArtifactPath) + } + } + } + assignDecisions(r.decisions) + for i := range r.operations { + op := &r.operations[i] + op.Reference.ProposalIndex = structuredoutput.OffsetProposalIndex(op.Reference.ProposalIndex, offset) + op.ProposalIndices = slices.Clone(op.ProposalIndices) + for j := range op.ProposalIndices { + op.ProposalIndices[j] += offset + } + op.Filtered = slices.Clone(op.Filtered) + for j := range op.Filtered { + op.Filtered[j].ProposalIndex += offset + } + } + for i := range r.filtered { + r.filtered[i].ProposalIndex += offset + } + for i := range r.repairs { + repair := &r.repairs[i] + repair.ProposalIndex += offset + if capture { + repair.Original.ProposalIndex += offset + repair.Final.ProposalIndex += offset + } + for j := range repair.Attempts { + attempt := &repair.Attempts[j] + if attempt.Revision != nil { + attempt.Revision.ProposalIndex += offset + } + assignDecisions(attempt.Decisions) + if capture { + attempt.Artifacts = r.interactions.artifacts(attempt.Artifacts) + } + } + } + if r.failure != nil && r.failure.Origin != nil { + r.failure.Origin.ProposalIndex = structuredoutput.OffsetProposalIndex(r.failure.Origin.ProposalIndex, offset) + } + r.interactions = nil +} diff --git a/internal/framework/runner/window_test.go b/internal/framework/runner/window_test.go new file mode 100644 index 0000000..4cedfd3 --- /dev/null +++ b/internal/framework/runner/window_test.go @@ -0,0 +1,275 @@ +package runner + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" + "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar" + ck "go.maximumdirect.net/configkit" +) + +type sharedBackendModule struct { + *grammar.Module + validator validators.Validator + headEntered <-chan struct{} +} + +func (m sharedBackendModule) Validators() []validators.Validator { + return []validators.Validator{m.validator} +} + +func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { + if req.Section.Index > 0 { + select { + case <-m.headEntered: + case <-ctx.Done(): + return contracts.ProposalResult{}, ctx.Err() + } + } + return m.Module.Propose(ctx, req) +} + +func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + dir := t.TempDir() + path := filepath.Join(dir, "config.yml") + if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil { + t.Fatal(err) + } + resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n" + if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil { + t.Fatal(err) + } + loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "") + if err != nil { + t.Fatal(err) + } + release := make(chan struct{}) + entered := make(chan struct{}) + validationStarted := make(chan struct{}, 1) + var other, validating, active, peak atomic.Int32 + client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) { + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + n := active.Add(1) + for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() { + } + defer active.Add(-1) + output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}` + switch { + case bytes.Contains(body, []byte(`"validations"`)): + validating.Add(1) + select { + case validationStarted <- struct{}{}: + default: + } + output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}` + case bytes.Contains(body, []byte("marker-000")): + close(entered) + select { + case <-release: + case <-r.Context().Done(): + return nil, r.Context().Err() + } + default: + other.Add(1) + } + content, _ := json.Marshal(output) + wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content) + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil + })} + runtime, err := llm.NewRuntime(ctx, loaded, client) + if err != nil { + t.Fatal(err) + } + role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{}) + if err != nil { + t.Fatal(err) + } + validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) + if err != nil { + t.Fatal(err) + } + module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered} + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}) + cfg := runnerTestConfig() + cfg.Chunking.MaxSectionTokens = 1 + cfg.Chunking.MinSectionTokens = 0 + transcript := &schema.Transcript{} + for i := 0; i < 36; i++ { + transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)}) + } + type result struct { + out RunOutput + err error + } + finished := make(chan result, 1) + go func() { + out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}}, + }) + finished <- result{out, err} + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case <-validationStarted: + case <-ctx.Done(): + t.Fatal("later validation did not reach the backend while the earlier proposal held a slot") + } + close(release) + completed := <-finished + if completed.err != nil { + t.Fatal(completed.err) + } + if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 { + t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load()) + } + changes := completed.out.ModuleResults[0].AppliedChanges + if len(changes) != len(transcript.Segments) { + t.Fatalf("got %d applied changes", len(changes)) + } + for i, change := range changes { + if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID { + t.Fatalf("change %d: %+v", i, change) + } + } +} + +func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) { + for _, canceled := range []bool{false, true} { + name := "completed" + if canceled { + name = "canceled" + } + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + laterRevalidated := make(chan struct{}) + proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) { + section := *req.Context.SectionIndex + if section == 0 { + select { + case <-laterRevalidated: + case <-ctx.Done(): + return contracts.StructuredCompletionResponse{}, ctx.Err() + } + } + text := []string{"alpha", "bravo"}[section] + dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{ + SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text), + }} + return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil + }) + validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) { + section := *req.Context.SectionIndex + first := section == 0 + revision := req.Context.RepairAttempt != nil + if first && canceled { + cancel() + return contracts.StructuredCompletionResponse{}, ctx.Err() + } + dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{ + CorrectionIndex: "1", Approved: first || revision, Reason: "checked", + }} + if revision { + close(laterRevalidated) + } + return contracts.StructuredCompletionResponse{}, nil + }) + validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) + if err != nil { + t.Fatal(err) + } + module := sharedBackendModule{Module: &grammar.Module{}, validator: validator} + // The module's transport gates are supplied by the scripted calls above. + started := make(chan struct{}) + close(started) + module.headEntered = started + repair := &repairClient{t: t, outputs: []string{ + `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`, + }} + cfg := runnerTestConfig() + cfg.Chunking.MaxSectionTokens = 1 + cfg.Chunking.MinSectionTokens = 0 + out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ + Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ + ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation, + RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}}, + }}, + }) + if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) { + t.Fatal(err) + } + later := out.WorkUnits[1] + if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 || + len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 { + t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs) + } + attempt := later.Repairs[0].Attempts[0] + if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 || + len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 { + t.Fatalf("lost ordered repair evidence: %+v", attempt) + } + var repaired struct { + Decoded proposal_generation.RepairResponse + Revision *proposals.EnrichedCorrectionProposal + Decisions []ValidatorDecisionRecord + } + readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired) + if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 || + len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 || + repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath || + !strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") { + t.Fatalf("repair artifact uses different identities: %+v", repaired) + } + var checked struct { + Decoded validators.LLMValidationResponse + Reconciliation struct{ Decisions []validators.Decision } + } + readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked) + if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" || + len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 { + t.Fatalf("validation evidence uses different identities: %+v", checked) + } + }) + } +} + +func readInteraction(t *testing.T, path string, result any) { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(raw, result); err != nil { + t.Fatal(err) + } +} diff --git a/internal/framework/structuredoutput/diagnostics.go b/internal/framework/structuredoutput/diagnostics.go index 0b0bf1e..3339011 100644 --- a/internal/framework/structuredoutput/diagnostics.go +++ b/internal/framework/structuredoutput/diagnostics.go @@ -22,3 +22,24 @@ type InteractionArtifacts struct { ResponsePayloadPath string `json:"response_payload_path,omitempty"` ErrorPayloadPath string `json:"error_payload_path,omitempty"` } + +// IndexedInteraction builds an artifact after the section's ordered proposal +// indices and paths from earlier interactions are available. +type IndexedInteraction func(offset int, resolvePath func(string) string) Interaction + +func WriteIndexedInteraction(sink InteractionSink, build IndexedInteraction) (InteractionArtifacts, error) { + if deferred, ok := sink.(interface { + WriteIndexedInteraction(IndexedInteraction) (InteractionArtifacts, error) + }); ok { + return deferred.WriteIndexedInteraction(build) + } + return sink.WriteInteraction(build(0, func(path string) string { return path })) +} + +func OffsetProposalIndex(index *int, offset int) *int { + if index == nil || offset == 0 { + return index + } + next := *index + offset + return &next +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 618a11d..86b3bc0 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "slices" "sort" "strings" @@ -141,33 +142,44 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, if req.Interactions == nil { return structuredoutput.InteractionArtifacts{} } - stageInstance := req.ModuleInstance - if req.SectionIndex != nil { - stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex) - } - if req.RepairProposalIndex != nil && req.RepairAttempt != nil { - stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt) - } - stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex) - responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) - responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model} - if reconciled != nil { - responsePayload["reconciliation"] = *reconciled - } - artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{ - Name: stage, Context: coordinates, RequestMetadata: map[string]any{ - "stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey, - "validator_name": v.name, - "validator_type": v.validatorType, - "batch_index": batch.BatchIndex, - "proposal_indices": batchProposalIndices(batch.Items), - "repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt, - "prompt": completion.Prompt, - "response_schema": responseSchema.DiagnosticsMap(), - }, - RequestPayload: completion.Prepared, - ResponsePayload: responsePayload, - ErrorPayload: errPayload(callErr), + artifacts, _ := structuredoutput.WriteIndexedInteraction(req.Interactions, func(offset int, _ func(string) string) structuredoutput.Interaction { + coordinates := coordinates + coordinates.ProposalIndex = structuredoutput.OffsetProposalIndex(req.RepairProposalIndex, offset) + stageInstance := req.ModuleInstance + if req.SectionIndex != nil { + stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex) + } + if coordinates.ProposalIndex != nil && req.RepairAttempt != nil { + stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *coordinates.ProposalIndex, *req.RepairAttempt) + } + responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) + responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model} + if reconciled != nil { + view := *reconciled + view.Decisions = slices.Clone(view.Decisions) + for i := range view.Decisions { + view.Decisions[i].ProposalIndex += offset + view.Decisions[i].DiagnosticArtifactPath = "" + } + view.UnavailableIndices = slices.Clone(view.UnavailableIndices) + for i := range view.UnavailableIndices { + view.UnavailableIndices[i] += offset + } + responsePayload["reconciliation"] = view + } + indices := batchProposalIndices(batch.Items) + for i := range indices { + indices[i] += offset + } + return structuredoutput.Interaction{ + Name: stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex), Context: coordinates, RequestMetadata: map[string]any{ + "stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey, + "validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex, + "proposal_indices": indices, "repair_proposal_index": coordinates.ProposalIndex, "repair_attempt": req.RepairAttempt, + "prompt": completion.Prompt, "response_schema": responseSchema.DiagnosticsMap(), + }, + RequestPayload: completion.Prepared, ResponsePayload: responsePayload, ErrorPayload: errPayload(callErr), + } }) return artifacts }