From a0c70e8dce38028a04a2749459c4c3d0a83b46a6 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 14:30:57 +0000 Subject: [PATCH] Carry one proposal result from generation to the runner --- docs/internal/diagnostics-reporting.md | 4 +- docs/internal/pipeline.md | 6 +- docs/roadmap/implementation.md | 8 + .../adapter_integration_test.go | 4 +- .../framework/proposal_generation/generate.go | 52 +--- .../proposal_generation/generate_test.go | 106 +------- .../proposal_generation/module_proposal.go | 12 +- internal/framework/runner/runner.go | 20 +- internal/framework/runner/runner_test.go | 255 +++++++----------- 9 files changed, 153 insertions(+), 314 deletions(-) diff --git a/docs/internal/diagnostics-reporting.md b/docs/internal/diagnostics-reporting.md index 5238d62..6f91c64 100644 --- a/docs/internal/diagnostics-reporting.md +++ b/docs/internal/diagnostics-reporting.md @@ -43,7 +43,9 @@ rendered by ConfigKit, the correction ledger, LLM execution observations, and model interactions. Candidate capture includes proposals later rejected or interrupted. Repair and revalidation artifacts use section, proposal, attempt, validator, and batch coordinates as applicable. Request metadata records -these coordinates with the interaction role. Each interaction retains its available +these coordinates with the interaction role. Proposal request metadata carries +the zero-based response origin (`start_index`) and the ordered `segment_ids` +used to interpret response references. Each interaction retains its available prepared request, response, and failure evidence. Ledger mapping preserves empty proposed and applied replacements and distinguishes unavailable candidates from interrupted work. diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 2417b85..d98fef5 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -13,8 +13,10 @@ formats; [configuration](../config.md) owns settings; the 3. The runner partitions the normalized transcript exactly once before its module loop. Initial section identity, segment membership, and token estimates stay fixed while text evolves. -4. Module instances execute in configured order. Sections may propose - concurrently, but proposal indices follow section order and response order. +4. Module instances execute in configured order. Each section hands its decoded + proposals and original local segment references to the runner along with + operation facts and any captured exchange. Sections may propose concurrently. + The runner assigns indices in section order and response order. 5. Deterministic validators precede LLM-backed validators. Every submitted candidate receives one decision per validator on a normal return. Fatal returns may preserve a checked subset of completed decisions as diagnostic evidence. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6d2cb3b..eb4fdcd 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -472,6 +472,14 @@ Audit #46. verify indices and original references. Keep model response and reference coverage in the generator's owning tests. +**Complete.** The generator and runner share one decoded proposal result with +raw local references, operation facts, and the optional native exchange. The +runner assigns indices in section and response order. A production-module +fixture observed the later response first and verified applied indices, +operation associations, and raw references. Proposal diagnostics retain +the response-local `start_index`. The full offline suite and repeated +ordering checks under the race detector passed. + ## Stage 14 — Own each correction repair branch explicitly Audit #34, #45, #48, #64 (branch integration). diff --git a/internal/framework/proposal_generation/adapter_integration_test.go b/internal/framework/proposal_generation/adapter_integration_test.go index f30604f..a43d127 100644 --- a/internal/framework/proposal_generation/adapter_integration_test.go +++ b/internal/framework/proposal_generation/adapter_integration_test.go @@ -73,10 +73,10 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) { if err != nil { t.Fatal(err) } - if calls != 2 || len(generated.Enriched) != 1 { + if calls != 2 || len(generated.Proposals) != 1 { t.Fatalf("repair path: calls=%d, result=%+v", calls, generated) } - input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript} + input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: generated.Proposals[0]}}, WorkingTranscript: transcript} missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input) if err != nil { t.Fatal(err) diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 434cf75..a8613b7 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -42,7 +42,6 @@ type Request struct { PromptID string `json:"prompt_id"` Variables map[string]string `json:"variables"` StageName string `json:"stage_name,omitempty"` - StartIndex int `json:"start_index"` CaptureExchange bool LLMClient contracts.StructuredLLMClient Interactions structuredoutput.InteractionSink @@ -50,12 +49,8 @@ type Request struct { // Result contains generated candidate proposals and optional diagnostics paths. type Result struct { - SegmentReferences []int `json:"-"` - Exchange *structuredoutput.Exchange `json:"-"` - Corrections []proposals.CorrectionProposal `json:"corrections"` - Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"` - Operations []outcomes.Operation `json:"operations,omitempty"` - Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"` + contracts.ProposalResult + Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"` } // GenerateCandidates executes one structured LLM call and deterministically maps @@ -67,9 +62,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { if strings.TrimSpace(req.ModuleInstance) == "" { return Result{}, fmt.Errorf("module instance must not be empty") } - if req.StartIndex < 0 { - return Result{}, fmt.Errorf("start index must be non-negative") - } if req.LLMClient == nil { return Result{}, fmt.Errorf("structured LLM client is required") } @@ -115,7 +107,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { "module_instance": req.ModuleInstance, "replacement_policy": req.ReplacementPolicy, "section": req.Section, - "start_index": req.StartIndex, + "start_index": 0, // Position of the first item within this response. "segment_ids": segmentIDs(req.WorkingTranscript), "prompt": completion.Prompt, } @@ -129,7 +121,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { if callErr != nil { structuredoutput.RecordFailure(&operation, callErr) - result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts} + result := Result{ProposalResult: contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, Artifacts: artifacts} if operation.FailureClass == outcomes.FailureRecoverable { return result, nil } @@ -140,39 +132,23 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { references := make([]int, 0, len(response.Corrections)) corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections)) - enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections)) - for i, raw := range response.Corrections { + for _, raw := range response.Corrections { references = append(references, raw.SegmentReference) // Unknown references remain invalid candidates for the existing domain validators. - candidate := proposals.CorrectionProposal{ + corrections = append(corrections, proposals.CorrectionProposal{ TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference), OriginalText: raw.OriginalText, CorrectedText: raw.CorrectedText, - } - - corrections = append(corrections, candidate) - enrichedCandidate := proposals.EnrichedCorrectionProposal{ - CorrectionProposal: candidate, - ProposalMetadata: proposals.ProposalMetadata{ - ProposalIndex: req.StartIndex + i, - ModuleKey: req.ModuleKey, - ModuleInstance: req.ModuleInstance, - }, - } - if req.Section != nil { - sectionIndex := req.Section.Index - enrichedCandidate.SectionIndex = §ionIndex - } - enriched = append(enriched, enrichedCandidate) + }) } - return Result{ - Corrections: corrections, - SegmentReferences: references, - Exchange: completion.Exchange, - Enriched: enriched, - Operations: []outcomes.Operation{operation}, - Artifacts: artifacts, + ProposalResult: contracts.ProposalResult{ + Proposals: corrections, + SegmentReferences: references, + Exchange: completion.Exchange, + Operations: []outcomes.Operation{operation}, + }, + Artifacts: artifacts, }, nil } diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 06afa42..3c8ea67 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -6,7 +6,6 @@ import ( "errors" "os" "path/filepath" - "reflect" "strings" "testing" @@ -73,7 +72,6 @@ func defaultRequest(t *testing.T) Request { PromptID: "modules.grammar.proposal", Variables: map[string]string{"SectionJSON": "[]"}, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}}, - StartIndex: 0, } } @@ -90,24 +88,17 @@ func TestGenerateCandidatesSuccess(t *testing.T) { section := contracts.SectionMetadata{Index: 2} req := defaultRequest(t) req.LLMClient = client - req.StartIndex = 10 req.Section = §ion got, err := GenerateCandidates(context.Background(), req) if err != nil { t.Fatalf("GenerateCandidates error: %v", err) } - if len(got.Corrections) != 1 || len(got.Enriched) != 1 { + if len(got.Proposals) != 1 || len(got.SegmentReferences) != 1 || got.SegmentReferences[0] != 1 { t.Fatalf("unexpected proposal lengths: %+v", got) } - if got.Corrections[0].TargetSegmentID != 7 { - t.Fatalf("local reference not resolved: %+v", got.Corrections) - } - if got.Enriched[0].ProposalIndex != 10 { - t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex) - } - if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 { - t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex) + if got.Proposals[0].TargetSegmentID != 7 { + t.Fatalf("local reference not resolved: %+v", got.Proposals) } if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal { t.Fatalf("proposal operation identity: %+v", got.Operations) @@ -184,11 +175,11 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes if err != nil { t.Fatalf("expected invalid correction to survive generation, got %v", err) } - if len(result.Corrections) != 1 { + if len(result.Proposals) != 1 { t.Fatalf("expected one correction, got %+v", result) } - if result.Corrections[0].TargetSegmentID != 0 { - t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0]) + if result.Proposals[0].TargetSegmentID != 0 { + t.Fatalf("unexpected preserved correction: %+v", result.Proposals[0]) } raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath) if err != nil { @@ -218,7 +209,7 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te if err != nil { t.Fatalf("expected recoverable model operation, got %v", err) } - if len(result.Corrections) != 0 || len(result.Enriched) != 0 { + if len(result.Proposals) != 0 { t.Fatalf("expected no proposals on malformed response, got %+v", result) } if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted { @@ -255,87 +246,11 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) { req := defaultRequest(t) req.LLMClient = client result, err := GenerateCandidates(context.Background(), req) - if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded { + if err != nil || len(result.Proposals) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded { t.Fatalf("valid empty response: %+v, %v", result, err) } } -func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) { - baseResponse := StructuredCorrectionSet{ - Corrections: []StructuredCorrectionProposal{ - {SegmentReference: 1, OriginalText: "a", CorrectedText: "A"}, - {SegmentReference: 2, OriginalText: "b", CorrectedText: "B"}, - }, - } - clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} - clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} - - reqA := defaultRequest(t) - reqA.LLMClient = clientA - reqA.StartIndex = 3 - first, err := GenerateCandidates(context.Background(), reqA) - if err != nil { - t.Fatalf("first generation failed: %v", err) - } - - reqB := defaultRequest(t) - reqB.LLMClient = clientB - reqB.StartIndex = 3 - second, err := GenerateCandidates(context.Background(), reqB) - if err != nil { - t.Fatalf("second generation failed: %v", err) - } - - if !reflect.DeepEqual(first.Enriched, second.Enriched) { - t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched) - } -} - -func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { - client := &fakeStructuredClient{ - responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}}, - {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}}, - }, - } - section0 := contracts.SectionMetadata{Index: 0} - section1 := contracts.SectionMetadata{Index: 1} - - req0 := defaultRequest(t) - req0.LLMClient = client - req0.Section = §ion0 - req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}} - req0.StartIndex = 0 - part0, err := GenerateCandidates(context.Background(), req0) - if err != nil { - t.Fatalf("section 0 generation failed: %v", err) - } - - req1 := defaultRequest(t) - req1.LLMClient = client - req1.Section = §ion1 - req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}} - req1.StartIndex = len(part0.Enriched) - part1, err := GenerateCandidates(context.Background(), req1) - if err != nil { - t.Fatalf("section 1 generation failed: %v", err) - } - - all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...) - if len(all) != 2 { - t.Fatalf("expected 2 proposals, got %d", len(all)) - } - if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 { - t.Fatalf("section references did not reset: %+v", all) - } - if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 { - t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex) - } - if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 { - t.Fatalf("unexpected section metadata: %+v", all) - } -} - func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ @@ -375,6 +290,9 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) { if err := json.Unmarshal(raw, &metadata); err != nil { t.Fatalf("unmarshal metadata artifact: %v", err) } + if start, ok := metadata["start_index"].(float64); !ok || start != 0 { + t.Fatalf("missing response-local origin: %s", raw) + } schemaMap, ok := metadata["response_schema"].(map[string]any) if !ok { t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"]) @@ -405,7 +323,7 @@ func TestGenerateCandidatesClientError(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "completion failed") { t.Fatalf("expected completion failure, got %v", err) } - if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 { + if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Proposals) != 0 { t.Fatalf("expected fatal call facts without proposals, got %+v", got) } matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json")) diff --git a/internal/framework/proposal_generation/module_proposal.go b/internal/framework/proposal_generation/module_proposal.go index 517164a..1af8f90 100644 --- a/internal/framework/proposal_generation/module_proposal.go +++ b/internal/framework/proposal_generation/module_proposal.go @@ -62,19 +62,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont PromptID: promptMetadata.PromptID, Variables: variables, StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)), - StartIndex: 0, CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange, LLMClient: req.ProposalRequest.LLMClient, Interactions: req.ProposalRequest.Interactions, }) - if err != nil { - return contracts.ProposalResult{Operations: generated.Operations}, err - } - - return contracts.ProposalResult{ - Proposals: generated.Corrections, - SegmentReferences: generated.SegmentReferences, - Exchange: generated.Exchange, - Operations: generated.Operations, - }, nil + return generated.ProposalResult, err } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 92a45a6..8e59012 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -343,10 +343,7 @@ type collectSectionProposalsInput struct { type sectionProposalResult struct { sectionPos int section chunking.Section - corrected []proposals.CorrectionProposal - references []int - exchange *structuredoutput.Exchange - operations []outcomes.Operation + proposal contracts.ProposalResult started bool err error } @@ -413,10 +410,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI results <- sectionProposalResult{ sectionPos: sectionPos, section: section, - corrected: proposalResult.Proposals, - references: proposalResult.SegmentReferences, - exchange: proposalResult.Exchange, - operations: proposalResult.Operations, + proposal: proposalResult, started: true, err: err, } @@ -486,7 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) if result.started { unit.State = outcomes.WorkInterrupted } - unit.Operations = append(unit.Operations, result.operations...) + unit.Operations = append(unit.Operations, result.proposal.Operations...) if result.err != nil { setErr(result.err) continue @@ -509,8 +503,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } delete(pending, nextSectionToProcess) sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section) - sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected)) - for i, corrected := range sectionResult.corrected { + sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals)) + for i, corrected := range sectionResult.proposal.Proposals { sectionIndex := sectionMeta.Index sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{ CorrectionProposal: corrected, @@ -544,7 +538,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) SectionIndex: §ionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, - Exchange: sectionResult.exchange, + Exchange: sectionResult.proposal.Exchange, ValidationLLMClient: input.ValidationClient, GlossaryView: input.GlossaryView, Interactions: input.Interactions, @@ -559,7 +553,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok { promptID = provider.RepairPromptID() } - validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID) + validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, promptID) } setErr(err) validationResults <- sectionValidationResult{ diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 83f3784..a982187 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -20,7 +20,9 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" + "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) @@ -449,86 +451,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) { } } -func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) { - cfg := runnerTestConfig() - cfg.Chunking.MaxSectionTokens = 3 - cfg.Chunking.MinSectionTokens = 0 - - transcript := &schema.Transcript{Segments: []schema.Segment{ - {ID: 1, Text: "alpha one"}, - {ID: 2, Text: "bravo two"}, - {ID: 3, Text: "charlie three"}, - }} - - started := make(chan int, len(transcript.Segments)) - releaseByID := map[int]chan struct{}{ - 1: make(chan struct{}), - 2: make(chan struct{}), - 3: make(chan struct{}), - } - - r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { - if len(req.WorkingTranscript.Segments) != 1 { - t.Fatalf("expected one segment per section, got %d", len(req.WorkingTranscript.Segments)) - } - seg := req.WorkingTranscript.Segments[0] - started <- seg.ID - <-releaseByID[seg.ID] - return []proposals.CorrectionProposal{ - {TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)}, - }, nil - }}, - }}) - - resultCh := make(chan struct { - out RunOutput - err error - }, 1) - go func() { - out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: transcript, - ModuleSpecs: []contracts.ModuleRunSpec{ - {ModuleKey: "m", InstanceName: "m"}, - }, - }) - resultCh <- struct { - out RunOutput - err error - }{out: out, err: err} - }() - - waitForRunnerSectionIDs(t, started, map[int]struct{}{1: {}, 2: {}, 3: {}}) - close(releaseByID[3]) - close(releaseByID[2]) - close(releaseByID[1]) - - result := <-resultCh - out, err := result.out, result.err - if err != nil { - t.Fatalf("Run error: %v", err) - } - if len(out.ModuleResults) != 1 { - t.Fatalf("expected one module result, got %+v", out.ModuleResults) - } - if got := len(out.ModuleResults[0].AppliedChanges); got != 3 { - t.Fatalf("expected three applied changes, got %d", got) - } - for i, change := range out.ModuleResults[0].AppliedChanges { - wantID := i + 1 - if change.TargetSegmentID != wantID { - t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges) - } - } - wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"} - for i, seg := range out.FinalTranscript.Segments { - if seg.Text != wantFinal[i] { - t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments) - } - } -} - func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) { t.Helper() deadline := time.After(350 * time.Millisecond) @@ -1228,90 +1150,117 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) { } } -type proposalGenerationModule struct { - key string - policy proposals.ReplacementPolicy +type proposalClientFunc func(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error) + +func (f proposalClientFunc) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + return f(ctx, req, out) } -func (m proposalGenerationModule) Key() string { return m.key } -func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } -func (m proposalGenerationModule) Validators() []validators.Validator { return nil } -func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { - result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ - ModuleKey: req.RunSpec.ModuleKey, - ModuleInstance: req.RunSpec.InstanceName, - ReplacementPolicy: req.RunSpec.ReplacementPolicy, - WorkingTranscript: req.WorkingTranscript, - PromptID: "modules.grammar.proposal", - Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, - LLMClient: req.LLMClient, - Interactions: req.Interactions, - }) - if err != nil { - return contracts.ProposalResult{}, err - } - return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil -} - -type fakeProposalStructuredClient struct { - responses []proposal_generation.StructuredCorrectionSet -} - -func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - _ = ctx - _ = req - if len(f.responses) == 0 { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") - } - target, ok := out.(*proposal_generation.StructuredCorrectionSet) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type") - } - *target = f.responses[0] - f.responses = f.responses[1:] - return contracts.StructuredCompletionResponse{}, nil -} - -func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) { - client := &fakeProposalStructuredClient{ - responses: []proposal_generation.StructuredCorrectionSet{ - { - Corrections: []proposal_generation.StructuredCorrectionProposal{ - {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}, - }, - }, - }, - } - +func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) { cfg := runnerTestConfig() - diagDir := t.TempDir() + cfg.Chunking.MaxSectionTokens = 3 + cfg.Chunking.MinSectionTokens = 0 + transcript := &schema.Transcript{Segments: []schema.Segment{ + {ID: 71, Text: "alpha one"}, + {ID: 903, Text: "bravo two"}, + }} + responses := []proposal_generation.StructuredCorrectionSet{ + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {SegmentReference: 1, OriginalText: "one", CorrectedText: "ONE"}, + {SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {SegmentReference: 1, OriginalText: "two", CorrectedText: "TWO"}, + }}, + } + completedLaterSection := make(chan struct{}) + client := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + if req.Context.SectionIndex == nil { + return contracts.StructuredCompletionResponse{}, errors.New("missing section index") + } + index := *req.Context.SectionIndex + if index < 0 || index >= len(responses) { + return contracts.StructuredCompletionResponse{}, errors.New("unexpected section index") + } + if index == 0 { + select { + case <-completedLaterSection: + case <-ctx.Done(): + return contracts.StructuredCompletionResponse{}, ctx.Err() + } + } + *out.(*proposal_generation.StructuredCorrectionSet) = responses[index] + return contracts.StructuredCompletionResponse{}, nil + }) - r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique}, - }}) - - out, err := r.Run(context.Background(), RunInput{ - Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, - ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}}, - Interactions: llm.NewDiagnosticsWriter(diagDir), + dir := t.TempDir() + writer := llm.NewDiagnosticsWriter(dir) + interactions := proposalOrderSink{writer: writer, laterResponse: completedLaterSection} + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": &grammar.Module{}}}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + out, err := r.Run(ctx, RunInput{ + Config: &cfg, Transcript: transcript, Interactions: interactions, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client}}, }) if err != nil { - t.Fatalf("unexpected run error: %v", err) + t.Fatal(err) } - if out.FinalTranscript.Segments[0].Text != "the cat" { - t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text) + want := []struct { + target int + original string + }{{71, "one"}, {71, "alpha"}, {903, "two"}} + changes := out.ModuleResults[0].AppliedChanges + if len(changes) != len(want) { + t.Fatalf("applied changes: %+v", changes) } - if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 { - t.Fatalf("expected one applied change, got %+v", out.ModuleResults) + for i, change := range changes { + if change.ProposalIndex != i || change.TargetSegmentID != want[i].target || change.OriginalText != want[i].original { + t.Fatalf("change %d lost response order or segment identity: %+v", i, change) + } } - matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json")) - if globErr != nil { - t.Fatalf("glob diagnostics: %v", globErr) + wantIndices := [][]int{{0, 1}, {2}} + if len(out.WorkUnits) != len(wantIndices) { + t.Fatalf("work units: %+v", out.WorkUnits) } - if len(matches) == 0 { - t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m")) + for i, unit := range out.WorkUnits { + if len(unit.Operations) != 1 || !reflect.DeepEqual(unit.Operations[0].ProposalIndices, wantIndices[i]) { + t.Fatalf("section %d lost proposal indices: %+v", i, unit.Operations) + } } + + paths, err := filepath.Glob(filepath.Join(dir, "grammar", "*response-payload.json")) + if err != nil || len(paths) != len(responses) { + t.Fatalf("proposal diagnostic paths=%v error=%v", paths, err) + } + for i, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var evidence struct { + Decoded proposal_generation.StructuredCorrectionSet `json:"decoded"` + } + if err := json.Unmarshal(raw, &evidence); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(evidence.Decoded, responses[i]) { + t.Fatalf("section %d lost original references: %+v", i, evidence.Decoded) + } + } +} + +type proposalOrderSink struct { + writer *llm.DiagnosticsWriter + laterResponse chan struct{} +} + +func (s proposalOrderSink) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) { + artifacts, err := s.writer.WriteInteraction(interaction) + if interaction.Context.SectionIndex != nil && *interaction.Context.SectionIndex == 1 { + close(s.laterResponse) + } + return artifacts, err } func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {