package runner import ( "context" "encoding/json" "errors" "os" "path/filepath" "reflect" "strings" "sync" "testing" "time" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) func runnerTestConfig() config.Config { return config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048}, Validation: config.Validation{MaxPromptTokens: 2048}} } type fakeFactory struct { modules map[string]contracts.TranscriptModule } func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) { m, ok := f.modules[spec.InstanceName] if !ok { return nil, errors.New("module not registered") } return m, nil } type fakeModule struct { key string policy proposals.ReplacementPolicy validators []validators.Validator proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) } type outcomeModule struct { fakeModule proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error) } func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) { candidate := proposals.EnrichedCorrectionProposal{ CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}, } contractFailure := errors.New("validator also returned a runtime error") for _, cause := range []error{nil, contractFailure} { contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) { return validators.Result{Decisions: []validators.Decision{{ ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable, }}}, cause }} later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) { t.Fatal("candidate advanced past a contradictory decision") return validators.Result{}, nil }} result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{ Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, }) if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) || len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" { t.Fatalf("contradictory result = %+v (%v)", result, err) } } } func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) { failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonTransportFailed}}}, nil }} second := fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"first": failed, "second": second}}) out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {ModuleKey: "second", InstanceName: "second"}}}) if err != nil { t.Fatal(err) } if out.FinalTranscript.Segments[0].Text != "the" || len(out.WorkUnits) != 2 || out.WorkUnits[0].State != outcomes.WorkCompleted || out.WorkUnits[1].CorrectionsApplied != 1 { t.Fatalf("recoverable failure lost later work: %+v", out) } if len(out.WorkUnits[0].Operations) != 1 || out.WorkUnits[0].Operations[0].FailureReason != outcomes.ReasonTransportFailed { t.Fatalf("missing proposal failure: %+v", out.WorkUnits[0]) } } func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) { secondSaw := make([]int, 0) partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) != 2 { t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal)) } return validators.Result{ValidatorName: "partial", Decisions: []validators.Decision{ {ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: false, Disposition: validators.DispositionUnavailable, ReasonCode: string(outcomes.ReasonTransportFailed)}, {ProposalIndex: req.CandidateProposal[1].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}, }, Operations: []outcomes.Operation{ {Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonTransportFailed, Cause: errors.New("transport")}, {Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonProviderFailed, Cause: errors.New("provider")}, }}, nil }} last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) { for _, candidate := range req.CandidateProposal { secondSaw = append(secondSaw, candidate.ProposalIndex) } return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil }} module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"}, {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, }, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired}}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}}) out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatal(err) } if out.FinalTranscript.Segments[0].Text != "teh dog" || len(secondSaw) != 1 || secondSaw[0] != 2 { t.Fatalf("unchecked candidate applied or survivor lost: text=%q final=%v", out.FinalTranscript.Segments[0].Text, secondSaw) } unit := out.WorkUnits[0] if unit.State != outcomes.WorkCompleted || unit.CorrectionsApplied != 1 || len(unit.Operations) != 3 { t.Fatalf("expected one completed unit with two operation failures: %+v", unit) } proposal := unit.Operations[0] if len(proposal.RecoveryReasons) != 2 || proposal.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || proposal.RecoveryReasons[1] != outcomes.RecoveryProposalsFiltered || len(proposal.Filtered) != 1 || proposal.Filtered[0].ProposalIndex != 0 { t.Fatalf("repair and filtering must share one recovery: %+v", proposal) } if len(proposal.ProposalIndices) != 0 || unit.Operations[1].Cause != nil || unit.Operations[2].Cause != nil || out.Totals.ModelOperations != 3 || out.Totals.WithErrors != 1 || out.Totals.Recoveries != 1 { t.Fatalf("report detail or counts: operations=%+v totals=%+v", unit.Operations, out.Totals) } } func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) { origin := errors.New("validator setup failed") proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}} broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin }} result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}}) if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 { t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err) } } func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) { module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}) input := RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}} without, err := r.Run(context.Background(), input) if err != nil { t.Fatal(err) } if len(without.Candidates) != 0 { t.Fatalf("debug-off candidate capture: %+v", without.Candidates) } input.Evidence = &EvidenceSelection{Reports: true, Debug: true} with, err := r.Run(context.Background(), input) if err != nil { t.Fatal(err) } if len(with.Candidates) != 1 || with.Candidates[0].ProposalIndex != 0 || len(with.ModuleResults[0].ValidatorRejected) != 1 { t.Fatalf("filtered candidate missing from debug evidence: %+v", with) } } func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { return m.proposeResult(req) } func (m fakeModule) Key() string { return m.key } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m fakeModule) Validators() []validators.Validator { return m.validators } func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { if m.proposeF == nil { return contracts.ProposalResult{}, nil } proposalsOut, err := m.proposeF(req) return contracts.ProposalResult{Proposals: proposalsOut}, err } type fakeValidator struct { name string validateF func(req validators.Request) (validators.Result, error) } func (v fakeValidator) Name() string { return v.name } func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) { _ = ctx return v.validateF(req) } type classifiedFakeValidator struct { fakeValidator class validatormetadata.ExecutionClass } func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionClass { return v.class } func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) { llm := classifiedFakeValidator{ fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "llm"}, nil }}, class: validatormetadata.ExecutionClassLLMBacked, } deterministic := classifiedFakeValidator{ fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "deterministic"}, nil }}, class: validatormetadata.ExecutionClassDeterministic, } ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic}) if len(ordered) != 2 { t.Fatalf("expected 2 validators, got %d", len(ordered)) } if ordered[0].Name() != "deterministic" || ordered[1].Name() != "llm" { t.Fatalf("unexpected validator order: %s, %s", ordered[0].Name(), ordered[1].Name()) } } func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) { unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "plain"}, nil }} llm := classifiedFakeValidator{ fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "llm"}, nil }}, class: validatormetadata.ExecutionClassLLMBacked, } ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified}) if len(ordered) != 2 { t.Fatalf("expected 2 validators, got %d", len(ordered)) } if ordered[0].Name() != "plain" || ordered[1].Name() != "llm" { t.Fatalf("unexpected validator order with unclassified validator: %s, %s", ordered[0].Name(), ordered[1].Name()) } } func TestRunnerOneModuleAppliesProposal(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected corrected text, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 { t.Fatalf("expected one module with one applied change, got %+v", out.ModuleResults) } } func TestRunnerModulesRunSequentially(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if req.WorkingTranscript.Segments[0].Text != "the cat" { t.Fatalf("second module did not see first module changes: %q", req.WorkingTranscript.Segments[0].Text) } return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the dog" { t.Fatalf("expected sequential updates, got %q", out.FinalTranscript.Segments[0].Text) } } func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Speaker: "A", Start: 0, End: 1, Text: "a"}, {ID: 2, Speaker: "B", Start: 1, End: 2, Text: "b"}, {ID: 3, Speaker: "A", Start: 2, End: 3, Text: "c"}, {ID: 4, Speaker: "B", Start: 3, End: 4, Text: "d"}, }} var mu sync.Mutex seen := make(map[int][]int) updatedText := "a one two three four five" r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "grow": fakeModule{key: "grow", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if req.Section.Index != 0 { return nil, nil } return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "a", CorrectedText: updatedText}}, nil }}, "inspect": fakeModule{key: "inspect", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { ids := make([]int, len(req.WorkingTranscript.Segments)) for i, segment := range req.WorkingTranscript.Segments { ids[i] = segment.ID } mu.Lock() seen[req.Section.Index] = ids mu.Unlock() if req.Section.Index == 0 && req.WorkingTranscript.Segments[0].Text != updatedText { return nil, errors.New("later module did not see updated section text") } if req.Section.Index == 0 { return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C"}}, nil } return nil, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grow", InstanceName: "grow"}, {ModuleKey: "inspect", InstanceName: "inspect"}}}) if err != nil { t.Fatal(err) } if len(out.Sections) != 2 || out.Sections[0].StartSegmentID != 1 || out.Sections[0].EndSegmentID != 2 || out.Sections[0].EstimatedTokens != 2 || len(out.WorkUnits) != 4 { t.Fatalf("initial inventory changed: sections=%+v units=%+v", out.Sections, out.WorkUnits) } if len(seen) != 2 || len(seen[0]) != 2 || seen[0][0] != 1 || seen[0][1] != 2 || len(seen[1]) != 2 || seen[1][0] != 3 || seen[1][1] != 4 { t.Fatalf("later module saw changed section membership: %+v", seen) } if len(out.FinalTranscript.Segments) != 4 || out.FinalTranscript.Segments[0].Text != updatedText || out.FinalTranscript.Segments[2].Text != "c" || out.FinalTranscript.Segments[3].ID != 4 { t.Fatalf("recombined transcript: %+v", out.FinalTranscript) } for i, segment := range out.FinalTranscript.Segments { original := transcript.Segments[i] if segment.ID != original.ID || segment.Speaker != original.Speaker || segment.Start != original.Start || segment.End != original.End { t.Fatalf("segment %d metadata changed: %+v", i, segment) } } if out.WorkUnits[0].CorrectionsApplied != 1 || out.WorkUnits[1].CorrectionsApplied != 0 { t.Fatalf("section application counts: %+v", out.WorkUnits) } } func TestRunnerInventoriesEmptyWorkAndRejectsImpossibleChunking(t *testing.T) { r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{}}) empty, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{}}) if err != nil || len(empty.Sections) != 0 || len(empty.WorkUnits) != 0 || len(empty.FinalTranscript.Segments) != 0 { t.Fatalf("empty transcript and pipeline: %+v, %v", empty, err) } input := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one"}, {ID: 2, Text: "two"}}} noModules, err := r.Run(context.Background(), RunInput{Transcript: input}) if err != nil || len(noModules.Sections) != 1 || len(noModules.WorkUnits) != 0 || len(noModules.FinalTranscript.Segments) != 2 { t.Fatalf("empty pipeline inventory: %+v, %v", noModules, err) } cfg := runnerTestConfig() target := 3 cfg.Chunking.TargetSections = &target if _, err := r.Run(context.Background(), RunInput{Config: &cfg, Transcript: input}); err == nil { t.Fatal("impossible section target was accepted") } } func TestRunnerProposalsExecutePerChunkSection(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Text: "one two"}, {ID: 2, Text: "three four"}, {ID: 3, Text: "five six"}, }} var ( mu sync.Mutex seenBySection = make(map[int]contracts.SectionMetadata) segmentCountBySec = make(map[int]int) ) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if req.Section == nil { t.Fatalf("expected section metadata on proposal request") } mu.Lock() seenBySection[req.Section.Index] = *req.Section segmentCountBySec[req.Section.Index] = len(req.WorkingTranscript.Segments) mu.Unlock() return nil, nil }}, }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{ {ModuleKey: "m", InstanceName: "m"}, }, }) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults) != 1 { t.Fatalf("expected one module result, got %+v", out.ModuleResults) } if !reflect.DeepEqual(out.FinalTranscript, transcript) || len(out.WorkUnits) != 3 { t.Fatalf("no-edit run changed the transcript or inventory: %+v", out) } for _, unit := range out.WorkUnits { if unit.State != outcomes.WorkCompleted || unit.CorrectionsApplied != 0 { t.Fatalf("no-edit section was not completed: %+v", unit) } } if len(seenBySection) != 3 { t.Fatalf("expected 3 chunked proposal calls, got %d", len(seenBySection)) } for i := 0; i < 3; i++ { section, ok := seenBySection[i] if !ok { t.Fatalf("expected section %d to be observed", i) } if section.Index != i { t.Fatalf("expected section index %d, got %+v", i, section) } if segmentCountBySec[i] != 1 { t.Fatalf("expected one segment per section call, got %d at index %d", segmentCountBySec[i], i) } } } func mapKeys(values map[int]struct{}) []int { keys := make([]int, 0, len(values)) for key := range values { keys = append(keys, key) } return keys } func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Text: "alpha one"}, {ID: 2, Text: "bravo two"}, }} releaseSectionOne := make(chan struct{}) validatorStarted := make(chan struct{}, 1) validator := fakeValidator{ name: "capture", validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 { select { case validatorStarted <- struct{}{}: default: } } decisions := make([]validators.Decision, 0, len(req.CandidateProposal)) for _, p := range req.CandidateProposal { decisions = append(decisions, validators.Decision{ ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok", }) } return validators.Result{ValidatorName: "capture", Decisions: decisions}, nil }, } r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{ key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { seg := req.WorkingTranscript.Segments[0] if req.Section != nil && req.Section.Index == 1 { <-releaseSectionOne } return []proposals.CorrectionProposal{ {TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)}, }, nil }, }, }}) resultCh := make(chan error, 1) go func() { _, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{ {ModuleKey: "m", InstanceName: "m"}, }, }) resultCh <- err }() select { case <-validatorStarted: case <-time.After(350 * time.Millisecond): t.Fatal("expected section-level validator work before all proposal sections complete") } close(releaseSectionOne) if err := <-resultCh; err != nil { t.Fatalf("Run error: %v", err) } } func TestRunnerSkippedRecorded(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults[0].SkippedChanges) != 1 { t.Fatalf("expected one skipped change, got %+v", out.ModuleResults[0].SkippedChanges) } } func TestRunnerFailureReturnsPartialProgress(t *testing.T) { origin := errors.New("boom") transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, origin }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {Index: 1, ModuleKey: "m2", InstanceName: "m2"}}}) var failure *ExecutionFailure if !errors.Is(err, origin) || !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection || *failure.Origin.StageIndex != 1 || failure.Origin.ModuleInstance != "m2" || failure.Origin.SectionIndex != 0 { t.Fatalf("lost initiating proposal failure: %+v", failure) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial transcript progress preserved, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed { t.Fatalf("expected second module failed in results, got %+v", out.ModuleResults) } } func TestRunnerDoesNotMutateInputTranscript(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}} before := transcript.Segments[0].Text r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, }}) _, err := r.Run(context.Background(), RunInput{Config: ptrConfig(runnerTestConfig()), Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if transcript.Segments[0].Text != before { t.Fatalf("expected input transcript unchanged, got %q", transcript.Segments[0].Text) } } func TestRunnerRepeatedModuleInstanceNames(t *testing.T) { specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}}}}) if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" { t.Fatalf("unexpected instance names: %+v", specs) } } func TestRunnerValidatorApprovedProposalApplied(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, p := range req.CandidateProposal { decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"} } return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{ key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allowAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }, }, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected proposal applied, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults[0].ValidatorDecisions) != 1 { t.Fatalf("expected validator decisions recorded") } } func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, p := range req.CandidateProposal { decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"} } return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "teh cat" { t.Fatalf("expected rejected proposal not applied, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults[0].ValidatorRejected) != 1 { t.Fatalf("expected validator rejection recorded, got %+v", out.ModuleResults[0].ValidatorRejected) } } func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}} first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) != 2 { t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal)) } return validators.Result{ ValidatorName: "first", Decisions: []validators.Decision{ {ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}, {ProposalIndex: req.CandidateProposal[1].ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "reject"}, }, }, nil }} second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) != 1 { t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal)) } return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"}, {TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"}, }, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "ONE two" { t.Fatalf("expected only survivor applied, got %q", out.FinalTranscript.Segments[0].Text) } } func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }} badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) { // Missing one decision triggers cardinality error. return validators.Result{ValidatorName: "bad", Decisions: nil}, nil }} bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}}) if err == nil { t.Fatal("expected cardinality error") } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial progress preserved, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed { t.Fatalf("expected second module failed") } } func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}} allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": outcomeModule{ fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{ Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}, }, nil }, }, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults[0].SkippedChanges) != 1 { t.Fatalf("expected application skip recorded") } if len(out.ModuleResults[0].ValidatorRejected) != 0 { t.Fatalf("expected no validator rejection") } operations := out.WorkUnits[0].Operations if out.Totals.Corrections != 0 || out.Totals.FilteringResponses != 1 || out.Totals.Recoveries != 1 || len(operations) != 1 || len(operations[0].Filtered) != 1 || operations[0].Filtered[0].Reason != outcomes.ReasonCode(proposals.SkipReasonAmbiguousOriginal) { t.Fatalf("original defect lost during application: totals=%+v operations=%+v", out.Totals, operations) } } func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": outcomeModule{ fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{ Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "a", CorrectedText: ""}, {TargetSegmentID: 1, OriginalText: "b", CorrectedText: ""}, }, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}, }, nil }, }, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, Evidence: &EvidenceSelection{Reports: true, Debug: true}}) if err != nil { t.Fatal(err) } if got := out.FinalTranscript.Segments[0].Text; got != " b" { t.Fatalf("expected first deletion only, got %q", got) } if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].SkipReason != proposals.SkipReasonEmptyResultingText { t.Fatalf("expected skipped debug evidence: %+v", out.ModuleResults) } if len(out.ModuleResults[0].ValidatorRejected) != 0 || len(out.WorkUnits) != 1 || out.WorkUnits[0].CorrectionsApplied != 1 { t.Fatalf("unexpected validation or application outcome: %+v", out) } if len(out.WorkUnits[0].Operations) != 0 || out.Totals.ProposalResponses != 1 || out.Totals.WithErrors != 0 || out.Totals.Recoveries != 0 { t.Fatalf("apply-time conflict changed report facts: operations=%+v totals=%+v", out.WorkUnits[0].Operations, out.Totals) } } func ptrConfig(c config.Config) *config.Config { return &c } type fakeStructuredClient struct { responses []validators.LLMValidationResponse err error calls int } func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx _ = req f.calls++ if f.err != nil { return contracts.StructuredCompletionResponse{}, f.err } if len(f.responses) == 0 { return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") } resp := f.responses[0] f.responses = f.responses[1:] target := out.(*validators.LLMValidationResponse) *target = resp return contracts.StructuredCompletionResponse{}, nil } func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}} llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) if err != nil { t.Fatalf("NewLLMBackedValidator: %v", err) } r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "There were Jesters at the temple." { t.Fatalf("expected applied LLM-approved proposal, got %q", out.FinalTranscript.Segments[0].Text) } } func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "There were gestures at the temple." { t.Fatalf("expected rejected proposal not applied") } if len(out.ModuleResults[0].ValidatorRejected) != 1 { t.Fatalf("expected validator rejection record") } } func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil }}, }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("expected malformed validator response to downgrade, got %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial progress retained") } if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 { t.Fatalf("expected second module rejection, got %+v", out.ModuleResults) } if len(out.WorkUnits[1].Operations) != 1 || out.WorkUnits[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid { t.Fatalf("expected invalid decision-set operation, got %+v", out.WorkUnits[1].Operations) } } func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ {CorrectionIndex: "1", Approved: true, Reason: "ok"}, {CorrectionIndex: "3", Approved: true, Reason: "ok"}, {CorrectionIndex: "99", Approved: true, Reason: "unexpected"}, }}}} first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) var laterIndices []int later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) { decisions := make([]validators.Decision, 0, len(req.CandidateProposal)) for _, candidate := range req.CandidateProposal { laterIndices = append(laterIndices, candidate.ProposalIndex) approved := candidate.ProposalIndex == 0 disposition := validators.DispositionRejected if approved { disposition = validators.DispositionApproved } decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: "later_review"}) } return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil }} module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good"}, }, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 { t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err) } unit := out.WorkUnits[0] if unit.CorrectionsApplied != 1 || out.Totals.ProposalResponses != 1 || len(unit.Operations) != 1 || unit.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || len(unit.Operations[0].ProposalIndices) != 1 || unit.Operations[0].ProposalIndices[0] != 1 || len(unit.Operations[0].RecoveryReasons) != 0 { t.Fatalf("operation accounting: %+v", unit) } moduleResult := out.ModuleResults[0] if len(moduleResult.ValidatorDecisions) != 5 || len(moduleResult.ValidatorRejected) != 2 || moduleResult.ValidatorDecisions[2].Disposition != validators.DispositionUnavailable || moduleResult.ValidatorRejected[0].Disposition != validators.DispositionUnavailable || moduleResult.ValidatorRejected[1].Disposition != validators.DispositionRejected { t.Fatalf("validator dispositions: %+v", moduleResult) } } func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) cfg := runnerTestConfig() _, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("expected missing decision downgrade, got %v", err) } } func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ {CorrectionIndex: "1", Approved: true, Reason: "ok"}, {CorrectionIndex: "1", Approved: false, Reason: "dup"}, }}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) cfg := runnerTestConfig() _, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("expected duplicate decision downgrade, got %v", err) } } type lenEstimator struct{} func (lenEstimator) EstimateTokens(text string) int { return len(text) } func TestRunnerLLMValidatorBatching(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ {Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}, {Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}, }} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator.SetTokenEstimator(lenEstimator{}) cfg := runnerTestConfig() cfg.Validation.MaxPromptTokens = 260 r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)}, {TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)}, }, nil }}, }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, }) if err != nil { t.Fatalf("unexpected run error: %v", err) } if client.calls < 2 { t.Fatalf("expected multiple llm calls for batching, got %d", client.calls) } if len(out.ModuleResults[0].AppliedChanges) != 2 { t.Fatalf("expected both approved proposals applied") } } func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) cfg := runnerTestConfig() diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, Interactions: llm.NewDiagnosticsWriter(diagDir), }) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" { t.Fatalf("expected diagnostic artifact path on decision") } matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*.json")) if globErr != nil { t.Fatalf("glob diagnostics: %v", globErr) } if len(matches) == 0 { t.Fatalf("expected diagnostics JSON artifacts under %s", filepath.Join(diagDir, "m")) } for _, path := range matches { raw, readErr := os.ReadFile(path) if readErr != nil { t.Fatalf("read diagnostic %q: %v", path, readErr) } if !json.Valid(raw) { t.Fatalf("invalid diagnostic %q: %s", path, raw) } } raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath) if readErr != nil { t.Fatalf("read decision diagnostic: %v", readErr) } if !strings.Contains(string(raw), "supported by context") { t.Fatalf("missing decision evidence: %s", raw) } } type proposalClientFunc func(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error) func (f proposalClientFunc) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { return f(ctx, req, out) } func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 71, Text: "alpha one"}, {ID: 903, Text: "bravo two"}, }} responses := []proposal_generation.StructuredCorrectionSet{ {Corrections: []proposal_generation.StructuredCorrectionProposal{ {SegmentReference: 1, OriginalText: "one", CorrectedText: "ONE"}, {SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}, }}, {Corrections: []proposal_generation.StructuredCorrectionProposal{ {SegmentReference: 1, OriginalText: "two", CorrectedText: "TWO"}, }}, } completedLaterSection := make(chan struct{}) client := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { if req.Context.SectionIndex == nil { return contracts.StructuredCompletionResponse{}, errors.New("missing section index") } index := *req.Context.SectionIndex if index < 0 || index >= len(responses) { return contracts.StructuredCompletionResponse{}, errors.New("unexpected section index") } if index == 0 { select { case <-completedLaterSection: case <-ctx.Done(): return contracts.StructuredCompletionResponse{}, ctx.Err() } } *out.(*proposal_generation.StructuredCorrectionSet) = responses[index] return contracts.StructuredCompletionResponse{}, nil }) dir := t.TempDir() writer := llm.NewDiagnosticsWriter(dir) interactions := proposalOrderSink{writer: writer, laterResponse: completedLaterSection} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": &grammar.Module{}}}) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out, err := r.Run(ctx, RunInput{ Config: &cfg, Transcript: transcript, Interactions: interactions, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client}}, }) if err != nil { t.Fatal(err) } want := []struct { target int original string }{{71, "one"}, {71, "alpha"}, {903, "two"}} changes := out.ModuleResults[0].AppliedChanges if len(changes) != len(want) { t.Fatalf("applied changes: %+v", changes) } for i, change := range changes { if change.ProposalIndex != i || change.TargetSegmentID != want[i].target || change.OriginalText != want[i].original { t.Fatalf("change %d lost response order or segment identity: %+v", i, change) } } paths, err := filepath.Glob(filepath.Join(dir, "grammar", "*response-payload.json")) if err != nil || len(paths) != len(responses) { t.Fatalf("proposal diagnostic paths=%v error=%v", paths, err) } for i, path := range paths { raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var evidence struct { Decoded proposal_generation.StructuredCorrectionSet `json:"decoded"` } if err := json.Unmarshal(raw, &evidence); err != nil { t.Fatal(err) } if !reflect.DeepEqual(evidence.Decoded, responses[i]) { t.Fatalf("section %d lost original references: %+v", i, evidence.Decoded) } } } type proposalOrderSink struct { writer *llm.DiagnosticsWriter laterResponse chan struct{} } func (s proposalOrderSink) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) { artifacts, err := s.writer.WriteInteraction(interaction) if interaction.Context.SectionIndex != nil && *interaction.Context.SectionIndex == 1 { close(s.laterResponse) } return artifacts, err } func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) { origin := errors.New("original fatal failure") candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}} for _, decisions := range [][]validators.Decision{ {{ProposalIndex: 8, Approved: true}}, {{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}}, } { v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) { return validators.Result{Decisions: decisions}, origin }} result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}) if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 { t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err) } } }