package runner import ( "context" "errors" "net/http" "os" "path/filepath" "sync/atomic" "testing" "time" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" ck "go.maximumdirect.net/configkit" pk "go.maximumdirect.net/promptkit" ) type contextValidator struct { validate func(context.Context, validators.Request) (validators.Result, error) } func (v contextValidator) Name() string { return "cancel-test" } func (v contextValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) { return v.validate(ctx, req) } type cancelTransport struct { entered chan struct{} calls atomic.Int32 } func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) { t.calls.Add(1) t.entered <- struct{}{} <-req.Context().Done() return nil, req.Context().Err() } func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { dir := t.TempDir() main := filepath.Join(dir, "config.yml") if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { t.Fatal(err) } resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: base\n backend: shared\n model: local\n" if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil { t.Fatal(err) } loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") if err != nil { t.Fatal(err) } transport := &cancelTransport{entered: make(chan struct{}, 3)} runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } collector := llm.NewObservationCollector() hooks := collector.Hooks() started := make(chan struct{}, 2) hooks.OnEvent = func(event pk.ExecutionEvent) { if event.Kind == pk.ExecutionStarted { started <- struct{}{} } } client, err := runtime.NewRoleClient("validation", "base", config.LLM{OperationTimeout: 30 * time.Second}, hooks) if err != nil { t.Fatal(err) } origin := errors.New("originating validator failure") ctx, cancel := context.WithCancel(context.Background()) defer cancel() validator := contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) { if *req.SectionIndex == 0 { // Hold the failure until one call is active and both peer executions started. for i := 0; i < 2; i++ { select { case <-started: case <-ctx.Done(): return validators.Result{}, ctx.Err() } } select { case <-transport.entered: case <-ctx.Done(): return validators.Result{}, ctx.Err() } return validators.Result{}, origin } var out map[string]any _, err := client.CompleteStructured(ctx, structuredoutput.Request{ PromptID: prompts.PromptIDValidatorEditorialReview, Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey, Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex}, }, &out) return validators.Result{}, err }} module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil }} sections := []chunking.Section{} for i := 0; i < 3; i++ { sections = append(sections, chunking.Section{Index: i, Segments: []schema.Segment{{ID: 1, Text: "hello"}}}) } type pipelineOutcome struct { result modulePipelineResult err error } done := make(chan pipelineOutcome, 1) go func() { result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Sections: sections}) done <- pipelineOutcome{result: result, err: err} }() select { case outcome := <-done: if !errors.Is(outcome.err, origin) { t.Fatalf("lost originating failure: %v", outcome.err) } if len(outcome.result.WorkUnits) != 3 { t.Fatalf("lost work inventory: %+v", outcome.result.WorkUnits) } var failure *ExecutionFailure if !errors.As(outcome.err, &failure) || failure.Phase != "validation" || !failure.HasSection || failure.Origin.SectionIndex != 0 || failure.Origin.ValidatorName != validator.Name() || outcome.result.WorkUnits[0].Failure != failure { t.Fatalf("lost primary validation owner: failure=%+v units=%+v", failure, outcome.result.WorkUnits) } for _, peer := range outcome.result.WorkUnits[1:] { if peer.State != outcomes.WorkInterrupted || peer.Failure == nil || !errors.Is(peer.Failure, context.Canceled) || peer.Failure == failure { t.Fatalf("lost peer interruption: %+v", peer) } } if len(outcome.result.Approved) != 0 { t.Fatalf("fatal validation returned unchecked candidates: %+v", outcome.result.Approved) } case <-time.After(2 * time.Second): t.Fatal("validator failure did not cancel and join peers") } if transport.calls.Load() != 1 { t.Fatalf("pending call dispatched: %d", transport.calls.Load()) } snapshot := collector.Snapshot() if len(snapshot.Calls) != 2 { t.Fatalf("partial observations missing: %+v", snapshot) } for _, call := range snapshot.Calls { if call.Finished.IsZero() || call.Failure != structuredoutput.FailureRuntime { t.Fatalf("unfinished/cause lost: %+v", call) } } if got := llm.Summarize(snapshot); got.ProviderAttempts != 1 { t.Fatalf("attempt count: %+v", got) } } func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() out, err := New(fakeFactory{}).Run(ctx, RunInput{ Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original"}}}, }) if !errors.Is(err, context.Canceled) || out.FinalTranscript == nil || out.FinalTranscript.Segments[0].Text != "original" { t.Fatalf("canceled entry: result=%+v error=%v", out, err) } } func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) { origin := errors.New("earlier proposal failed") laterValidationStarted := make(chan struct{}) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() module := outcomeModule{ fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{ contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) { close(laterValidationStarted) <-ctx.Done() return validators.Result{}, ctx.Err() }}, }}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { if req.Section.Index == 0 { select { case <-laterValidationStarted: return contracts.ProposalResult{}, origin case <-ctx.Done(): return contracts.ProposalResult{}, ctx.Err() } } return contracts.ProposalResult{ Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}}, SegmentReferences: []int{1}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}, }, nil }, } cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true}, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, }) if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 { t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err) } var failure *ExecutionFailure if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection || failure.Origin.SectionIndex != 0 || failure.Origin.Kind != outcomes.OperationProposal || out.WorkUnits[0].Failure != failure { t.Fatalf("lost originating proposal failure: %+v", failure) } candidate := out.Candidates[0] later := out.WorkUnits[1] if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) || len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex || out.FinalTranscript.Segments[1].Text != "other text" { t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later) } } func TestRunnerCancellationAfterApprovalKeepsCommittedModuleText(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() propose := func(original, replacement string) func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: replacement}}, nil } } last := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) { cancel() return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "first": fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("teh", "the")}, "second": fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("cat", "dog"), validators: []validators.Validator{last}}, }}) out, err := r.Run(ctx, RunInput{ Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {Index: 1, ModuleKey: "second", InstanceName: "second"}}, }) var failure *ExecutionFailure if !errors.Is(err, context.Canceled) || !errors.As(err, &failure) || failure.Origin == nil || *failure.Origin.StageIndex != 1 || len(out.ModuleResults) != 2 || out.ModuleResults[0].Status != ModuleStatusSuccess || out.ModuleResults[1].Status != ModuleStatusFailed || len(out.ModuleResults[1].AppliedChanges) != 0 || len(out.ModuleResults[1].ValidatorDecisions) != 1 || !out.ModuleResults[1].ValidatorDecisions[0].Approved || out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("commit boundary lost earlier text or approval evidence: result=%+v error=%+v", out, failure) } }