package runner import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "os" "path/filepath" "strings" "sync/atomic" "testing" "time" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar" ck "go.maximumdirect.net/configkit" ) type sharedBackendModule struct { *grammar.Module validator validators.Validator headEntered <-chan struct{} } func (m sharedBackendModule) Validators() []validators.Validator { return []validators.Validator{m.validator} } func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { if req.Section.Index > 0 { select { case <-m.headEntered: case <-ctx.Done(): return contracts.ProposalResult{}, ctx.Err() } } return m.Module.Propose(ctx, req) } func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) defer cancel() dir := t.TempDir() path := filepath.Join(dir, "config.yml") if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil { t.Fatal(err) } resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n" if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil { t.Fatal(err) } loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "") if err != nil { t.Fatal(err) } release := make(chan struct{}) entered := make(chan struct{}) validationStarted := make(chan struct{}, 1) var other, validating, active, peak atomic.Int32 client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) { body, err := io.ReadAll(r.Body) if err != nil { return nil, err } n := active.Add(1) for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() { } defer active.Add(-1) output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}` switch { case bytes.Contains(body, []byte(`"validations"`)): validating.Add(1) select { case validationStarted <- struct{}{}: default: } output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}` case bytes.Contains(body, []byte("marker-000")): close(entered) select { case <-release: case <-r.Context().Done(): return nil, r.Context().Err() } default: other.Add(1) } content, _ := json.Marshal(output) wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content) return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil })} runtime, err := llm.NewRuntime(ctx, loaded, client) if err != nil { t.Fatal(err) } role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{}) if err != nil { t.Fatal(err) } validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) if err != nil { t.Fatal(err) } module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}) cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 1 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{} for i := 0; i < 36; i++ { transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)}) } type result struct { out RunOutput err error } finished := make(chan result, 1) go func() { out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}}, }) finished <- result{out, err} }() select { case <-entered: case <-ctx.Done(): t.Fatal(ctx.Err()) } select { case <-validationStarted: case <-ctx.Done(): t.Fatal("later validation did not reach the backend while the earlier proposal held a slot") } close(release) completed := <-finished if completed.err != nil { t.Fatal(completed.err) } if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 { t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load()) } changes := completed.out.ModuleResults[0].AppliedChanges if len(changes) != len(transcript.Segments) { t.Fatalf("got %d applied changes", len(changes)) } for i, change := range changes { if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID { t.Fatalf("change %d: %+v", i, change) } } } func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) { for _, canceled := range []bool{false, true} { name := "completed" if canceled { name = "canceled" } t.Run(name, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() laterRevalidated := make(chan struct{}) proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) { section := *req.Context.SectionIndex if section == 0 { select { case <-laterRevalidated: case <-ctx.Done(): return contracts.StructuredCompletionResponse{}, ctx.Err() } } text := []string{"alpha", "bravo"}[section] dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{ SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text), }} return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil }) validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) { section := *req.Context.SectionIndex first := section == 0 revision := req.Context.RepairAttempt != nil if first && canceled { cancel() return contracts.StructuredCompletionResponse{}, ctx.Err() } dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{ CorrectionIndex: "1", Approved: first || revision, Reason: "checked", }} if revision { close(laterRevalidated) } return contracts.StructuredCompletionResponse{}, nil }) validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) if err != nil { t.Fatal(err) } module := sharedBackendModule{Module: &grammar.Module{}, validator: validator} // The module's transport gates are supplied by the scripted calls above. started := make(chan struct{}) close(started) module.headEntered = started repair := &repairClient{t: t, outputs: []string{ `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`, }} cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 1 cfg.Chunking.MinSectionTokens = 0 out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true}, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation, RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}}, }}, }) if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) { t.Fatal(err) } later := out.WorkUnits[1] if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 || len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 { t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs) } attempt := later.Repairs[0].Attempts[0] if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 || len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 { t.Fatalf("lost ordered repair evidence: %+v", attempt) } var repaired struct { Decoded proposal_generation.RepairResponse Revision *proposals.EnrichedCorrectionProposal Decisions []ValidatorDecisionRecord } readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired) if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 || len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 || repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath || !strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") { t.Fatalf("repair artifact uses different identities: %+v", repaired) } var checked struct { Decoded validators.LLMValidationResponse Reconciliation struct{ Decisions []validators.Decision } } readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked) if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" || len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 { t.Fatalf("validation evidence uses different identities: %+v", checked) } }) } } func readInteraction(t *testing.T, path string, result any) { t.Helper() raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if err := json.Unmarshal(raw, result); err != nil { t.Fatal(err) } }