package runner import ( "context" "fmt" "os" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) { candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal { return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}} } var seen []int semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { var decisions []validators.Decision for _, p := range req.CandidateProposal { seen = append(seen, p.ProposalIndex) decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "semantic_rejection", Message: "meaning changed"}) } return validators.Result{Decisions: decisions}, nil }}, validatormetadata.ExecutionClassLLMBacked) input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) } reasons := map[int]map[string]bool{} for _, d := range result.rejected { if reasons[d.ProposalIndex] == nil { reasons[d.ProposalIndex] = map[string]bool{} } reasons[d.ProposalIndex][d.ReasonCode] = true } if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] { t.Fatalf("findings = %+v", reasons) } if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 { t.Fatalf("approved=%v semantic input=%v", result.approved, seen) } } func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) { candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} for _, budget := range []int{0, 1} { t.Run(fmt.Sprint(budget), func(t *testing.T) { var later int first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { var decisions []validators.Decision for _, p := range req.CandidateProposal { decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"}) } return validators.Result{Decisions: decisions}, nil }} second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { later = len(req.CandidateProposal) var decisions []validators.Decision for _, p := range req.CandidateProposal { decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}) } return validators.Result{Decisions: decisions}, nil }} input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) } if len(result.approved) != 0 || len(result.rejected) != 1 || result.rejected[0].Disposition != validators.DispositionUnavailable || later != budget { t.Fatalf("result=%+v later=%d", result, later) } }) } } func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) { rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}} findings := repairFindings(2, rejected) if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" { t.Fatalf("findings: %+v", findings) } } func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}, {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}, }} validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) if err != nil { t.Fatal(err) } dir := t.TempDir() attempt := 1 var paths []string for _, index := range []int{2, 3} { candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}} input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) } if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" { t.Fatalf("decisions: %+v", result.decisions) } paths = append(paths, result.decisions[0].DiagnosticArtifactPath) } if paths[0] == paths[1] { t.Fatalf("colliding paths: %v", paths) } for _, path := range paths { if _, err := os.Stat(path); err != nil { t.Fatal(err) } } } func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}} validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) if err != nil { t.Fatal(err) } candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) } if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved { t.Fatalf("result: %+v", result) } }