diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index f304b77..b22c45b 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -264,6 +264,9 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n" } configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs")) + if mode == "decision-mixed" || mode == "all-validation-failure" || mode == "invalid-decision-after-repair" { + configText = strings.Replace(configText, " transient_retries:", " correction_repair_attempts: 0\n transient_retries:", 1) + } if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n" } diff --git a/docs/internal/validators.md b/docs/internal/validators.md index 1131bc1..fa03188 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -46,6 +46,11 @@ For each module section: 3. record decisions and compact operation outcomes; 4. carry only approved proposals forward. +When correction repair is enabled, the runner also gathers independent findings +for rejected candidates when a validator has usable evidence. Unavailable +assessments remain unavailable, and only candidates that pass every applicable +check can advance. A zero repair budget retains the original shortlist flow. + Decision cardinality is enforced inside Audita: each candidate proposal must receive exactly one outcome per validator it reaches. The runner still rejects duplicate, missing, or unknown identities in an internal validator result. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e2b4622..13bb11f 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -172,6 +172,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery) ## Stage 5 — Collect usable findings while preserving approval semantics +**Completed.** + 1. Refactor `validateSectionCandidates` in `internal/framework/runner` so the initial pass and later revisions use the same validation code. Keep the existing shortlist flow when the effective repair budget is zero. In diff --git a/internal/framework/runner/feedback.go b/internal/framework/runner/feedback.go new file mode 100644 index 0000000..6e4a50d --- /dev/null +++ b/internal/framework/runner/feedback.go @@ -0,0 +1,87 @@ +package runner + +import ( + "math" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" + validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" +) + +func feedbackPreview(input validateSectionCandidatesInput, candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult { + if input.WorkingTranscript != nil { + for i := range input.WorkingTranscript.Segments { + segment := &input.WorkingTranscript.Segments[i] + if segment.ID == candidate.TargetSegmentID { + correction := candidate.CorrectionProposal + if math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1 { + correction.Confidence = 0.5 + } + return proposals.PreviewProposalForSegment(segment, correction, input.Policy) + } + } + } + return proposals.PreviewProposalForSegment(nil, candidate.CorrectionProposal, input.Policy) +} + +// feedbackCandidates submits only candidates for which the validator has usable evidence. +func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool) []proposals.EnrichedCorrectionProposal { + var result []proposals.EnrichedCorrectionProposal + for _, candidate := range input.SectionEnriched { + if noEffect[candidate.ProposalIndex] { + continue + } + preview := feedbackPreview(input, candidate) + usableSpan := candidate.OriginalText != "" + target := candidate.TargetSegmentID > 0 + confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1 + var applicable bool + switch validator.Name() { + case "proposal_shape": + applicable = true + case "no_effect": + applicable = usableSpan + case "confidence_threshold": + applicable = confidence + case "original_text_presence": + applicable = usableSpan && target + case "protected_terms": + applicable = usableSpan + if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord { + applicable = preview.Applicable + } + case "non_empty_resulting_segment": + applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText + default: + if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked { + applicable = preview.Applicable + } else { + applicable = true + } + } + if applicable { + result = append(result, candidate) + } + } + return result +} + +// repairFindings extracts ordered, safe feedback for one candidate version. +func repairFindings(index int, rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding { + var findings []proposal_generation.RepairFinding + for _, decision := range rejected { + if decision.ProposalIndex != index { + continue + } + finding := proposal_generation.RepairFinding{Validator: decision.ValidatorName, Reason: decision.ReasonCode, Explanation: decision.Message} + if decision.Disposition == validators.DispositionUnavailable { + finding.Status = "unavailable" + finding.Explanation = "assessment was unavailable" + } + findings = append(findings, finding) + } + return findings +} diff --git a/internal/framework/runner/feedback_test.go b/internal/framework/runner/feedback_test.go new file mode 100644 index 0000000..0ea7001 --- /dev/null +++ b/internal/framework/runner/feedback_test.go @@ -0,0 +1,87 @@ +package runner + +import ( + "context" + "fmt" + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" + validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" +) + +func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) { + candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal { + return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}} + } + var seen []int + semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + var decisions []validators.Decision + for _, p := range req.CandidateProposal { + seen = append(seen, p.ProposalIndex) + decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "semantic_rejection", Message: "meaning changed"}) + } + return validators.Result{Decisions: decisions}, nil + }}, validatormetadata.ExecutionClassLLMBacked) + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}} + result, err := validateSectionCandidates(context.Background(), input) + if err != nil { + t.Fatal(err) + } + reasons := map[int]map[string]bool{} + for _, d := range result.rejected { + if reasons[d.ProposalIndex] == nil { + reasons[d.ProposalIndex] = map[string]bool{} + } + reasons[d.ProposalIndex][d.ReasonCode] = true + } + if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] { + t.Fatalf("findings = %+v", reasons) + } + if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 { + t.Fatalf("approved=%v semantic input=%v", result.approved, seen) + } +} + +func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) { + candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} + for _, budget := range []int{0, 1} { + t.Run(fmt.Sprint(budget), func(t *testing.T) { + var later int + first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + var decisions []validators.Decision + for _, p := range req.CandidateProposal { + decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"}) + } + return validators.Result{Decisions: decisions}, nil + }} + second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + later = len(req.CandidateProposal) + var decisions []validators.Decision + for _, p := range req.CandidateProposal { + decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}) + } + return validators.Result{Decisions: decisions}, nil + }} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} + result, err := validateSectionCandidates(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(result.approved) != 0 || len(result.rejected) != 1 || result.rejected[0].Disposition != validators.DispositionUnavailable || later != budget { + t.Fatalf("result=%+v later=%d", result, later) + } + }) + } +} + +func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) { + rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}} + findings := repairFindings(2, rejected) + if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" { + t.Fatalf("findings: %+v", findings) + } +} diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 42c6b61..5c537b2 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -613,6 +613,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida operations := make([]outcomes.Operation, 0) filtered := make([]outcomes.FilteredProposal, 0) eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) + feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 + blocked := make(map[int]bool) + noEffect := make(map[int]bool) var interactionWriter *llmDiagnosticsWriterAdapter for validatorOrder, validator := range input.Validators { @@ -633,10 +636,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida diagnosticsWriter = interactionWriter } + submitted := eligible + if feedback { + submitted = feedbackCandidates(input, validator, noEffect) + } vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ StageIndex: &input.Spec.Index, StageID: input.Spec.ID, WorkingTranscript: input.WorkingTranscript, - CandidateProposal: eligible, + CandidateProposal: submitted, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, @@ -654,9 +661,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida var decisionErr error if err != nil { - decisionErr = validators.EnforcePartialDecisionCardinality(eligible, vResult.Decisions) + decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions) } else { - decisionErr = validators.EnforceDecisionCardinality(eligible, vResult.Decisions) + decisionErr = validators.EnforceDecisionCardinality(submitted, vResult.Decisions) } if decisionErr != nil { return validateSectionCandidatesResult{ @@ -664,13 +671,28 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida }, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr)) } - nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible)) - byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible)) - for _, p := range eligible { + nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted)) + byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(submitted)) + for _, p := range submitted { byIndex[p.ProposalIndex] = p } for _, d := range vResult.Decisions { + if feedback && validator.Name() == "original_text_presence" && d.Approved { + preview := feedbackPreview(input, byIndex[d.ProposalIndex]) + if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal { + d.Approved = false + d.Disposition = validators.DispositionRejected + d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal) + d.Message = "original_text matches multiple locations in the target segment" + } + } + if !d.Approved { + blocked[d.ProposalIndex] = true + if d.ReasonCode == validators.ReasonNoEffect { + noEffect[d.ProposalIndex] = true + } + } decisions = append(decisions, ValidatorDecisionRecord{ ValidatorName: validator.Name(), ProposalIndex: d.ProposalIndex, @@ -714,6 +736,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida eligible = nextEligible } + if feedback { + eligible = eligible[:0] + for _, candidate := range input.SectionEnriched { + if !blocked[candidate.ProposalIndex] { + eligible = append(eligible, candidate) + } + } + } return validateSectionCandidatesResult{ approved: eligible, decisions: decisions,