Collect independent correction validation findings

This commit is contained in:
2026-11-11 04:23:34 +00:00
parent 9fd36001b4
commit a5003a1e52
6 changed files with 220 additions and 6 deletions

View File

@@ -264,6 +264,9 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n" pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
} }
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs")) configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
if mode == "decision-mixed" || mode == "all-validation-failure" || mode == "invalid-decision-after-repair" {
configText = strings.Replace(configText, " transient_retries:", " correction_repair_attempts: 0\n transient_retries:", 1)
}
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n" configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
} }

View File

@@ -46,6 +46,11 @@ For each module section:
3. record decisions and compact operation outcomes; 3. record decisions and compact operation outcomes;
4. carry only approved proposals forward. 4. carry only approved proposals forward.
When correction repair is enabled, the runner also gathers independent findings
for rejected candidates when a validator has usable evidence. Unavailable
assessments remain unavailable, and only candidates that pass every applicable
check can advance. A zero repair budget retains the original shortlist flow.
Decision cardinality is enforced inside Audita: each candidate proposal must Decision cardinality is enforced inside Audita: each candidate proposal must
receive exactly one outcome per validator it reaches. The runner still rejects receive exactly one outcome per validator it reaches. The runner still rejects
duplicate, missing, or unknown identities in an internal validator result. duplicate, missing, or unknown identities in an internal validator result.

View File

@@ -172,6 +172,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
## Stage 5 — Collect usable findings while preserving approval semantics ## Stage 5 — Collect usable findings while preserving approval semantics
**Completed.**
1. Refactor `validateSectionCandidates` in `internal/framework/runner` so the 1. Refactor `validateSectionCandidates` in `internal/framework/runner` so the
initial pass and later revisions use the same validation code. Keep the initial pass and later revisions use the same validation code. Keep the
existing shortlist flow when the effective repair budget is zero. In existing shortlist flow when the effective repair budget is zero. In

View File

@@ -0,0 +1,87 @@
package runner
import (
"math"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
func feedbackPreview(input validateSectionCandidatesInput, candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult {
if input.WorkingTranscript != nil {
for i := range input.WorkingTranscript.Segments {
segment := &input.WorkingTranscript.Segments[i]
if segment.ID == candidate.TargetSegmentID {
correction := candidate.CorrectionProposal
if math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1 {
correction.Confidence = 0.5
}
return proposals.PreviewProposalForSegment(segment, correction, input.Policy)
}
}
}
return proposals.PreviewProposalForSegment(nil, candidate.CorrectionProposal, input.Policy)
}
// feedbackCandidates submits only candidates for which the validator has usable evidence.
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool) []proposals.EnrichedCorrectionProposal {
var result []proposals.EnrichedCorrectionProposal
for _, candidate := range input.SectionEnriched {
if noEffect[candidate.ProposalIndex] {
continue
}
preview := feedbackPreview(input, candidate)
usableSpan := candidate.OriginalText != ""
target := candidate.TargetSegmentID > 0
confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1
var applicable bool
switch validator.Name() {
case "proposal_shape":
applicable = true
case "no_effect":
applicable = usableSpan
case "confidence_threshold":
applicable = confidence
case "original_text_presence":
applicable = usableSpan && target
case "protected_terms":
applicable = usableSpan
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
applicable = preview.Applicable
}
case "non_empty_resulting_segment":
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
default:
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
applicable = preview.Applicable
} else {
applicable = true
}
}
if applicable {
result = append(result, candidate)
}
}
return result
}
// repairFindings extracts ordered, safe feedback for one candidate version.
func repairFindings(index int, rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
var findings []proposal_generation.RepairFinding
for _, decision := range rejected {
if decision.ProposalIndex != index {
continue
}
finding := proposal_generation.RepairFinding{Validator: decision.ValidatorName, Reason: decision.ReasonCode, Explanation: decision.Message}
if decision.Disposition == validators.DispositionUnavailable {
finding.Status = "unavailable"
finding.Explanation = "assessment was unavailable"
}
findings = append(findings, finding)
}
return findings
}

View File

@@ -0,0 +1,87 @@
package runner
import (
"context"
"fmt"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) {
candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
}
var seen []int
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
seen = append(seen, p.ProposalIndex)
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "semantic_rejection", Message: "meaning changed"})
}
return validators.Result{Decisions: decisions}, nil
}}, validatormetadata.ExecutionClassLLMBacked)
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
reasons := map[int]map[string]bool{}
for _, d := range result.rejected {
if reasons[d.ProposalIndex] == nil {
reasons[d.ProposalIndex] = map[string]bool{}
}
reasons[d.ProposalIndex][d.ReasonCode] = true
}
if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
t.Fatalf("findings = %+v", reasons)
}
if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 {
t.Fatalf("approved=%v semantic input=%v", result.approved, seen)
}
}
func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
for _, budget := range []int{0, 1} {
t.Run(fmt.Sprint(budget), func(t *testing.T) {
var later int
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
}
return validators.Result{Decisions: decisions}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
later = len(req.CandidateProposal)
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
}
return validators.Result{Decisions: decisions}, nil
}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.approved) != 0 || len(result.rejected) != 1 || result.rejected[0].Disposition != validators.DispositionUnavailable || later != budget {
t.Fatalf("result=%+v later=%d", result, later)
}
})
}
}
func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
findings := repairFindings(2, rejected)
if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" {
t.Fatalf("findings: %+v", findings)
}
}

View File

@@ -613,6 +613,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
operations := make([]outcomes.Operation, 0) operations := make([]outcomes.Operation, 0)
filtered := make([]outcomes.FilteredProposal, 0) filtered := make([]outcomes.FilteredProposal, 0)
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
blocked := make(map[int]bool)
noEffect := make(map[int]bool)
var interactionWriter *llmDiagnosticsWriterAdapter var interactionWriter *llmDiagnosticsWriterAdapter
for validatorOrder, validator := range input.Validators { for validatorOrder, validator := range input.Validators {
@@ -633,10 +636,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
diagnosticsWriter = interactionWriter diagnosticsWriter = interactionWriter
} }
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect)
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript, WorkingTranscript: input.WorkingTranscript,
CandidateProposal: eligible, CandidateProposal: submitted,
ModuleKey: input.Spec.ModuleKey, ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName, ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex, SectionIndex: input.SectionIndex,
@@ -654,9 +661,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
var decisionErr error var decisionErr error
if err != nil { if err != nil {
decisionErr = validators.EnforcePartialDecisionCardinality(eligible, vResult.Decisions) decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions)
} else { } else {
decisionErr = validators.EnforceDecisionCardinality(eligible, vResult.Decisions) decisionErr = validators.EnforceDecisionCardinality(submitted, vResult.Decisions)
} }
if decisionErr != nil { if decisionErr != nil {
return validateSectionCandidatesResult{ return validateSectionCandidatesResult{
@@ -664,13 +671,28 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr)) }, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
} }
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible)) nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted))
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible)) byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(submitted))
for _, p := range eligible { for _, p := range submitted {
byIndex[p.ProposalIndex] = p byIndex[p.ProposalIndex] = p
} }
for _, d := range vResult.Decisions { for _, d := range vResult.Decisions {
if feedback && validator.Name() == "original_text_presence" && d.Approved {
preview := feedbackPreview(input, byIndex[d.ProposalIndex])
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
d.Approved = false
d.Disposition = validators.DispositionRejected
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
d.Message = "original_text matches multiple locations in the target segment"
}
}
if !d.Approved {
blocked[d.ProposalIndex] = true
if d.ReasonCode == validators.ReasonNoEffect {
noEffect[d.ProposalIndex] = true
}
}
decisions = append(decisions, ValidatorDecisionRecord{ decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(), ValidatorName: validator.Name(),
ProposalIndex: d.ProposalIndex, ProposalIndex: d.ProposalIndex,
@@ -714,6 +736,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
eligible = nextEligible eligible = nextEligible
} }
if feedback {
eligible = eligible[:0]
for _, candidate := range input.SectionEnriched {
if !blocked[candidate.ProposalIndex] {
eligible = append(eligible, candidate)
}
}
}
return validateSectionCandidatesResult{ return validateSectionCandidatesResult{
approved: eligible, approved: eligible,
decisions: decisions, decisions: decisions,