Collect independent correction validation findings
This commit is contained in:
@@ -264,6 +264,9 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
||||
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
|
||||
}
|
||||
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
|
||||
if mode == "decision-mixed" || mode == "all-validation-failure" || mode == "invalid-decision-after-repair" {
|
||||
configText = strings.Replace(configText, " transient_retries:", " correction_repair_attempts: 0\n transient_retries:", 1)
|
||||
}
|
||||
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||
}
|
||||
|
||||
@@ -46,6 +46,11 @@ For each module section:
|
||||
3. record decisions and compact operation outcomes;
|
||||
4. carry only approved proposals forward.
|
||||
|
||||
When correction repair is enabled, the runner also gathers independent findings
|
||||
for rejected candidates when a validator has usable evidence. Unavailable
|
||||
assessments remain unavailable, and only candidates that pass every applicable
|
||||
check can advance. A zero repair budget retains the original shortlist flow.
|
||||
|
||||
Decision cardinality is enforced inside Audita: each candidate proposal must
|
||||
receive exactly one outcome per validator it reaches. The runner still rejects
|
||||
duplicate, missing, or unknown identities in an internal validator result.
|
||||
|
||||
@@ -172,6 +172,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
|
||||
|
||||
## Stage 5 — Collect usable findings while preserving approval semantics
|
||||
|
||||
**Completed.**
|
||||
|
||||
1. Refactor `validateSectionCandidates` in `internal/framework/runner` so the
|
||||
initial pass and later revisions use the same validation code. Keep the
|
||||
existing shortlist flow when the effective repair budget is zero. In
|
||||
|
||||
87
internal/framework/runner/feedback.go
Normal file
87
internal/framework/runner/feedback.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
func feedbackPreview(input validateSectionCandidatesInput, candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult {
|
||||
if input.WorkingTranscript != nil {
|
||||
for i := range input.WorkingTranscript.Segments {
|
||||
segment := &input.WorkingTranscript.Segments[i]
|
||||
if segment.ID == candidate.TargetSegmentID {
|
||||
correction := candidate.CorrectionProposal
|
||||
if math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1 {
|
||||
correction.Confidence = 0.5
|
||||
}
|
||||
return proposals.PreviewProposalForSegment(segment, correction, input.Policy)
|
||||
}
|
||||
}
|
||||
}
|
||||
return proposals.PreviewProposalForSegment(nil, candidate.CorrectionProposal, input.Policy)
|
||||
}
|
||||
|
||||
// feedbackCandidates submits only candidates for which the validator has usable evidence.
|
||||
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool) []proposals.EnrichedCorrectionProposal {
|
||||
var result []proposals.EnrichedCorrectionProposal
|
||||
for _, candidate := range input.SectionEnriched {
|
||||
if noEffect[candidate.ProposalIndex] {
|
||||
continue
|
||||
}
|
||||
preview := feedbackPreview(input, candidate)
|
||||
usableSpan := candidate.OriginalText != ""
|
||||
target := candidate.TargetSegmentID > 0
|
||||
confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1
|
||||
var applicable bool
|
||||
switch validator.Name() {
|
||||
case "proposal_shape":
|
||||
applicable = true
|
||||
case "no_effect":
|
||||
applicable = usableSpan
|
||||
case "confidence_threshold":
|
||||
applicable = confidence
|
||||
case "original_text_presence":
|
||||
applicable = usableSpan && target
|
||||
case "protected_terms":
|
||||
applicable = usableSpan
|
||||
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
|
||||
applicable = preview.Applicable
|
||||
}
|
||||
case "non_empty_resulting_segment":
|
||||
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
|
||||
default:
|
||||
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
|
||||
applicable = preview.Applicable
|
||||
} else {
|
||||
applicable = true
|
||||
}
|
||||
}
|
||||
if applicable {
|
||||
result = append(result, candidate)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// repairFindings extracts ordered, safe feedback for one candidate version.
|
||||
func repairFindings(index int, rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
|
||||
var findings []proposal_generation.RepairFinding
|
||||
for _, decision := range rejected {
|
||||
if decision.ProposalIndex != index {
|
||||
continue
|
||||
}
|
||||
finding := proposal_generation.RepairFinding{Validator: decision.ValidatorName, Reason: decision.ReasonCode, Explanation: decision.Message}
|
||||
if decision.Disposition == validators.DispositionUnavailable {
|
||||
finding.Status = "unavailable"
|
||||
finding.Explanation = "assessment was unavailable"
|
||||
}
|
||||
findings = append(findings, finding)
|
||||
}
|
||||
return findings
|
||||
}
|
||||
87
internal/framework/runner/feedback_test.go
Normal file
87
internal/framework/runner/feedback_test.go
Normal file
@@ -0,0 +1,87 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) {
|
||||
candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
}
|
||||
var seen []int
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
seen = append(seen, p.ProposalIndex)
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "semantic_rejection", Message: "meaning changed"})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}, validatormetadata.ExecutionClassLLMBacked)
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reasons := map[int]map[string]bool{}
|
||||
for _, d := range result.rejected {
|
||||
if reasons[d.ProposalIndex] == nil {
|
||||
reasons[d.ProposalIndex] = map[string]bool{}
|
||||
}
|
||||
reasons[d.ProposalIndex][d.ReasonCode] = true
|
||||
}
|
||||
if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
|
||||
t.Fatalf("findings = %+v", reasons)
|
||||
}
|
||||
if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 {
|
||||
t.Fatalf("approved=%v semantic input=%v", result.approved, seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
for _, budget := range []int{0, 1} {
|
||||
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
||||
var later int
|
||||
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
later = len(req.CandidateProposal)
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.approved) != 0 || len(result.rejected) != 1 || result.rejected[0].Disposition != validators.DispositionUnavailable || later != budget {
|
||||
t.Fatalf("result=%+v later=%d", result, later)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
||||
rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
|
||||
findings := repairFindings(2, rejected)
|
||||
if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" {
|
||||
t.Fatalf("findings: %+v", findings)
|
||||
}
|
||||
}
|
||||
@@ -613,6 +613,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
operations := make([]outcomes.Operation, 0)
|
||||
filtered := make([]outcomes.FilteredProposal, 0)
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
|
||||
blocked := make(map[int]bool)
|
||||
noEffect := make(map[int]bool)
|
||||
var interactionWriter *llmDiagnosticsWriterAdapter
|
||||
|
||||
for validatorOrder, validator := range input.Validators {
|
||||
@@ -633,10 +636,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
diagnosticsWriter = interactionWriter
|
||||
}
|
||||
|
||||
submitted := eligible
|
||||
if feedback {
|
||||
submitted = feedbackCandidates(input, validator, noEffect)
|
||||
}
|
||||
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
WorkingTranscript: input.WorkingTranscript,
|
||||
CandidateProposal: eligible,
|
||||
CandidateProposal: submitted,
|
||||
ModuleKey: input.Spec.ModuleKey,
|
||||
ModuleInstance: input.Spec.InstanceName,
|
||||
SectionIndex: input.SectionIndex,
|
||||
@@ -654,9 +661,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
|
||||
var decisionErr error
|
||||
if err != nil {
|
||||
decisionErr = validators.EnforcePartialDecisionCardinality(eligible, vResult.Decisions)
|
||||
decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions)
|
||||
} else {
|
||||
decisionErr = validators.EnforceDecisionCardinality(eligible, vResult.Decisions)
|
||||
decisionErr = validators.EnforceDecisionCardinality(submitted, vResult.Decisions)
|
||||
}
|
||||
if decisionErr != nil {
|
||||
return validateSectionCandidatesResult{
|
||||
@@ -664,13 +671,28 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
|
||||
}
|
||||
|
||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
||||
for _, p := range eligible {
|
||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted))
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(submitted))
|
||||
for _, p := range submitted {
|
||||
byIndex[p.ProposalIndex] = p
|
||||
}
|
||||
|
||||
for _, d := range vResult.Decisions {
|
||||
if feedback && validator.Name() == "original_text_presence" && d.Approved {
|
||||
preview := feedbackPreview(input, byIndex[d.ProposalIndex])
|
||||
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
|
||||
d.Approved = false
|
||||
d.Disposition = validators.DispositionRejected
|
||||
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
|
||||
d.Message = "original_text matches multiple locations in the target segment"
|
||||
}
|
||||
}
|
||||
if !d.Approved {
|
||||
blocked[d.ProposalIndex] = true
|
||||
if d.ReasonCode == validators.ReasonNoEffect {
|
||||
noEffect[d.ProposalIndex] = true
|
||||
}
|
||||
}
|
||||
decisions = append(decisions, ValidatorDecisionRecord{
|
||||
ValidatorName: validator.Name(),
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
@@ -714,6 +736,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
eligible = nextEligible
|
||||
}
|
||||
|
||||
if feedback {
|
||||
eligible = eligible[:0]
|
||||
for _, candidate := range input.SectionEnriched {
|
||||
if !blocked[candidate.ProposalIndex] {
|
||||
eligible = append(eligible, candidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
return validateSectionCandidatesResult{
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
|
||||
Reference in New Issue
Block a user