Collect independent correction validation findings
This commit is contained in:
@@ -264,6 +264,9 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
|
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
|
||||||
}
|
}
|
||||||
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
|
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
|
||||||
|
if mode == "decision-mixed" || mode == "all-validation-failure" || mode == "invalid-decision-after-repair" {
|
||||||
|
configText = strings.Replace(configText, " transient_retries:", " correction_repair_attempts: 0\n transient_retries:", 1)
|
||||||
|
}
|
||||||
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||||
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,6 +46,11 @@ For each module section:
|
|||||||
3. record decisions and compact operation outcomes;
|
3. record decisions and compact operation outcomes;
|
||||||
4. carry only approved proposals forward.
|
4. carry only approved proposals forward.
|
||||||
|
|
||||||
|
When correction repair is enabled, the runner also gathers independent findings
|
||||||
|
for rejected candidates when a validator has usable evidence. Unavailable
|
||||||
|
assessments remain unavailable, and only candidates that pass every applicable
|
||||||
|
check can advance. A zero repair budget retains the original shortlist flow.
|
||||||
|
|
||||||
Decision cardinality is enforced inside Audita: each candidate proposal must
|
Decision cardinality is enforced inside Audita: each candidate proposal must
|
||||||
receive exactly one outcome per validator it reaches. The runner still rejects
|
receive exactly one outcome per validator it reaches. The runner still rejects
|
||||||
duplicate, missing, or unknown identities in an internal validator result.
|
duplicate, missing, or unknown identities in an internal validator result.
|
||||||
|
|||||||
@@ -172,6 +172,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
|
|||||||
|
|
||||||
## Stage 5 — Collect usable findings while preserving approval semantics
|
## Stage 5 — Collect usable findings while preserving approval semantics
|
||||||
|
|
||||||
|
**Completed.**
|
||||||
|
|
||||||
1. Refactor `validateSectionCandidates` in `internal/framework/runner` so the
|
1. Refactor `validateSectionCandidates` in `internal/framework/runner` so the
|
||||||
initial pass and later revisions use the same validation code. Keep the
|
initial pass and later revisions use the same validation code. Keep the
|
||||||
existing shortlist flow when the effective repair budget is zero. In
|
existing shortlist flow when the effective repair budget is zero. In
|
||||||
|
|||||||
87
internal/framework/runner/feedback.go
Normal file
87
internal/framework/runner/feedback.go
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||||
|
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||||
|
)
|
||||||
|
|
||||||
|
func feedbackPreview(input validateSectionCandidatesInput, candidate proposals.EnrichedCorrectionProposal) proposals.SegmentPreviewResult {
|
||||||
|
if input.WorkingTranscript != nil {
|
||||||
|
for i := range input.WorkingTranscript.Segments {
|
||||||
|
segment := &input.WorkingTranscript.Segments[i]
|
||||||
|
if segment.ID == candidate.TargetSegmentID {
|
||||||
|
correction := candidate.CorrectionProposal
|
||||||
|
if math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1 {
|
||||||
|
correction.Confidence = 0.5
|
||||||
|
}
|
||||||
|
return proposals.PreviewProposalForSegment(segment, correction, input.Policy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return proposals.PreviewProposalForSegment(nil, candidate.CorrectionProposal, input.Policy)
|
||||||
|
}
|
||||||
|
|
||||||
|
// feedbackCandidates submits only candidates for which the validator has usable evidence.
|
||||||
|
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool) []proposals.EnrichedCorrectionProposal {
|
||||||
|
var result []proposals.EnrichedCorrectionProposal
|
||||||
|
for _, candidate := range input.SectionEnriched {
|
||||||
|
if noEffect[candidate.ProposalIndex] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
preview := feedbackPreview(input, candidate)
|
||||||
|
usableSpan := candidate.OriginalText != ""
|
||||||
|
target := candidate.TargetSegmentID > 0
|
||||||
|
confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1
|
||||||
|
var applicable bool
|
||||||
|
switch validator.Name() {
|
||||||
|
case "proposal_shape":
|
||||||
|
applicable = true
|
||||||
|
case "no_effect":
|
||||||
|
applicable = usableSpan
|
||||||
|
case "confidence_threshold":
|
||||||
|
applicable = confidence
|
||||||
|
case "original_text_presence":
|
||||||
|
applicable = usableSpan && target
|
||||||
|
case "protected_terms":
|
||||||
|
applicable = usableSpan
|
||||||
|
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
|
||||||
|
applicable = preview.Applicable
|
||||||
|
}
|
||||||
|
case "non_empty_resulting_segment":
|
||||||
|
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
|
||||||
|
default:
|
||||||
|
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
|
||||||
|
applicable = preview.Applicable
|
||||||
|
} else {
|
||||||
|
applicable = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if applicable {
|
||||||
|
result = append(result, candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// repairFindings extracts ordered, safe feedback for one candidate version.
|
||||||
|
func repairFindings(index int, rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
|
||||||
|
var findings []proposal_generation.RepairFinding
|
||||||
|
for _, decision := range rejected {
|
||||||
|
if decision.ProposalIndex != index {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
finding := proposal_generation.RepairFinding{Validator: decision.ValidatorName, Reason: decision.ReasonCode, Explanation: decision.Message}
|
||||||
|
if decision.Disposition == validators.DispositionUnavailable {
|
||||||
|
finding.Status = "unavailable"
|
||||||
|
finding.Explanation = "assessment was unavailable"
|
||||||
|
}
|
||||||
|
findings = append(findings, finding)
|
||||||
|
}
|
||||||
|
return findings
|
||||||
|
}
|
||||||
87
internal/framework/runner/feedback_test.go
Normal file
87
internal/framework/runner/feedback_test.go
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||||
|
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) {
|
||||||
|
candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal {
|
||||||
|
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||||
|
}
|
||||||
|
var seen []int
|
||||||
|
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
|
var decisions []validators.Decision
|
||||||
|
for _, p := range req.CandidateProposal {
|
||||||
|
seen = append(seen, p.ProposalIndex)
|
||||||
|
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "semantic_rejection", Message: "meaning changed"})
|
||||||
|
}
|
||||||
|
return validators.Result{Decisions: decisions}, nil
|
||||||
|
}}, validatormetadata.ExecutionClassLLMBacked)
|
||||||
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}}
|
||||||
|
result, err := validateSectionCandidates(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reasons := map[int]map[string]bool{}
|
||||||
|
for _, d := range result.rejected {
|
||||||
|
if reasons[d.ProposalIndex] == nil {
|
||||||
|
reasons[d.ProposalIndex] = map[string]bool{}
|
||||||
|
}
|
||||||
|
reasons[d.ProposalIndex][d.ReasonCode] = true
|
||||||
|
}
|
||||||
|
if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
|
||||||
|
t.Fatalf("findings = %+v", reasons)
|
||||||
|
}
|
||||||
|
if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 {
|
||||||
|
t.Fatalf("approved=%v semantic input=%v", result.approved, seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||||
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||||
|
for _, budget := range []int{0, 1} {
|
||||||
|
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
||||||
|
var later int
|
||||||
|
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
|
var decisions []validators.Decision
|
||||||
|
for _, p := range req.CandidateProposal {
|
||||||
|
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
|
||||||
|
}
|
||||||
|
return validators.Result{Decisions: decisions}, nil
|
||||||
|
}}
|
||||||
|
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||||
|
later = len(req.CandidateProposal)
|
||||||
|
var decisions []validators.Decision
|
||||||
|
for _, p := range req.CandidateProposal {
|
||||||
|
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
|
||||||
|
}
|
||||||
|
return validators.Result{Decisions: decisions}, nil
|
||||||
|
}}
|
||||||
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||||
|
result, err := validateSectionCandidates(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(result.approved) != 0 || len(result.rejected) != 1 || result.rejected[0].Disposition != validators.DispositionUnavailable || later != budget {
|
||||||
|
t.Fatalf("result=%+v later=%d", result, later)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
||||||
|
rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
|
||||||
|
findings := repairFindings(2, rejected)
|
||||||
|
if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" {
|
||||||
|
t.Fatalf("findings: %+v", findings)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -613,6 +613,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
operations := make([]outcomes.Operation, 0)
|
operations := make([]outcomes.Operation, 0)
|
||||||
filtered := make([]outcomes.FilteredProposal, 0)
|
filtered := make([]outcomes.FilteredProposal, 0)
|
||||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||||
|
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
|
||||||
|
blocked := make(map[int]bool)
|
||||||
|
noEffect := make(map[int]bool)
|
||||||
var interactionWriter *llmDiagnosticsWriterAdapter
|
var interactionWriter *llmDiagnosticsWriterAdapter
|
||||||
|
|
||||||
for validatorOrder, validator := range input.Validators {
|
for validatorOrder, validator := range input.Validators {
|
||||||
@@ -633,10 +636,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
diagnosticsWriter = interactionWriter
|
diagnosticsWriter = interactionWriter
|
||||||
}
|
}
|
||||||
|
|
||||||
|
submitted := eligible
|
||||||
|
if feedback {
|
||||||
|
submitted = feedbackCandidates(input, validator, noEffect)
|
||||||
|
}
|
||||||
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||||
WorkingTranscript: input.WorkingTranscript,
|
WorkingTranscript: input.WorkingTranscript,
|
||||||
CandidateProposal: eligible,
|
CandidateProposal: submitted,
|
||||||
ModuleKey: input.Spec.ModuleKey,
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
ModuleInstance: input.Spec.InstanceName,
|
ModuleInstance: input.Spec.InstanceName,
|
||||||
SectionIndex: input.SectionIndex,
|
SectionIndex: input.SectionIndex,
|
||||||
@@ -654,9 +661,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
|
|
||||||
var decisionErr error
|
var decisionErr error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
decisionErr = validators.EnforcePartialDecisionCardinality(eligible, vResult.Decisions)
|
decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions)
|
||||||
} else {
|
} else {
|
||||||
decisionErr = validators.EnforceDecisionCardinality(eligible, vResult.Decisions)
|
decisionErr = validators.EnforceDecisionCardinality(submitted, vResult.Decisions)
|
||||||
}
|
}
|
||||||
if decisionErr != nil {
|
if decisionErr != nil {
|
||||||
return validateSectionCandidatesResult{
|
return validateSectionCandidatesResult{
|
||||||
@@ -664,13 +671,28 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
|
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
|
||||||
}
|
}
|
||||||
|
|
||||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted))
|
||||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(submitted))
|
||||||
for _, p := range eligible {
|
for _, p := range submitted {
|
||||||
byIndex[p.ProposalIndex] = p
|
byIndex[p.ProposalIndex] = p
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, d := range vResult.Decisions {
|
for _, d := range vResult.Decisions {
|
||||||
|
if feedback && validator.Name() == "original_text_presence" && d.Approved {
|
||||||
|
preview := feedbackPreview(input, byIndex[d.ProposalIndex])
|
||||||
|
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
|
||||||
|
d.Approved = false
|
||||||
|
d.Disposition = validators.DispositionRejected
|
||||||
|
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
|
||||||
|
d.Message = "original_text matches multiple locations in the target segment"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !d.Approved {
|
||||||
|
blocked[d.ProposalIndex] = true
|
||||||
|
if d.ReasonCode == validators.ReasonNoEffect {
|
||||||
|
noEffect[d.ProposalIndex] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
decisions = append(decisions, ValidatorDecisionRecord{
|
decisions = append(decisions, ValidatorDecisionRecord{
|
||||||
ValidatorName: validator.Name(),
|
ValidatorName: validator.Name(),
|
||||||
ProposalIndex: d.ProposalIndex,
|
ProposalIndex: d.ProposalIndex,
|
||||||
@@ -714,6 +736,14 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
eligible = nextEligible
|
eligible = nextEligible
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if feedback {
|
||||||
|
eligible = eligible[:0]
|
||||||
|
for _, candidate := range input.SectionEnriched {
|
||||||
|
if !blocked[candidate.ProposalIndex] {
|
||||||
|
eligible = append(eligible, candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
return validateSectionCandidatesResult{
|
return validateSectionCandidatesResult{
|
||||||
approved: eligible,
|
approved: eligible,
|
||||||
decisions: decisions,
|
decisions: decisions,
|
||||||
|
|||||||
Reference in New Issue
Block a user