eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Retain repair diagnostics and correction lineage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:37:52 +00:00
parent 735dfc1834
commit f4c638313e
15 changed files with 308 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
internal/framework/runner/feedback_test.go
View File

@@ -3,6 +3,7 @@ package runner
import (
"context"
"fmt"
"os"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -85,3 +86,54 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
t.Fatalf("findings: %+v", findings)
}
}
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
attempt := 1
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" {
t.Fatalf("decisions: %+v", result.decisions)
}
paths = append(paths, result.decisions[0].DiagnosticArtifactPath)
}
if paths[0] == paths[1] {
t.Fatalf("colliding paths: %v", paths)
}
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
t.Fatal(err)
}
}
}
func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
t.Fatalf("result: %+v", result)
}
}