eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Validate correction repair through the processing command

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:47:43 +00:00
parent f4c638313e
commit aa8073cab1
19 changed files with 511 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/framework/llm/native_adapter_test.go
View File

@@ -112,7 +112,12 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
t.Fatal(err)
}
var result map[string]any
response, err := client.CompleteStructured(context.Background(), request, &result)
capturing := request
capturing.CaptureExchange = true
response, err := client.CompleteStructured(context.Background(), capturing, &result)
if tc.success && (response.Exchange == nil || response.Exchange.FinalContent != tc.outputs[len(tc.outputs)-1]) {
t.Fatalf("captured final response: %+v", response.Exchange)
}
if (err == nil) != tc.success || calls != tc.wantCalls {
t.Fatalf("error=%v calls=%d", err, calls)
}

4
internal/framework/proposals/preview.go
View File

@@ -65,7 +65,7 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{
Applicable: true,
@@ -76,7 +76,7 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
case ReplacementPolicyReplaceAll:
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{
Applicable: true,

2
internal/framework/runner/feedback.go
View File

@@ -51,7 +51,7 @@ func feedbackCandidates(input validateSectionCandidatesInput, validator contract
case "protected_terms":
applicable = usableSpan
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
applicable = preview.Applicable
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
}
case "non_empty_resulting_segment":
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText

18
internal/framework/runner/feedback_test.go
View File

@@ -4,9 +4,11 @@ import (
"context"
"fmt"
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -137,3 +139,19 @@ func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
t.Fatalf("result: %+v", result)
}
}
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
identity := glossary.Identity{Collection: "party", ID: "jesters"}
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.ValidateProtectedTerms(req)
}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.rejected) != 2 || result.rejected[0].ReasonCode != validators.ReasonProtectedGlossaryTerm || !strings.Contains(result.rejected[0].Message, "Jesters") || result.rejected[1].ReasonCode != validators.ReasonEmptyResultingText {
t.Fatalf("findings: %+v", result.rejected)
}
}

33
internal/framework/runner/repair.go
View File

@@ -6,6 +6,8 @@ import (
"fmt"
"path/filepath"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -84,7 +86,15 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
}
current := original
findings := repairFindings(original.ProposalIndex, initial.rejected)
safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
findings := repairFindings(original.ProposalIndex, rejected)
redactor := redaction.New(repairSecrets(ctx, input))
for i := range findings {
findings[i].Explanation = redactor.String(findings[i].Explanation)
}
return findings
}
findings := safeFindings(initial.rejected)
replay := &structuredoutput.Replay{Exchange: input.Exchange}
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText, Confidence: current.Confidence}, Findings: findings}).JSON()
@@ -100,12 +110,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if input.DiagnosticsDir == "" {
return
}
var secrets []string
if source, ok := input.Spec.RepairClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
secrets := repairSecrets(ctx, input)
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
@@ -201,7 +206,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
initial.filtered = retained
break
}
findings = repairFindings(original.ProposalIndex, validated.rejected)
findings = safeFindings(validated.rejected)
if actionable(original.ProposalIndex, validated.rejected) {
if attempt+1 == input.Spec.Settings.LLM.CorrectionRepairAttempts {
branch.Outcome = "exhausted"
@@ -243,3 +248,15 @@ func repairOperations(input validateSectionCandidatesInput, branches []RepairOut
}
return result
}
func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string {
var secrets []string
for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} {
if source, ok := client.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = append(secrets, source.ConfiguredSecrets(ctx)...)
}
}
return secrets
}

38
internal/framework/runner/repair_test.go
View File

@@ -83,6 +83,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
}{
{"withdraw", []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}, nil, 1, "withdrawn", 1},
{"exhausted", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.1}}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.2}}`}, nil, 2, "exhausted", 2},
{"no-effect revision", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat","confidence":0.9}}`}, nil, 2, "rejected", 1},
{"wrong reference", []string{`{"proposal_ref":2,"action":"withdraw","correction":null}`}, nil, 1, "unavailable", 1},
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
} {
@@ -240,3 +241,40 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
}
}
func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
client := &repairClient{}
for _, rejected := range [][]ValidatorRejectedChange{
{{ProposalIndex: 0, Disposition: validators.DispositionRejected, ReasonCode: validators.ReasonNoEffect}},
{{ProposalIndex: 0, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}},
} {
candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
_, branches, err := repairCandidates(context.Background(), input, nil, validateSectionCandidatesResult{rejected: rejected}, "repair")
if err != nil || len(branches) != 0 || len(client.calls) != 0 {
t.Fatalf("branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
}
}
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
validation := &repairClient{secrets: []string{"validation-secret"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") {
t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"])
}
data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "validation-secret") {
t.Fatalf("credential in artifact: %s", data)
}
}

2
internal/framework/validators/deterministic.go
View File

@@ -156,7 +156,7 @@ func ValidateProtectedTerms(req Request) (Result, error) {
continue
}
preview := previewForEvidence(req, segment, c.CorrectionProposal)
if preview.Applicable {
if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) {
if glossaryStage {
// A correction span may sit inside a longer exempt name.
reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText)

30
internal/framework/validators/protected_terms.go
View File

@@ -1,6 +1,7 @@
package validators
import (
"fmt"
"regexp"
"sort"
"strings"
@@ -95,32 +96,35 @@ func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence {
}
func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var spellings []string
for identity, beforeItems := range beforeByID {
if len(afterByID[identity]) < len(beforeItems) {
return "proposal may alter protected glossary terminology"
if len(afterByID[identity]) < len(beforeItems) && len(beforeItems) > 0 {
spellings = append(spellings, beforeItems[0].canonical)
}
}
return ""
return protectedSpellingMessage(spellings)
}
func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var spellings []string
for identity, afterItems := range afterByID {
beforeItems := beforeByID[identity]
for i, afterItem := range afterItems {
if i >= len(beforeItems) {
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
continue
}
beforeItem := beforeItems[i]
if afterItem.text == beforeItem.text {
continue
}
if afterItem.text == afterItem.canonical {
continue
}
return "proposal may alter protected glossary terminology"
spellings = append(spellings, afterItem.canonical)
}
}
return ""
return protectedSpellingMessage(spellings)
}
func protectedSpellingMessage(spellings []string) string {
if len(spellings) == 0 {
return ""
}
sort.Strings(spellings)
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spellings[0])
}
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated