Retain repair diagnostics and correction lineage

This commit is contained in:
2026-11-11 04:37:52 +00:00
parent 735dfc1834
commit f4c638313e
15 changed files with 308 additions and 38 deletions

View File

@@ -4,6 +4,9 @@ import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -20,6 +23,7 @@ type repairClient struct {
outputs []string
calls []structuredoutput.Request
err error
secrets []string
}
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
@@ -36,9 +40,11 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
if err := json.Unmarshal([]byte(value), out); err != nil {
return structuredoutput.Response{}, err
}
return structuredoutput.Response{Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
}
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
@@ -182,3 +188,55 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
t.Fatalf("branches=%+v error=%v", branches, err)
}
}
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if len(branches) != 2 {
t.Fatalf("branches: %+v", branches)
}
a, b := branches[0].Attempts[0].Artifacts.RequestPayloadPath, branches[1].Attempts[0].Artifacts.RequestPayloadPath
if a == "" || a == b {
t.Fatalf("artifact paths: %q %q", a, b)
}
for _, path := range []string{a, b} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
}
}
}
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
t.Fatal(err)
}
warnings := 0
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 {
t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
}
}