Retain repair diagnostics and correction lineage

2026-11-11 04:37:52 +00:00
parent 735dfc1834
commit f4c638313e
15 changed files with 308 additions and 38 deletions
--- a/internal/framework/runner/feedback_test.go
+++ b/internal/framework/runner/feedback_test.go
@@ -3,6 +3,7 @@ package runner
 import (
 	"context"
 	"fmt"
+	"os"
 	"testing"

 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -85,3 +86,54 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
 		t.Fatalf("findings: %+v", findings)
 	}
 }
+
+func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
+	client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
+		{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
+		{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
+	}}
+	validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dir := t.TempDir()
+	attempt := 1
+	var paths []string
+	for _, index := range []int{2, 3} {
+		candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
+		input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
+		result, err := validateSectionCandidates(context.Background(), input)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" {
+			t.Fatalf("decisions: %+v", result.decisions)
+		}
+		paths = append(paths, result.decisions[0].DiagnosticArtifactPath)
+	}
+	if paths[0] == paths[1] {
+		t.Fatalf("colliding paths: %v", paths)
+	}
+	for _, path := range paths {
+		if _, err := os.Stat(path); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
+	client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
+	validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
+	if err != nil {
+		t.Fatal(err)
+	}
+	candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
+	input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
+	result, err := validateSectionCandidates(context.Background(), input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
+		t.Fatalf("result: %+v", result)
+	}
+}
--- a/internal/framework/runner/repair.go
+++ b/internal/framework/runner/repair.go
@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"path/filepath"

+	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -15,6 +17,7 @@ import (

 // RepairAttempt retains compact facts for a single semantic request.
 type RepairAttempt struct {
+	Artifacts     llm.InteractionArtifacts
 	Revision      *proposals.EnrichedCorrectionProposal
 	Decisions     []ValidatorDecisionRecord
 	Rejected      []ValidatorRejectedChange
@@ -90,8 +93,38 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
 				return initial, branches, err
 			}
 			var response proposal_generation.RepairResponse
-			completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex}}, &response)
+			proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
+			completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
 			record := RepairAttempt{Counts: completion.Execution}
+			save := func(err error) {
+				if input.DiagnosticsDir == "" {
+					return
+				}
+				var secrets []string
+				if source, ok := input.Spec.RepairClient.(interface {
+					ConfiguredSecrets(context.Context) []string
+				}); ok {
+					secrets = source.ConfiguredSecrets(ctx)
+				}
+				section := 0
+				if input.SectionIndex != nil {
+					section = *input.SectionIndex
+				}
+				name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
+				var failure any
+				if err != nil {
+					failure = map[string]string{"error": err.Error()}
+				}
+				writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), secrets)
+				artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": record.Rejected}, failure)
+				if writeErr != nil {
+					if input.ReportDiagnosticError != nil {
+						input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
+					}
+					return
+				}
+				record.Artifacts = artifacts
+			}
 			if callErr != nil {
 				var failure *structuredoutput.Failure
 				if errors.As(callErr, &failure) {
@@ -99,6 +132,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
 				} else {
 					record.FailureReason = outcomes.ReasonInternalFailure
 				}
+				save(callErr)
 				branch.Attempts = append(branch.Attempts, record)
 				if structuredoutput.IsRecoverable(callErr) {
 					branch.Outcome = "unavailable"
@@ -110,6 +144,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
 			decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
 			if err != nil {
 				record.FailureReason = outcomes.ReasonInvalidOutput
+				save(err)
 				branch.Attempts = append(branch.Attempts, record)
 				branch.Outcome = "unavailable"
 				break
@@ -118,6 +153,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
 			replay.Turns = append(replay.Turns, completion.RepairInstruction...)
 			replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
 			if decoded.Action == "withdraw" {
+				save(nil)
 				branch.Attempts = append(branch.Attempts, record)
 				branch.Outcome = "withdrawn"
 				break
@@ -132,6 +168,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
 			current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
 			branch.Final = current
 			next := input
+			next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber
 			next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
 			validated, validationErr := validateSectionCandidates(ctx, next)
 			revisionSnapshot := current
@@ -144,9 +181,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
 				validated.operations[i].Reference.RepairAttempt = &attemptNumber
 			}
 			record.Operations = validated.operations
+			save(validationErr)
 			branch.Attempts = append(branch.Attempts, record)
 			initial.operations = append(initial.operations, validated.operations...)
-			initial.decisions = append(initial.decisions, validated.decisions...)
 			if validationErr != nil {
 				branches = append(branches, branch)
 				return initial, branches, validationErr
--- a/internal/framework/runner/repair_test.go
+++ b/internal/framework/runner/repair_test.go
@@ -4,6 +4,9 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"

 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -20,6 +23,7 @@ type repairClient struct {
 	outputs []string
 	calls   []structuredoutput.Request
 	err     error
+	secrets []string
 }

 func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
@@ -36,9 +40,11 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
 	if err := json.Unmarshal([]byte(value), out); err != nil {
 		return structuredoutput.Response{}, err
 	}
-	return structuredoutput.Response{Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
+	return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
 }

+func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
+
 func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
 	client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
 	candidates := []proposals.EnrichedCorrectionProposal{
@@ -182,3 +188,55 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
 		t.Fatalf("branches=%+v error=%v", branches, err)
 	}
 }
+
+func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
+	client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
+	candidates := []proposals.EnrichedCorrectionProposal{
+		{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
+		{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
+	}
+	input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
+	initial, err := validateSectionCandidates(context.Background(), input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(branches) != 2 {
+		t.Fatalf("branches: %+v", branches)
+	}
+	a, b := branches[0].Attempts[0].Artifacts.RequestPayloadPath, branches[1].Attempts[0].Artifacts.RequestPayloadPath
+	if a == "" || a == b {
+		t.Fatalf("artifact paths: %q %q", a, b)
+	}
+	for _, path := range []string{a, b} {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
+			t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
+		}
+	}
+}
+
+func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
+	client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
+	path := filepath.Join(t.TempDir(), "file")
+	if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	warnings := 0
+	candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
+	input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
+	initial, err := validateSectionCandidates(context.Background(), input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
+	if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 {
+		t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
+	}
+}
--- a/internal/framework/runner/runner.go
+++ b/internal/framework/runner/runner.go
@@ -646,7 +646,9 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
 }

 type validateSectionCandidatesInput struct {
-	Exchange *structuredoutput.Exchange
+	RepairProposalIndex *int
+	RepairAttempt       *int
+	Exchange            *structuredoutput.Exchange
 	// ReportDiagnosticError may be called concurrently; it must not affect processing.
 	ReportDiagnosticError func(error) `json:"-"`
 	Spec                  contracts.ModuleRunSpec
@@ -703,11 +705,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
 		}
 		vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
 			StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
-			WorkingTranscript:     input.WorkingTranscript,
-			CandidateProposal:     submitted,
-			ModuleKey:             input.Spec.ModuleKey,
-			ModuleInstance:        input.Spec.InstanceName,
-			SectionIndex:          input.SectionIndex,
+			WorkingTranscript:   input.WorkingTranscript,
+			CandidateProposal:   submitted,
+			ModuleKey:           input.Spec.ModuleKey,
+			ModuleInstance:      input.Spec.InstanceName,
+			SectionIndex:        input.SectionIndex,
+			IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
 			ValidatorOrder:        &validatorOrder,
 			ReplacementPolicy:     input.Policy,
 			Glossary:              input.Spec.Glossary,