From f4c638313e96885f426d656bc51757d253d375eb Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 11 Nov 2026 04:37:52 +0000 Subject: [PATCH] Retain repair diagnostics and correction lineage --- docs/integrations/output-files.md | 3 + docs/internal/diagnostics-reporting.md | 4 +- docs/roadmap/implementation.md | 2 + internal/framework/llm/observations.go | 25 +++++--- .../processreport/correction_ledger.go | 53 ++++++++++++++++ .../processreport/correction_ledger_test.go | 34 +++++++++++ internal/framework/processreport/llm.go | 19 +++--- internal/framework/runner/feedback_test.go | 52 ++++++++++++++++ internal/framework/runner/repair.go | 41 ++++++++++++- internal/framework/runner/repair_test.go | 60 ++++++++++++++++++- internal/framework/runner/runner.go | 15 +++-- internal/framework/structuredoutput/call.go | 2 + .../framework/validators/deterministic.go | 12 +++- .../framework/validators/llm_validators.go | 21 ++++--- internal/framework/validators/models.go | 3 + 15 files changed, 308 insertions(+), 38 deletions(-) diff --git a/docs/integrations/output-files.md b/docs/integrations/output-files.md index fb2cb85..d7ddd78 100644 --- a/docs/integrations/output-files.md +++ b/docs/integrations/output-files.md @@ -101,6 +101,9 @@ holds logical calls and PromptKit execution events. Interaction request/response artifacts and input snapshots are also debug-only. Records carry zero-based `stage_index` and optional `stage_id` where an entry is identified. These files are diagnostic evidence, not an alternate ordinary report schema. +For a repaired proposal, one ledger row preserves the original proposed edit, +the final applied segment and span when applicable, and repair attempts with +revision decisions and interaction artifact paths. Each candidate-backed ledger entry includes `proposed_corrected_text`, the literal replacement span, even when it is `""` for a deletion. Applied entries also include `applied_corrected_text`, likewise `""` for an applied deletion. diff --git a/docs/internal/diagnostics-reporting.md b/docs/internal/diagnostics-reporting.md index cd0bccf..ec3c924 100644 --- a/docs/internal/diagnostics-reporting.md +++ b/docs/internal/diagnostics-reporting.md @@ -37,7 +37,9 @@ under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers when debug is off. With debug enabled, it captures source and normalized snapshots, invocation, effective settings, correction ledger, LLM execution observations, and model interactions. Candidate capture includes proposals later -rejected or interrupted. Ledger mapping preserves empty proposed and applied +rejected or interrupted. Repair and revalidation artifacts use section, proposal, +attempt, validator, and batch coordinates as applicable; their retained content +is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied replacements and distinguishes unavailable candidates from interrupted work. A concurrency-safe diagnostic callback reports optional write failures without diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e4b1062..2e9cab5 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -322,6 +322,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery) ## Stage 8 — Retain optional repair evidence and preserve ledger lineage +**Completed.** + 1. Extend the current interaction writers and logical execution observation data with a repair role, original proposal index, and one-based semantic attempt number. Use generated stage instance, section, proposal, and diff --git a/internal/framework/llm/observations.go b/internal/framework/llm/observations.go index 5b90e4e..d0f5fad 100644 --- a/internal/framework/llm/observations.go +++ b/internal/framework/llm/observations.go @@ -53,13 +53,14 @@ type ExecutionObservation struct { } type LogicalCall struct { - CallID uint64 `json:"call_id"` - Context structuredoutput.DiagnosticContext `json:"context"` - Prompt structuredoutput.PromptMetadata `json:"prompt"` - Started time.Time `json:"started"` - Finished time.Time `json:"finished,omitempty"` - WallDuration time.Duration `json:"wall_duration_ns"` - Failure structuredoutput.FailureKind `json:"failure,omitempty"` + RequestedPromptID string `json:"requested_prompt_id,omitempty"` + CallID uint64 `json:"call_id"` + Context structuredoutput.DiagnosticContext `json:"context"` + Prompt structuredoutput.PromptMetadata `json:"prompt"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` + WallDuration time.Duration `json:"wall_duration_ns"` + Failure structuredoutput.FailureKind `json:"failure,omitempty"` } type ObservationSnapshot struct { @@ -92,6 +93,14 @@ func (c *ObservationCollector) Hooks() CallHooks { } func copyContext(value structuredoutput.DiagnosticContext) structuredoutput.DiagnosticContext { + if value.ProposalIndex != nil { + index := *value.ProposalIndex + value.ProposalIndex = &index + } + if value.RepairAttempt != nil { + index := *value.RepairAttempt + value.RepairAttempt = &index + } if value.StageIndex != nil { index := *value.StageIndex value.StageIndex = &index @@ -117,7 +126,7 @@ func (c *ObservationCollector) recordCall(record structuredoutput.CallRecord) { } c.mu.Lock() defer c.mu.Unlock() - call := LogicalCall{CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure} + call := LogicalCall{RequestedPromptID: record.RequestedPromptID, CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure} if !record.Finished.IsZero() { call.WallDuration = record.Finished.Sub(record.Started) } diff --git a/internal/framework/processreport/correction_ledger.go b/internal/framework/processreport/correction_ledger.go index 8ce32de..a24783f 100644 --- a/internal/framework/processreport/correction_ledger.go +++ b/internal/framework/processreport/correction_ledger.go @@ -4,6 +4,7 @@ import ( "path/filepath" "sort" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" @@ -33,6 +34,9 @@ type CorrectionLedgerEntry struct { SegmentID int `json:"segment_id,omitempty"` OriginalText string `json:"original_text,omitempty"` ProposedCorrectedText *string `json:"proposed_corrected_text,omitempty"` + AppliedSegmentID *int `json:"applied_segment_id,omitempty"` + AppliedOriginalText *string `json:"applied_original_text,omitempty"` + RepairAttempts []LedgerRepairAttempt `json:"repair_attempts,omitempty"` AppliedCorrectedText *string `json:"applied_corrected_text,omitempty"` ReplacementPolicy string `json:"replacement_policy,omitempty"` Disposition string `json:"disposition"` @@ -42,7 +46,20 @@ type CorrectionLedgerEntry struct { LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"` } +type LedgerRepairAttempt struct { + Attempt int `json:"attempt"` + Revision *proposals.EnrichedCorrectionProposal `json:"revision,omitempty"` + FailureReason string `json:"failure_reason,omitempty"` + RequestMetadataPath string `json:"request_metadata_path,omitempty"` + RequestPayloadPath string `json:"request_payload_path,omitempty"` + ResponsePayloadPath string `json:"response_payload_path,omitempty"` + ErrorPayloadPath string `json:"error_payload_path,omitempty"` + DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"` + LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"` +} + type LedgerValidatorDecisionRecord struct { + ArtifactPath string `json:"artifact_path,omitempty"` ValidatorKey string `json:"validator_key"` Approved bool `json:"approved"` Disposition validators.DecisionDisposition `json:"disposition"` @@ -71,6 +88,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry policy string decisions map[int][]runner.ValidatorDecisionRecord } + repairs := make(map[proposalIdentity]runner.RepairOutcome) + for _, unit := range runOutput.WorkUnits { + for _, repair := range unit.Repairs { + repairs[proposalIdentity{unit.ModuleInstance, repair.ProposalIndex}] = repair + } + } seen := make(map[proposalIdentity]bool) stages := make(map[string]stageEvidence, len(runOutput.ModuleResults)) for _, module := range runOutput.ModuleResults { @@ -85,6 +108,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry } for _, change := range module.AppliedChanges { + if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok { + continue + } seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true entries = append(entries, CorrectionLedgerEntry{ StageIndex: module.StageIndex, StageID: module.StageID, @@ -103,6 +129,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry }) } for _, change := range module.SkippedChanges { + if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok { + continue + } seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true entries = append(entries, CorrectionLedgerEntry{ StageIndex: module.StageIndex, StageID: module.StageID, @@ -122,6 +151,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry }) } for _, rejection := range module.ValidatorRejected { + if seen[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}] { + continue + } + if _, ok := repairs[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}]; ok { + continue + } disposition := correctionDispositionRejected if rejection.Disposition == validators.DispositionUnavailable { disposition = correctionDispositionUnavailable @@ -156,6 +191,23 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry }) } } + for identity, repair := range repairs { + stage := stages[identity.instance] + original := repair.Original + entry := CorrectionLedgerEntry{StageIndex: stage.index, StageID: stage.id, RunID: runID, ModuleKey: original.ModuleKey, ModuleInstance: identity.instance, ProposalIndex: identity.index, SegmentID: original.TargetSegmentID, OriginalText: original.OriginalText, ProposedCorrectedText: &original.CorrectedText, ReplacementPolicy: stage.policy, Disposition: repair.Outcome, DeterministicValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], false), LLMValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], true)} + for i, attempt := range repair.Attempts { + item := LedgerRepairAttempt{Attempt: i + 1, Revision: attempt.Revision, FailureReason: string(attempt.FailureReason), RequestMetadataPath: attempt.Artifacts.RequestMetadataPath, RequestPayloadPath: attempt.Artifacts.RequestPayloadPath, ResponsePayloadPath: attempt.Artifacts.ResponsePayloadPath, ErrorPayloadPath: attempt.Artifacts.ErrorPayloadPath, DeterministicValidatorResults: filterLedgerDecisions(attempt.Decisions, false), LLMValidatorResults: filterLedgerDecisions(attempt.Decisions, true)} + entry.RepairAttempts = append(entry.RepairAttempts, item) + } + if repair.Outcome == "applied" { + segment, before, after := repair.Final.TargetSegmentID, repair.Final.OriginalText, repair.Final.CorrectedText + entry.AppliedSegmentID = &segment + entry.AppliedOriginalText = &before + entry.AppliedCorrectedText = &after + } + seen[identity] = true + entries = append(entries, entry) + } for _, candidate := range runOutput.Candidates { if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] { continue @@ -197,6 +249,7 @@ func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) [] continue } out = append(out, LedgerValidatorDecisionRecord{ + ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName, Approved: decision.Approved, Disposition: decision.Disposition, diff --git a/internal/framework/processreport/correction_ledger_test.go b/internal/framework/processreport/correction_ledger_test.go index ed98a96..cd867ae 100644 --- a/internal/framework/processreport/correction_ledger_test.go +++ b/internal/framework/processreport/correction_ledger_test.go @@ -208,3 +208,37 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) { } } } + +func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) { + original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 3, ModuleKey: "grammar", ModuleInstance: "grammar_1"}} + revised := original + revised.TargetSegmentID = 2 + revised.OriginalText = "uh " + revised.CorrectedText = "" + output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}} + ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output}) + if len(ledger) != 1 { + t.Fatalf("ledger: %+v", ledger) + } + entry := ledger[0] + if entry.SegmentID != 1 || entry.OriginalText != "cat" || entry.ProposedCorrectedText == nil || *entry.ProposedCorrectedText != "dog" || entry.AppliedSegmentID == nil || *entry.AppliedSegmentID != 2 || entry.AppliedOriginalText == nil || *entry.AppliedOriginalText != "uh " || entry.AppliedCorrectedText == nil || *entry.AppliedCorrectedText != "" || len(entry.RepairAttempts) != 1 || len(entry.DeterministicValidatorResults) != 1 || len(entry.RepairAttempts[0].LLMValidatorResults) != 1 || entry.RepairAttempts[0].LLMValidatorResults[0].ArtifactPath != "validation.json" { + t.Fatalf("lineage: %+v", entry) + } +} + +func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) { + original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}} + revision := original + revision.CorrectedText = "fox" + output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}} + ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output}) + var found *CorrectionLedgerEntry + for i := range ledger { + if ledger[i].Disposition == "interrupted" { + found = &ledger[i] + } + } + if found == nil || len(found.DeterministicValidatorResults) != 1 || len(found.RepairAttempts) != 1 || len(found.RepairAttempts[0].LLMValidatorResults) != 1 || found.AppliedCorrectedText != nil { + t.Fatalf("ledger: %+v", ledger) + } +} diff --git a/internal/framework/processreport/llm.go b/internal/framework/processreport/llm.go index b91c370..7f9c2f6 100644 --- a/internal/framework/processreport/llm.go +++ b/internal/framework/processreport/llm.go @@ -20,14 +20,15 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary { } type executionCall struct { - CallID uint64 `json:"call_id"` - Context structuredoutput.DiagnosticContext `json:"context"` - Prompt structuredoutput.PromptMetadata `json:"prompt"` - Started time.Time `json:"started"` - Finished *time.Time `json:"finished,omitempty"` - ElapsedMS int64 `json:"elapsed_ms"` - Failure structuredoutput.FailureKind `json:"failure,omitempty"` - Summary reporting.LLMSummary `json:"summary"` + RequestedPromptID string `json:"requested_prompt_id,omitempty"` + CallID uint64 `json:"call_id"` + Context structuredoutput.DiagnosticContext `json:"context"` + Prompt structuredoutput.PromptMetadata `json:"prompt"` + Started time.Time `json:"started"` + Finished *time.Time `json:"finished,omitempty"` + ElapsedMS int64 `json:"elapsed_ms"` + Failure structuredoutput.FailureKind `json:"failure,omitempty"` + Summary reporting.LLMSummary `json:"summary"` } type executionEvent struct { @@ -71,7 +72,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any { value := call.Finished finished = &value } - calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))}) + calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))}) } for _, event := range snapshot.Events { var generation, attempt *uint64 diff --git a/internal/framework/runner/feedback_test.go b/internal/framework/runner/feedback_test.go index 0ea7001..d881ba2 100644 --- a/internal/framework/runner/feedback_test.go +++ b/internal/framework/runner/feedback_test.go @@ -3,6 +3,7 @@ package runner import ( "context" "fmt" + "os" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" @@ -85,3 +86,54 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) { t.Fatalf("findings: %+v", findings) } } + +func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}, + }} + validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + attempt := 1 + var paths []string + for _, index := range []int{2, 3} { + candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt} + result, err := validateSectionCandidates(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" { + t.Fatalf("decisions: %+v", result.decisions) + } + paths = append(paths, result.decisions[0].DiagnosticArtifactPath) + } + if paths[0] == paths[1] { + t.Fatalf("colliding paths: %v", paths) + } + for _, path := range paths { + if _, err := os.Stat(path); err != nil { + t.Fatal(err) + } + } +} + +func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}} + validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) + if err != nil { + t.Fatal(err) + } + candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client} + result, err := validateSectionCandidates(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved { + t.Fatalf("result: %+v", result) + } +} diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index 0115edf..fbb2f4a 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "path/filepath" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -15,6 +17,7 @@ import ( // RepairAttempt retains compact facts for a single semantic request. type RepairAttempt struct { + Artifacts llm.InteractionArtifacts Revision *proposals.EnrichedCorrectionProposal Decisions []ValidatorDecisionRecord Rejected []ValidatorRejectedChange @@ -90,8 +93,38 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, return initial, branches, err } var response proposal_generation.RepairResponse - completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex}}, &response) + proposalIndex, attemptNumber := original.ProposalIndex, attempt+1 + completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response) record := RepairAttempt{Counts: completion.Execution} + save := func(err error) { + if input.DiagnosticsDir == "" { + return + } + var secrets []string + if source, ok := input.Spec.RepairClient.(interface { + ConfiguredSecrets(context.Context) []string + }); ok { + secrets = source.ConfiguredSecrets(ctx) + } + section := 0 + if input.SectionIndex != nil { + section = *input.SectionIndex + } + name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber) + var failure any + if err != nil { + failure = map[string]string{"error": err.Error()} + } + writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), secrets) + artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": record.Rejected}, failure) + if writeErr != nil { + if input.ReportDiagnosticError != nil { + input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr)) + } + return + } + record.Artifacts = artifacts + } if callErr != nil { var failure *structuredoutput.Failure if errors.As(callErr, &failure) { @@ -99,6 +132,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } else { record.FailureReason = outcomes.ReasonInternalFailure } + save(callErr) branch.Attempts = append(branch.Attempts, record) if structuredoutput.IsRecoverable(callErr) { branch.Outcome = "unavailable" @@ -110,6 +144,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1) if err != nil { record.FailureReason = outcomes.ReasonInvalidOutput + save(err) branch.Attempts = append(branch.Attempts, record) branch.Outcome = "unavailable" break @@ -118,6 +153,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, replay.Turns = append(replay.Turns, completion.RepairInstruction...) replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent}) if decoded.Action == "withdraw" { + save(nil) branch.Attempts = append(branch.Attempts, record) branch.Outcome = "withdrawn" break @@ -132,6 +168,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence} branch.Final = current next := input + next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current} validated, validationErr := validateSectionCandidates(ctx, next) revisionSnapshot := current @@ -144,9 +181,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, validated.operations[i].Reference.RepairAttempt = &attemptNumber } record.Operations = validated.operations + save(validationErr) branch.Attempts = append(branch.Attempts, record) initial.operations = append(initial.operations, validated.operations...) - initial.decisions = append(initial.decisions, validated.decisions...) if validationErr != nil { branches = append(branches, branch) return initial, branches, validationErr diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index 72fbfea..95b7aab 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "errors" + "os" + "path/filepath" + "strings" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" @@ -20,6 +23,7 @@ type repairClient struct { outputs []string calls []structuredoutput.Request err error + secrets []string } func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) { @@ -36,9 +40,11 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu if err := json.Unmarshal([]byte(value), out); err != nil { return structuredoutput.Response{}, err } - return structuredoutput.Response{Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil + return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil } +func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets } + func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) { client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}} candidates := []proposals.EnrichedCorrectionProposal{ @@ -182,3 +188,55 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) { t.Fatalf("branches=%+v error=%v", branches, err) } } + +func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) { + client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}} + candidates := []proposals.EnrichedCorrectionProposal{ + {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}, + {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}}, + } + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()} + initial, err := validateSectionCandidates(context.Background(), input) + if err != nil { + t.Fatal(err) + } + _, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair") + if err != nil { + t.Fatal(err) + } + if len(branches) != 2 { + t.Fatalf("branches: %+v", branches) + } + a, b := branches[0].Attempts[0].Artifacts.RequestPayloadPath, branches[1].Attempts[0].Artifacts.RequestPayloadPath + if a == "" || a == b { + t.Fatalf("artifact paths: %q %q", a, b) + } + for _, path := range []string{a, b} { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") { + t.Fatalf("unredacted %s: %s", filepath.Base(path), data) + } + } +} + +func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) { + client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}} + path := filepath.Join(t.TempDir(), "file") + if err := os.WriteFile(path, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + warnings := 0 + candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }} + initial, err := validateSectionCandidates(context.Background(), input) + if err != nil { + t.Fatal(err) + } + _, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair") + if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 { + t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err) + } +} diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index b9b5e1e..c5dcd6f 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -646,7 +646,9 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } type validateSectionCandidatesInput struct { - Exchange *structuredoutput.Exchange + RepairProposalIndex *int + RepairAttempt *int + Exchange *structuredoutput.Exchange // ReportDiagnosticError may be called concurrently; it must not affect processing. ReportDiagnosticError func(error) `json:"-"` Spec contracts.ModuleRunSpec @@ -703,11 +705,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida } vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ StageIndex: &input.Spec.Index, StageID: input.Spec.ID, - WorkingTranscript: input.WorkingTranscript, - CandidateProposal: submitted, - ModuleKey: input.Spec.ModuleKey, - ModuleInstance: input.Spec.InstanceName, - SectionIndex: input.SectionIndex, + WorkingTranscript: input.WorkingTranscript, + CandidateProposal: submitted, + ModuleKey: input.Spec.ModuleKey, + ModuleInstance: input.Spec.InstanceName, + SectionIndex: input.SectionIndex, + IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt, ValidatorOrder: &validatorOrder, ReplacementPolicy: input.Policy, Glossary: input.Spec.Glossary, diff --git a/internal/framework/structuredoutput/call.go b/internal/framework/structuredoutput/call.go index bc41432..27fa40c 100644 --- a/internal/framework/structuredoutput/call.go +++ b/internal/framework/structuredoutput/call.go @@ -19,6 +19,8 @@ type DiagnosticContext struct { SectionIndex *int `json:"section_index,omitempty"` BatchIndex *int `json:"batch_index,omitempty"` ValidatorName string `json:"validator_name,omitempty"` + ProposalIndex *int `json:"proposal_index,omitempty"` + RepairAttempt *int `json:"repair_attempt,omitempty"` } // Request names an embedded prompt and response schema. Profile selection and diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index 7ba69b8..94b3bf9 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -3,6 +3,7 @@ package validators import ( "context" "fmt" + "math" "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" @@ -104,7 +105,7 @@ func (v NonEmptyResultingSegmentValidator) Validate(_ context.Context, req Reque decisions = append(decisions, approval(c.ProposalIndex)) continue } - preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy) + preview := previewForEvidence(req, &segment, c.CorrectionProposal) if preview.SkipReason == proposals.SkipReasonEmptyResultingText { decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty")) continue @@ -154,7 +155,7 @@ func ValidateProtectedTerms(req Request) (Result, error) { if segment.ID != c.TargetSegmentID { continue } - preview := proposals.PreviewProposalForSegment(segment, c.CorrectionProposal, req.ReplacementPolicy) + preview := previewForEvidence(req, segment, c.CorrectionProposal) if preview.Applicable { if glossaryStage { // A correction span may sit inside a longer exempt name. @@ -177,3 +178,10 @@ func ValidateProtectedTerms(req Request) (Result, error) { } return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil } + +func previewForEvidence(req Request, segment *schema.Segment, correction proposals.CorrectionProposal) proposals.SegmentPreviewResult { + if req.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) { + correction.Confidence = 0.5 + } + return proposals.PreviewProposalForSegment(segment, correction, req.ReplacementPolicy) +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index baba17e..bb039ca 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -11,7 +11,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" @@ -128,7 +127,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{ PromptID: promptMetadata.PromptID, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, - Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name}, + Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt}, }, &response) batchIndex := batch.BatchIndex operation := outcomes.Operation{ @@ -144,6 +143,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, if req.SectionIndex != nil { stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex) } + if req.RepairProposalIndex != nil && req.RepairAttempt != nil { + stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt) + } stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex) responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model} @@ -155,12 +157,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, stage, map[string]any{ "stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey, - "validator_name": v.name, - "validator_type": v.validatorType, - "batch_index": batch.BatchIndex, - "proposal_indices": batchProposalIndices(batch.Items), - "prompt": completion.Prompt, - "response_schema": responseSchema.DiagnosticsMap(), + "validator_name": v.name, + "validator_type": v.validatorType, + "batch_index": batch.BatchIndex, + "proposal_indices": batchProposalIndices(batch.Items), + "repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt, + "prompt": completion.Prompt, + "response_schema": responseSchema.DiagnosticsMap(), }, completion.Prepared, responsePayload, @@ -263,7 +266,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT } seg := segments[position] - preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy) + preview := previewForEvidence(req, &seg, p.CorrectionProposal) if !preview.Applicable { immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation")) continue diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 88042da..2022208 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -36,6 +36,9 @@ type Request struct { StageID *string `json:"stage_id,omitempty"` // ReportDiagnosticError may be called concurrently; it must not affect processing. ReportDiagnosticError func(error) `json:"-"` + IndependentEvidence bool `json:"-"` + RepairProposalIndex *int `json:"repair_proposal_index,omitempty"` + RepairAttempt *int `json:"repair_attempt,omitempty"` SectionIndex *int `json:"section_index,omitempty"` WorkingTranscript *schema.Transcript `json:"-"` CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`