eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Keep detailed correction repair history in debug runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 12:31:08 +00:00
parent 6b724fd3cc
commit 04e0c2f913
6 changed files with 66 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/roadmap/remediation.md
View File

@@ -2,7 +2,7 @@
**Updated:** 2026-11-11
**Status:** 6 of nine findings resolved. Findings refer to implementation revision
**Status:** 7 of nine findings resolved. Findings refer to implementation revision
`aa8073c`. See the [implementation plan](implementation.md).
## Review Result
@@ -240,6 +240,8 @@ skipped by the existing conflict checks.
### R7 — Keep Detailed Repair History in Debug Runs (P3)
**Completed.** Ordinary runs retain a compact attempt count and disposition. Debug runs retain the revision history. Runner, reporting, and CLI tests pass with debug on and off.
**Affected plan:** Stage 7, item 3, and Stage 8, items 2–3.
**Evidence:** [repairCandidates](../../internal/framework/runner/repair.go#L178)

4
internal/framework/processreport/report.go
View File

@@ -124,10 +124,10 @@ func Build(input BuildInput) reporting.Snapshot {
}
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
for _, repair := range unit.Repairs {
if len(repair.Attempts) == 0 {
if repair.AttemptCount == 0 {
continue
}
snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: len(repair.Attempts), Outcome: repair.Outcome})
snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: repair.AttemptCount, Outcome: repair.Outcome})
}
}
sort.SliceStable(snapshot.RepairOutcomes, func(i, j int) bool {

2
internal/framework/processreport/report_test.go
View File

@@ -150,7 +150,7 @@ func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
}
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}})
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, AttemptCount: 1})
}
snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 {

132
internal/framework/runner/repair.go
View File

@@ -18,25 +18,20 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
// RepairAttempt retains compact facts for a single semantic request.
// RepairAttempt holds optional debug evidence for one semantic request.
type RepairAttempt struct {
Artifacts llm.InteractionArtifacts
Revision *proposals.EnrichedCorrectionProposal
Decisions []ValidatorDecisionRecord
Rejected []ValidatorRejectedChange
Counts outcomes.ExecutionCounts
FailureReason outcomes.ReasonCode
Succeeded bool
Approved bool
Operations []outcomes.Operation
Filtered []outcomes.FilteredProposal
}
// RepairOutcome keeps the proposal identity through revision and application.
// RepairOutcome summarizes the branch and carries its lineage when debug is enabled.
type RepairOutcome struct {
ProposalIndex int
Attempts []RepairAttempt
AttemptCount int
Outcome string
Attempts []RepairAttempt
Original proposals.EnrichedCorrectionProposal
Final proposals.EnrichedCorrectionProposal
}
@@ -68,12 +63,19 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
superseded := make(map[int]bool)
var branches []RepairOutcome
var repairOps []outcomes.Operation
debug := input.DiagnosticsDir != ""
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
}
finish := func(err error) (validateSectionCandidatesResult, []RepairOutcome, error) {
if len(superseded) > 0 {
initial.filtered = slices.DeleteFunc(initial.filtered, func(item outcomes.FilteredProposal) bool {
return superseded[item.ProposalIndex]
})
}
initial.operations = append(initial.operations, repairOps...)
return initial, branches, err
}
@@ -82,7 +84,10 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if !actionable(original.ProposalIndex, rejected) {
continue
}
branch := RepairOutcome{ProposalIndex: original.ProposalIndex, Original: original, Final: original, Outcome: "interrupted"}
branch := RepairOutcome{ProposalIndex: original.ProposalIndex, Outcome: "interrupted"}
if debug {
branch.Original, branch.Final = original, original
}
if err := ctx.Err(); err != nil {
branches = append(branches, branch)
return finish(err)
@@ -122,40 +127,47 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
var response proposal_generation.RepairResponse
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
record := RepairAttempt{Counts: completion.Execution}
save := func(err error) {
if input.DiagnosticsDir == "" {
operation := outcomes.Operation{
Reference: outcomes.Reference{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair,
ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber,
},
Counts: completion.Execution, ProposalIndices: []int{proposalIndex},
}
finishAttempt := func(err error, revision *proposals.EnrichedCorrectionProposal, validated validateSectionCandidatesResult) {
branch.AttemptCount++
repairOps = append(repairOps, operation)
if !debug {
return
}
secrets := repairSecrets(ctx, input)
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
record := RepairAttempt{Decisions: validated.decisions, FailureReason: operation.FailureReason}
if revision != nil {
snapshot := *revision
record.Revision = &snapshot
}
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
var failure any
if err != nil {
failure = map[string]string{"error": err.Error()}
}
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), secrets)
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": record.Rejected}, failure)
if writeErr != nil {
if input.ReportDiagnosticError != nil {
input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
}
return
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input))
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
if writeErr == nil {
record.Artifacts = artifacts
} else if input.ReportDiagnosticError != nil {
input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
}
record.Artifacts = artifacts
branch.Attempts = append(branch.Attempts, record)
}
if callErr != nil {
var failure *structuredoutput.Failure
if errors.As(callErr, &failure) {
record.FailureReason = failure.Reason
operation.FailureReason = failure.Reason
} else {
record.FailureReason = outcomes.ReasonInternalFailure
operation.FailureReason = outcomes.ReasonInternalFailure
}
save(callErr)
branch.Attempts = append(branch.Attempts, record)
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
if structuredoutput.IsRecoverable(callErr) {
branch.Outcome = "unavailable"
break
@@ -165,18 +177,17 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
if err != nil {
record.FailureReason = outcomes.ReasonInvalidOutput
save(err)
branch.Attempts = append(branch.Attempts, record)
operation.FailureReason = outcomes.ReasonInvalidOutput
finishAttempt(err, nil, validateSectionCandidatesResult{})
branch.Outcome = "unavailable"
break
}
record.Succeeded = true
operation.Succeeded = true
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
replay.Turns = append(replay.Turns, completion.RepairInstruction...)
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
if decoded.Action == "withdraw" {
save(nil)
branch.Attempts = append(branch.Attempts, record)
finishAttempt(nil, nil, validateSectionCandidatesResult{})
branch.Outcome = "withdrawn"
break
}
@@ -188,35 +199,34 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
current = original
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
branch.Final = current
if debug {
branch.Final = current
}
next := input
next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber
next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
validated, validationErr := validateSectionCandidates(ctx, next)
revisionSnapshot := current
record.Revision = &revisionSnapshot
record.Decisions = validated.decisions
record.Rejected = validated.rejected
if len(validated.filtered) > 0 {
// A repair response contains one proposal. Attribute its first
// defect once, as for proposals in the original response.
record.Filtered = validated.filtered[:1]
// Each repair response contains only one proposal.
operation.Filtered = validated.filtered[:1]
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
}
for i := range validated.operations {
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
validated.operations[i].Reference.ProposalIndex = &proposalIndex
validated.operations[i].Reference.RepairAttempt = &attemptNumber
}
record.Operations = validated.operations
save(validationErr)
branch.Attempts = append(branch.Attempts, record)
approved := validationErr == nil && len(validated.approved) == 1
if approved {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
}
finishAttempt(validationErr, &current, validated)
initial.operations = append(initial.operations, validated.operations...)
if validationErr != nil {
branches = append(branches, branch)
return finish(validationErr)
}
if len(validated.approved) == 1 {
branch.Attempts[len(branch.Attempts)-1].Approved = true
if approved {
branch.Outcome = "approved"
initial.approved = append(initial.approved, current)
superseded[original.ProposalIndex] = true
@@ -244,32 +254,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return finish(nil)
}
func repairOperations(input validateSectionCandidatesInput, branches []RepairOutcome) []outcomes.Operation {
var result []outcomes.Operation
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
}
for _, branch := range branches {
for i, attempt := range branch.Attempts {
index, number := branch.ProposalIndex, i+1
operation := outcomes.Operation{Reference: outcomes.Reference{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &index, RepairAttempt: &number}, Counts: attempt.Counts, Succeeded: attempt.Succeeded, FailureReason: attempt.FailureReason, ProposalIndices: []int{index}}
if attempt.Succeeded {
operation.RecoveryReasons = outcomes.ExecutionRecoveries(attempt.Counts)
if len(attempt.Filtered) > 0 {
operation.Filtered = attempt.Filtered
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
}
if attempt.Approved {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
}
}
result = append(result, operation)
}
}
return result
}
func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string {
var secrets []string
for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} {

4
internal/framework/runner/repair_test.go
View File

@@ -99,7 +99,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls {
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 {
t.Fatalf("branches=%+v calls=%d", branches, len(client.calls))
}
if tc.calls == 2 && len(client.calls[1].Replay.Turns) != 2 {
@@ -192,7 +192,7 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if !errors.Is(err, context.Canceled) || len(branches) != 1 || branches[0].Outcome != "interrupted" || len(branches[0].Attempts) != 1 {
if !errors.Is(err, context.Canceled) || len(branches) != 1 || branches[0].Outcome != "interrupted" || branches[0].AttemptCount != 1 || len(branches[0].Attempts) != 0 {
t.Fatalf("branches=%+v error=%v", branches, err)
}
}

1
internal/framework/runner/runner.go
View File

@@ -571,7 +571,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
promptID = provider.RepairPromptID()
}
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(validationInput, repairs)...)
}
setErr(err)
validationResults <- sectionValidationResult{