Report correction repair operations and outcomes

This commit is contained in:
2026-11-11 04:31:44 +00:00
parent d927c61199
commit 735dfc1834
10 changed files with 159 additions and 10 deletions

View File

@@ -48,7 +48,7 @@ remain in opt-in debug files.
An `errors` record with `scope: work_unit` identifies zero-based `stage_index` An `errors` record with `scope: work_unit` identifies zero-based `stage_index`
and `chunk_index`, optional `stage_id`, module key/instance, initial segment range, applied-correction and `chunk_index`, optional `stage_id`, module key/instance, initial segment range, applied-correction
count, and a `failures` sequence. Each failure identifies the operation count, and a `failures` sequence. Each failure identifies the operation
(`proposal`, `validation`, or `validation_input`), optional validator and (`proposal`, `validation`, `validation_input`, or `correction_repair`), optional validator and
zero-based batch index, safe reason/message, sorted proposal indices when zero-based batch index, safe reason/message, sorted proposal indices when
known, and execution counts when available. Multiple failed batches in one known, and execution counts when available. Multiple failed batches in one
work unit form one error record. `scope: run` instead records phase, safe reason work unit form one error record. `scope: run` instead records phase, safe reason
@@ -62,7 +62,9 @@ may still be approved or rejected normally.
Each recovery identifies its work unit, optional `stage_id`, and successful operation, with sorted Each recovery identifies its work unit, optional `stage_id`, and successful operation, with sorted
`reasons`, a safe message, available execution counts, and sorted `reasons`, a safe message, available execution counts, and sorted
`filtered_proposals` details. One recovery can contain `output_repaired`, `filtered_proposals` details. One recovery can contain `output_repaired`,
`service_recovered`, and `proposals_filtered` together. Filtering covers `service_recovered`, `correction_revised`, and `proposals_filtered` together.
`correction_revised` means a repair yielded a fully validated revision; its
application outcome is recorded in the Markdown report. Filtering covers
defective response proposals; low confidence, protected-term rejection, defective response proposals; low confidence, protected-term rejection,
ordinary editorial rejection, and an apply-time conflict caused by an earlier ordinary editorial rejection, and an apply-time conflict caused by an earlier
accepted correction are not filtering recoveries. Failed operations are errors, accepted correction are not filtering recoveries. Failed operations are errors,
@@ -84,8 +86,12 @@ the one-based position and module, plus the operator ID when supplied. Completed
counts. `—` means not started and `n*` means interrupted with n facts recorded. counts. `—` means not started and `n*` means interrupted with n facts recorded.
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
records containing each reason; a recovery can have several reasons, so those records containing each reason; a recovery can have several reasons, so those
counts need not sum to the recovery total. The report refers to both YAML counts need not sum to the recovery total. When a repair was attempted, a
files. It does not include transcript excerpts or correction lists. Correction repair table lists its stage, chunk,
original proposal index, semantic attempt count, and final outcome (`applied`,
`approved_skipped`, `withdrawn`, `rejected`, `exhausted`, `unavailable`, or
`interrupted`). The report refers to both YAML files. It does not include
transcript excerpts or correction lists.
## Debug-Only Files ## Debug-Only Files

View File

@@ -276,6 +276,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
## Stage 7 — Account for operations and publish compact ordinary outcomes ## Stage 7 — Account for operations and publish compact ordinary outcomes
**Completed.**
1. Add `correction_repair` as an operation kind in 1. Add `correction_repair` as an operation kind in
`internal/framework/outcomes`. For each repair call retain its proposal `internal/framework/outcomes`. For each repair call retain its proposal
identity, attempt number, operation counts, and safe terminal reason. identity, attempt number, operation counts, and safe terminal reason.

View File

@@ -53,6 +53,16 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
fmt.Fprintln(&b) fmt.Fprintln(&b)
writeMatrix(&b, snapshot, r, true) writeMatrix(&b, snapshot, r, true)
fmt.Fprintln(&b) fmt.Fprintln(&b)
if len(snapshot.RepairOutcomes) > 0 {
fmt.Fprintln(&b, "## Correction repair")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |")
fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |")
for _, row := range snapshot.RepairOutcomes {
fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, safe(row.Outcome))
}
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.") fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.")
fmt.Fprintln(&b) fmt.Fprintln(&b)
writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe) writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe)

View File

@@ -129,6 +129,15 @@ type WorkUnit struct {
RecoveryCount int RecoveryCount int
} }
// RepairOutcome is a compact correction repair row.
type RepairOutcome struct {
StageIndex int
ChunkIndex int
ProposalIndex int
Attempts int
Outcome string
}
// Snapshot is built once from processing state and shared by presentations. // Snapshot is built once from processing state and shared by presentations.
type Snapshot struct { type Snapshot struct {
RunID string RunID string
@@ -139,6 +148,7 @@ type Snapshot struct {
OutputPath string OutputPath string
StageNames []string StageNames []string
WorkUnits []WorkUnit WorkUnits []WorkUnit
RepairOutcomes []RepairOutcome
Summary Summary Summary Summary
Errors []ErrorRecord Errors []ErrorRecord
Recoveries []RecoveryRecord Recoveries []RecoveryRecord

View File

@@ -7,9 +7,10 @@ package outcomes
type OperationKind string type OperationKind string
const ( const (
OperationProposal OperationKind = "proposal" OperationProposal OperationKind = "proposal"
OperationValidation OperationKind = "validation" OperationCorrectionRepair OperationKind = "correction_repair"
OperationValidationInput OperationKind = "validation_input" OperationValidation OperationKind = "validation"
OperationValidationInput OperationKind = "validation_input"
) )
// Reference is the stable processing identity of an operation. Proposal indices // Reference is the stable processing identity of an operation. Proposal indices
@@ -22,6 +23,8 @@ type Reference struct {
ModuleInstance string `json:"module_instance"` ModuleInstance string `json:"module_instance"`
SectionIndex int `json:"section_index"` SectionIndex int `json:"section_index"`
Kind OperationKind `json:"operation"` Kind OperationKind `json:"operation"`
ProposalIndex *int `json:"proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
ValidatorName string `json:"validator_name,omitempty"` ValidatorName string `json:"validator_name,omitempty"`
ValidatorOrder *int `json:"validator_order,omitempty"` ValidatorOrder *int `json:"validator_order,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"` BatchIndex *int `json:"batch_index,omitempty"`
@@ -61,6 +64,7 @@ const (
type RecoveryReason string type RecoveryReason string
const ( const (
RecoveryCorrectionRevised RecoveryReason = "correction_revised"
RecoveryOutputRepaired RecoveryReason = "output_repaired" RecoveryOutputRepaired RecoveryReason = "output_repaired"
RecoveryServiceRecovered RecoveryReason = "service_recovered" RecoveryServiceRecovered RecoveryReason = "service_recovered"
RecoveryProposalsFiltered RecoveryReason = "proposals_filtered" RecoveryProposalsFiltered RecoveryReason = "proposals_filtered"

View File

@@ -123,7 +123,23 @@ func Build(input BuildInput) reporting.Snapshot {
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures}) StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
} }
snapshot.WorkUnits = append(snapshot.WorkUnits, row) snapshot.WorkUnits = append(snapshot.WorkUnits, row)
for _, repair := range unit.Repairs {
if len(repair.Attempts) == 0 {
continue
}
snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: len(repair.Attempts), Outcome: repair.Outcome})
}
} }
sort.SliceStable(snapshot.RepairOutcomes, func(i, j int) bool {
a, b := snapshot.RepairOutcomes[i], snapshot.RepairOutcomes[j]
if a.StageIndex != b.StageIndex {
return a.StageIndex < b.StageIndex
}
if a.ChunkIndex != b.ChunkIndex {
return a.ChunkIndex < b.ChunkIndex
}
return a.ProposalIndex < b.ProposalIndex
})
runErrors := len(input.RunFailures) runErrors := len(input.RunFailures)
snapshot.Summary = reporting.Summary{ snapshot.Summary = reporting.Summary{
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed, WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
@@ -247,15 +263,31 @@ func operationLess(a, b outcomes.Operation) bool {
return 2 return 2
} }
} }
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
order := func(p *int) int { order := func(p *int) int {
if p == nil { if p == nil {
return -1 return -1
} }
return *p return *p
} }
aRepair, bRepair := a.Reference.RepairAttempt != nil, b.Reference.RepairAttempt != nil
if aRepair != bRepair {
return !aRepair
}
if aRepair {
if order(a.Reference.ProposalIndex) != order(b.Reference.ProposalIndex) {
return order(a.Reference.ProposalIndex) < order(b.Reference.ProposalIndex)
}
if order(a.Reference.RepairAttempt) != order(b.Reference.RepairAttempt) {
return order(a.Reference.RepairAttempt) < order(b.Reference.RepairAttempt)
}
if (a.Reference.Kind == outcomes.OperationCorrectionRepair) != (b.Reference.Kind == outcomes.OperationCorrectionRepair) {
return a.Reference.Kind == outcomes.OperationCorrectionRepair
}
}
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) { if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) {
return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder) return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder)
} }

View File

@@ -2,6 +2,7 @@ package processreport
import ( import (
"reflect" "reflect"
"sort"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -137,3 +138,54 @@ func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
} }
func intPtr(value int) *int { return &value } func intPtr(value int) *int { return &value }
func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
}
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}})
}
snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || snapshot.Errors[0].Failures[0].Operation != "correction_repair" || len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"correction_revised", "output_repaired"}) {
t.Fatalf("reports: %+v %+v", snapshot.Errors, snapshot.Recoveries)
}
if len(snapshot.RepairOutcomes) != 7 || snapshot.RepairOutcomes[0].ProposalIndex != 0 || snapshot.RepairOutcomes[6].ProposalIndex != 6 {
t.Fatalf("outcomes: %+v", snapshot.RepairOutcomes)
}
text := string(reporting.Markdown(snapshot))
for _, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
if !strings.Contains(text, "| "+disposition+" |") {
t.Errorf("missing %s in %s", disposition, text)
}
}
summary := Build(BuildInput{SummaryOnly: true, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if !reflect.DeepEqual(summary.Summary, snapshot.Summary) || len(summary.RepairOutcomes) != 0 {
t.Fatalf("summary-only: %+v", summary)
}
}
func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &one, RepairAttempt: &two, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &two}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}},
}
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if operations[0].Reference.Kind != outcomes.OperationProposal || operations[1].Reference.Kind != outcomes.OperationValidation || *operations[2].Reference.ProposalIndex != 1 || *operations[2].Reference.RepairAttempt != 1 || *operations[3].Reference.RepairAttempt != 2 || operations[3].Reference.Kind != outcomes.OperationCorrectionRepair || operations[4].Reference.Kind != outcomes.OperationValidation || *operations[5].Reference.ProposalIndex != 2 {
t.Fatalf("order: %+v", operations)
}
}

View File

@@ -21,6 +21,7 @@ type RepairAttempt struct {
Counts outcomes.ExecutionCounts Counts outcomes.ExecutionCounts
FailureReason outcomes.ReasonCode FailureReason outcomes.ReasonCode
Succeeded bool Succeeded bool
Approved bool
Operations []outcomes.Operation Operations []outcomes.Operation
} }
@@ -137,6 +138,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
record.Revision = &revisionSnapshot record.Revision = &revisionSnapshot
record.Decisions = validated.decisions record.Decisions = validated.decisions
record.Rejected = validated.rejected record.Rejected = validated.rejected
for i := range validated.operations {
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
validated.operations[i].Reference.ProposalIndex = &proposalIndex
validated.operations[i].Reference.RepairAttempt = &attemptNumber
}
record.Operations = validated.operations record.Operations = validated.operations
branch.Attempts = append(branch.Attempts, record) branch.Attempts = append(branch.Attempts, record)
initial.operations = append(initial.operations, validated.operations...) initial.operations = append(initial.operations, validated.operations...)
@@ -146,6 +152,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return initial, branches, validationErr return initial, branches, validationErr
} }
if len(validated.approved) == 1 { if len(validated.approved) == 1 {
branch.Attempts[len(branch.Attempts)-1].Approved = true
branch.Outcome = "approved" branch.Outcome = "approved"
initial.approved = append(initial.approved, current) initial.approved = append(initial.approved, current)
retained := initial.filtered[:0] retained := initial.filtered[:0]
@@ -177,3 +184,25 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
} }
return initial, branches, nil return initial, branches, nil
} }
func repairOperations(input validateSectionCandidatesInput, branches []RepairOutcome) []outcomes.Operation {
var result []outcomes.Operation
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
}
for _, branch := range branches {
for i, attempt := range branch.Attempts {
index, number := branch.ProposalIndex, i+1
operation := outcomes.Operation{Reference: outcomes.Reference{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &index, RepairAttempt: &number}, Counts: attempt.Counts, Succeeded: attempt.Succeeded, FailureReason: attempt.FailureReason, ProposalIndices: []int{index}}
if attempt.Succeeded {
operation.RecoveryReasons = outcomes.ExecutionRecoveries(attempt.Counts)
if attempt.Approved {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
}
}
result = append(result, operation)
}
}
return result
}

View File

@@ -121,6 +121,9 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
if out.FinalTranscript.Segments[0].Text != "dog" || len(out.WorkUnits) != 1 || len(out.WorkUnits[0].Repairs) != 1 || out.WorkUnits[0].Repairs[0].Outcome != "approved_skipped" || out.WorkUnits[0].CorrectionsApplied != 1 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].ProposalIndex != 1 { if out.FinalTranscript.Segments[0].Text != "dog" || len(out.WorkUnits) != 1 || len(out.WorkUnits[0].Repairs) != 1 || out.WorkUnits[0].Repairs[0].Outcome != "approved_skipped" || out.WorkUnits[0].CorrectionsApplied != 1 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].ProposalIndex != 1 {
t.Fatalf("output=%+v section=%d", out, sectionIndex) t.Fatalf("output=%+v section=%d", out, sectionIndex)
} }
if len(out.WorkUnits[0].Operations) != 1 || out.WorkUnits[0].Operations[0].Reference.Kind != outcomes.OperationCorrectionRepair || len(out.WorkUnits[0].Operations[0].RecoveryReasons) != 1 || out.WorkUnits[0].Operations[0].RecoveryReasons[0] != outcomes.RecoveryCorrectionRevised {
t.Fatalf("repair operations: %+v", out.WorkUnits[0].Operations)
}
} }
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) { func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) {

View File

@@ -564,6 +564,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
} }
repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: &sectionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError} repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: &sectionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError}
validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID) validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(repairInput, repairs)...)
} }
setErr(err) setErr(err)
validationResults <- sectionValidationResult{ validationResults <- sectionValidationResult{