eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Report correction repair operations and outcomes

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:31:44 +00:00
parent d927c61199
commit 735dfc1834
10 changed files with 159 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
internal/framework/processreport/report.go
View File

@@ -123,7 +123,23 @@ func Build(input BuildInput) reporting.Snapshot {
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
}
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
for _, repair := range unit.Repairs {
if len(repair.Attempts) == 0 {
continue
}
snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: len(repair.Attempts), Outcome: repair.Outcome})
}
}
sort.SliceStable(snapshot.RepairOutcomes, func(i, j int) bool {
a, b := snapshot.RepairOutcomes[i], snapshot.RepairOutcomes[j]
if a.StageIndex != b.StageIndex {
return a.StageIndex < b.StageIndex
}
if a.ChunkIndex != b.ChunkIndex {
return a.ChunkIndex < b.ChunkIndex
}
return a.ProposalIndex < b.ProposalIndex
})
runErrors := len(input.RunFailures)
snapshot.Summary = reporting.Summary{
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
@@ -247,15 +263,31 @@ func operationLess(a, b outcomes.Operation) bool {
return 2
}
}
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
order := func(p *int) int {
if p == nil {
return -1
}
return *p
}
aRepair, bRepair := a.Reference.RepairAttempt != nil, b.Reference.RepairAttempt != nil
if aRepair != bRepair {
return !aRepair
}
if aRepair {
if order(a.Reference.ProposalIndex) != order(b.Reference.ProposalIndex) {
return order(a.Reference.ProposalIndex) < order(b.Reference.ProposalIndex)
}
if order(a.Reference.RepairAttempt) != order(b.Reference.RepairAttempt) {
return order(a.Reference.RepairAttempt) < order(b.Reference.RepairAttempt)
}
if (a.Reference.Kind == outcomes.OperationCorrectionRepair) != (b.Reference.Kind == outcomes.OperationCorrectionRepair) {
return a.Reference.Kind == outcomes.OperationCorrectionRepair
}
}
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) {
return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder)
}

52
internal/framework/processreport/report_test.go
View File

@@ -2,6 +2,7 @@ package processreport
import (
"reflect"
"sort"
"strings"
"testing"
"time"
@@ -137,3 +138,54 @@ func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
}
func intPtr(value int) *int { return &value }
func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
}
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}})
}
snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || snapshot.Errors[0].Failures[0].Operation != "correction_repair" || len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"correction_revised", "output_repaired"}) {
t.Fatalf("reports: %+v %+v", snapshot.Errors, snapshot.Recoveries)
}
if len(snapshot.RepairOutcomes) != 7 || snapshot.RepairOutcomes[0].ProposalIndex != 0 || snapshot.RepairOutcomes[6].ProposalIndex != 6 {
t.Fatalf("outcomes: %+v", snapshot.RepairOutcomes)
}
text := string(reporting.Markdown(snapshot))
for _, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
if !strings.Contains(text, "| "+disposition+" |") {
t.Errorf("missing %s in %s", disposition, text)
}
}
summary := Build(BuildInput{SummaryOnly: true, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if !reflect.DeepEqual(summary.Summary, snapshot.Summary) || len(summary.RepairOutcomes) != 0 {
t.Fatalf("summary-only: %+v", summary)
}
}
func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &one, RepairAttempt: &two, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &two}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}},
}
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if operations[0].Reference.Kind != outcomes.OperationProposal || operations[1].Reference.Kind != outcomes.OperationValidation || *operations[2].Reference.ProposalIndex != 1 || *operations[2].Reference.RepairAttempt != 1 || *operations[3].Reference.RepairAttempt != 2 || operations[3].Reference.Kind != outcomes.OperationCorrectionRepair || operations[4].Reference.Kind != outcomes.OperationValidation || *operations[5].Reference.ProposalIndex != 2 {
t.Fatalf("order: %+v", operations)
}
}