eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Report correction repair operations and outcomes

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:31:44 +00:00
parent d927c61199
commit 735dfc1834
10 changed files with 159 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
internal/framework/processreport/report_test.go
View File

@@ -2,6 +2,7 @@ package processreport
import (
"reflect"
"sort"
"strings"
"testing"
"time"
@@ -137,3 +138,54 @@ func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
}
func intPtr(value int) *int { return &value }
func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
}
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}})
}
snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || snapshot.Errors[0].Failures[0].Operation != "correction_repair" || len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"correction_revised", "output_repaired"}) {
t.Fatalf("reports: %+v %+v", snapshot.Errors, snapshot.Recoveries)
}
if len(snapshot.RepairOutcomes) != 7 || snapshot.RepairOutcomes[0].ProposalIndex != 0 || snapshot.RepairOutcomes[6].ProposalIndex != 6 {
t.Fatalf("outcomes: %+v", snapshot.RepairOutcomes)
}
text := string(reporting.Markdown(snapshot))
for _, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
if !strings.Contains(text, "| "+disposition+" |") {
t.Errorf("missing %s in %s", disposition, text)
}
}
summary := Build(BuildInput{SummaryOnly: true, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if !reflect.DeepEqual(summary.Summary, snapshot.Summary) || len(summary.RepairOutcomes) != 0 {
t.Fatalf("summary-only: %+v", summary)
}
}
func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &one, RepairAttempt: &two, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &two}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}},
}
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if operations[0].Reference.Kind != outcomes.OperationProposal || operations[1].Reference.Kind != outcomes.OperationValidation || *operations[2].Reference.ProposalIndex != 1 || *operations[2].Reference.RepairAttempt != 1 || *operations[3].Reference.RepairAttempt != 2 || operations[3].Reference.Kind != outcomes.OperationCorrectionRepair || operations[4].Reference.Kind != outcomes.OperationValidation || *operations[5].Reference.ProposalIndex != 2 {
t.Fatalf("order: %+v", operations)
}
}