eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Report correction repair operations and outcomes

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:31:44 +00:00
parent d927c61199
commit 735dfc1834
10 changed files with 159 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/framework/outcomes/outcomes.go
View File

@@ -7,9 +7,10 @@ package outcomes
type OperationKind string
const (
OperationProposal OperationKind = "proposal"
OperationValidation OperationKind = "validation"
OperationValidationInput OperationKind = "validation_input"
OperationProposal OperationKind = "proposal"
OperationCorrectionRepair OperationKind = "correction_repair"
OperationValidation OperationKind = "validation"
OperationValidationInput OperationKind = "validation_input"
)
// Reference is the stable processing identity of an operation. Proposal indices
@@ -22,6 +23,8 @@ type Reference struct {
ModuleInstance string `json:"module_instance"`
SectionIndex int `json:"section_index"`
Kind OperationKind `json:"operation"`
ProposalIndex *int `json:"proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ValidatorOrder *int `json:"validator_order,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"`
@@ -61,6 +64,7 @@ const (
type RecoveryReason string
const (
RecoveryCorrectionRevised RecoveryReason = "correction_revised"
RecoveryOutputRepaired RecoveryReason = "output_repaired"
RecoveryServiceRecovered RecoveryReason = "service_recovered"
RecoveryProposalsFiltered RecoveryReason = "proposals_filtered"

38
internal/framework/processreport/report.go
View File

@@ -123,7 +123,23 @@ func Build(input BuildInput) reporting.Snapshot {
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
}
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
for _, repair := range unit.Repairs {
if len(repair.Attempts) == 0 {
continue
}
snapshot.RepairOutcomes = append(snapshot.RepairOutcomes, reporting.RepairOutcome{StageIndex: unit.ModuleIndex, ChunkIndex: unit.Section.Index, ProposalIndex: repair.ProposalIndex, Attempts: len(repair.Attempts), Outcome: repair.Outcome})
}
}
sort.SliceStable(snapshot.RepairOutcomes, func(i, j int) bool {
a, b := snapshot.RepairOutcomes[i], snapshot.RepairOutcomes[j]
if a.StageIndex != b.StageIndex {
return a.StageIndex < b.StageIndex
}
if a.ChunkIndex != b.ChunkIndex {
return a.ChunkIndex < b.ChunkIndex
}
return a.ProposalIndex < b.ProposalIndex
})
runErrors := len(input.RunFailures)
snapshot.Summary = reporting.Summary{
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
@@ -247,15 +263,31 @@ func operationLess(a, b outcomes.Operation) bool {
return 2
}
}
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
order := func(p *int) int {
if p == nil {
return -1
}
return *p
}
aRepair, bRepair := a.Reference.RepairAttempt != nil, b.Reference.RepairAttempt != nil
if aRepair != bRepair {
return !aRepair
}
if aRepair {
if order(a.Reference.ProposalIndex) != order(b.Reference.ProposalIndex) {
return order(a.Reference.ProposalIndex) < order(b.Reference.ProposalIndex)
}
if order(a.Reference.RepairAttempt) != order(b.Reference.RepairAttempt) {
return order(a.Reference.RepairAttempt) < order(b.Reference.RepairAttempt)
}
if (a.Reference.Kind == outcomes.OperationCorrectionRepair) != (b.Reference.Kind == outcomes.OperationCorrectionRepair) {
return a.Reference.Kind == outcomes.OperationCorrectionRepair
}
}
if rank(a.Reference.Kind) != rank(b.Reference.Kind) && (a.Reference.Kind == outcomes.OperationProposal || b.Reference.Kind == outcomes.OperationProposal) {
return rank(a.Reference.Kind) < rank(b.Reference.Kind)
}
if order(a.Reference.ValidatorOrder) != order(b.Reference.ValidatorOrder) {
return order(a.Reference.ValidatorOrder) < order(b.Reference.ValidatorOrder)
}

52
internal/framework/processreport/report_test.go
View File

@@ -2,6 +2,7 @@ package processreport
import (
"reflect"
"sort"
"strings"
"testing"
"time"
@@ -137,3 +138,54 @@ func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
}
func intPtr(value int) *int { return &value }
func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
}
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
for i, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
units[0].Repairs = append(units[0].Repairs, runner.RepairOutcome{ProposalIndex: 6 - i, Outcome: disposition, Attempts: []runner.RepairAttempt{{}}})
}
snapshot := Build(BuildInput{RunID: "r", ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}}, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if *snapshot.Summary.LLMOperations != 5 || *snapshot.Summary.ProposalResponses != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || snapshot.Errors[0].Failures[0].Operation != "correction_repair" || len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{"correction_revised", "output_repaired"}) {
t.Fatalf("reports: %+v %+v", snapshot.Errors, snapshot.Recoveries)
}
if len(snapshot.RepairOutcomes) != 7 || snapshot.RepairOutcomes[0].ProposalIndex != 0 || snapshot.RepairOutcomes[6].ProposalIndex != 6 {
t.Fatalf("outcomes: %+v", snapshot.RepairOutcomes)
}
text := string(reporting.Markdown(snapshot))
for _, disposition := range []string{"applied", "approved_skipped", "withdrawn", "rejected", "exhausted", "unavailable", "interrupted"} {
if !strings.Contains(text, "| "+disposition+" |") {
t.Errorf("missing %s in %s", disposition, text)
}
}
summary := Build(BuildInput{SummaryOnly: true, RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: units}})
if !reflect.DeepEqual(summary.Summary, snapshot.Summary) || len(summary.RepairOutcomes) != 0 {
t.Fatalf("summary-only: %+v", summary)
}
}
func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) {
zero, one, two := 0, 1, 2
operations := []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &one, RepairAttempt: &two, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &two}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}},
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}},
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}},
}
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if operations[0].Reference.Kind != outcomes.OperationProposal || operations[1].Reference.Kind != outcomes.OperationValidation || *operations[2].Reference.ProposalIndex != 1 || *operations[2].Reference.RepairAttempt != 1 || *operations[3].Reference.RepairAttempt != 2 || operations[3].Reference.Kind != outcomes.OperationCorrectionRepair || operations[4].Reference.Kind != outcomes.OperationValidation || *operations[5].Reference.ProposalIndex != 2 {
t.Fatalf("order: %+v", operations)
}
}

29
internal/framework/runner/repair.go
View File

@@ -21,6 +21,7 @@ type RepairAttempt struct {
Counts outcomes.ExecutionCounts
FailureReason outcomes.ReasonCode
Succeeded bool
Approved bool
Operations []outcomes.Operation
}
@@ -137,6 +138,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
record.Revision = &revisionSnapshot
record.Decisions = validated.decisions
record.Rejected = validated.rejected
for i := range validated.operations {
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
validated.operations[i].Reference.ProposalIndex = &proposalIndex
validated.operations[i].Reference.RepairAttempt = &attemptNumber
}
record.Operations = validated.operations
branch.Attempts = append(branch.Attempts, record)
initial.operations = append(initial.operations, validated.operations...)
@@ -146,6 +152,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return initial, branches, validationErr
}
if len(validated.approved) == 1 {
branch.Attempts[len(branch.Attempts)-1].Approved = true
branch.Outcome = "approved"
initial.approved = append(initial.approved, current)
retained := initial.filtered[:0]
@@ -177,3 +184,25 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
return initial, branches, nil
}
func repairOperations(input validateSectionCandidatesInput, branches []RepairOutcome) []outcomes.Operation {
var result []outcomes.Operation
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
}
for _, branch := range branches {
for i, attempt := range branch.Attempts {
index, number := branch.ProposalIndex, i+1
operation := outcomes.Operation{Reference: outcomes.Reference{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: section, Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &index, RepairAttempt: &number}, Counts: attempt.Counts, Succeeded: attempt.Succeeded, FailureReason: attempt.FailureReason, ProposalIndices: []int{index}}
if attempt.Succeeded {
operation.RecoveryReasons = outcomes.ExecutionRecoveries(attempt.Counts)
if attempt.Approved {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryCorrectionRevised)
}
}
result = append(result, operation)
}
}
return result
}

3
internal/framework/runner/repair_test.go
View File

@@ -121,6 +121,9 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
if out.FinalTranscript.Segments[0].Text != "dog" || len(out.WorkUnits) != 1 || len(out.WorkUnits[0].Repairs) != 1 || out.WorkUnits[0].Repairs[0].Outcome != "approved_skipped" || out.WorkUnits[0].CorrectionsApplied != 1 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].ProposalIndex != 1 {
t.Fatalf("output=%+v section=%d", out, sectionIndex)
}
if len(out.WorkUnits[0].Operations) != 1 || out.WorkUnits[0].Operations[0].Reference.Kind != outcomes.OperationCorrectionRepair || len(out.WorkUnits[0].Operations[0].RecoveryReasons) != 1 || out.WorkUnits[0].Operations[0].RecoveryReasons[0] != outcomes.RecoveryCorrectionRevised {
t.Fatalf("repair operations: %+v", out.WorkUnits[0].Operations)
}
}
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) {

1
internal/framework/runner/runner.go
View File

@@ -564,6 +564,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
repairInput := validateSectionCandidatesInput{Spec: input.Spec, Policy: input.Policy, WorkingTranscript: transcriptFromSection(sectionResult.section), SectionIndex: &sectionMetadata.Index, Validators: validatorsOrdered, SectionEnriched: enriched, Exchange: sectionResult.exchange, ValidationLLMClient: input.ValidationClient, DiagnosticsDir: input.ValidationDiagnosticsDir, GlossaryView: input.GlossaryView, ReportDiagnosticError: input.ReportDiagnosticError}
validated, repairs, err = repairCandidates(runCtx, repairInput, sectionResult.references, validated, promptID)
validated.operations = append(validated.operations, repairOperations(repairInput, repairs)...)
}
setErr(err)
validationResults <- sectionValidationResult{