Retain processing evidence only for selected outputs

This commit is contained in:
2026-11-16 15:08:02 +00:00
parent 5167154a1c
commit 891d4aac2c
21 changed files with 494 additions and 229 deletions

View File

@@ -74,7 +74,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, Evidence: &runner.EvidenceSelection{Reports: true, Debug: true},
})
if !errors.Is(err, origin) || client.calls != 3 {
t.Fatalf("fatal cause/calls: %v %d", err, client.calls)

View File

@@ -66,59 +66,35 @@ func Build(input BuildInput) reporting.Snapshot {
return units[i].Section.Index < units[j].Section.Index
})
}
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0
var measured outcomes.WorkTotals
if input.RunOutput.Totals != nil {
measured = *input.RunOutput.Totals
} else {
measured.Total = len(units)
for _, unit := range units {
measured.Record(unit.State, unit.Operations)
measured.Corrections += unit.CorrectionsApplied
}
}
for _, unit := range units {
if unit.State == outcomes.WorkNotStarted {
notStarted++
} else {
started++
}
if unit.State == outcomes.WorkCompleted {
completed++
}
if unit.State == outcomes.WorkInterrupted {
interrupted++
}
corrections += unit.CorrectionsApplied
operations := unit.Operations
if details {
operations = append([]outcomes.Operation(nil), operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if !details {
break
}
operations := append([]outcomes.Operation(nil), unit.Operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
var failures []reporting.Failure
unitHasError := false
unitRecoveries := 0
for _, operation := range operations {
if operation.Reference.Kind != outcomes.OperationValidationInput {
llmOperations++
}
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
proposalResponses++
}
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
recoveryCount++
unitRecoveries++
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal {
filteringResponses++
}
if details {
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
}
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
}
if operation.FailureClass == outcomes.FailureRecoverable {
unitHasError = true
if details {
failures = append(failures, mapFailure(operation))
}
failures = append(failures, mapFailure(operation))
}
}
if unitHasError {
withErrors++
}
if !details {
continue
}
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries}
@@ -153,12 +129,13 @@ func Build(input BuildInput) reporting.Snapshot {
return a.ProposalIndex < b.ProposalIndex
})
runErrors := len(failures)
interrupted, notStarted := measured.Started-measured.Completed, measured.Total-measured.Started
snapshot.Summary = reporting.Summary{
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: &notStarted, WorkUnitsWithErrors: &withErrors,
RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations,
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount,
ProposalResponsesWithFiltering: &filteringResponses,
WorkUnitsTotal: &measured.Total, WorkUnitsStarted: &measured.Started, WorkUnitsCompleted: &measured.Completed,
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: &notStarted, WorkUnitsWithErrors: &measured.WithErrors,
RunErrors: &runErrors, CorrectionsApplied: &measured.Corrections, LLMOperations: &measured.ModelOperations,
ProposalResponses: &measured.ProposalResponses, Recoveries: &measured.Recoveries,
ProposalResponsesWithFiltering: &measured.FilteringResponses,
}
} else if len(failures) > 0 {
runErrors := len(failures)