Project typed failure origins and join correction evidence by identity
This commit is contained in:
@@ -51,10 +51,15 @@ count, and a `failures` sequence. Each failure identifies the operation
|
||||
(`proposal`, `validation`, `validation_input`, or `correction_repair`), optional validator and
|
||||
zero-based batch index, safe reason/message, sorted proposal indices when
|
||||
known, and execution counts when available. Multiple failed batches in one
|
||||
work unit form one error record. `scope: run` instead records phase, safe reason
|
||||
and message, optional origin, and output path where relevant. It can describe
|
||||
a fatal error or secondary output-write failure without inventing a failed
|
||||
model work unit. Raw provider errors and transcript text are absent.
|
||||
work unit form one error record. A record with scope `run` records a fatal
|
||||
processing failure or a separate output delivery failure. It identifies the
|
||||
phase, safe reason and message, and available origin. A processing origin can
|
||||
include the operation, validator, batch, proposal index, and repair attempt.
|
||||
A delivery failure includes the output path when known. A record with scope
|
||||
`interrupted` preserves an interrupted peer's cause and location when
|
||||
available. These records do not add to the recoverable work-unit or run-error
|
||||
counts. The work-unit matrix still marks the interrupted work. Raw provider
|
||||
errors and transcript text are absent.
|
||||
For `validation_decisions_invalid`, proposal indices identify only candidates
|
||||
whose required decisions were unavailable. Other candidates in the same batch
|
||||
may still be approved or rejected normally.
|
||||
|
||||
@@ -72,3 +72,10 @@ Useful tests: `internal/core/reporting/*_test.go`,
|
||||
`internal/core/diagnostics/*_test.go`,
|
||||
`internal/framework/processreport/*_test.go`, and
|
||||
`cmd/audita/main_integration_test.go`.
|
||||
|
||||
Report projection uses typed operation and delivery results. Recoverable
|
||||
operations count toward work-unit errors. A fatal processing cause retains its
|
||||
own phase and available operation coordinates, and peer interruptions and later
|
||||
file failures preserve their own attribution. The correction ledger joins one
|
||||
candidate row by module instance and proposal index, preserving repair lineage
|
||||
and the recorded validator execution class.
|
||||
|
||||
@@ -629,6 +629,15 @@ specified mixed run, including partial decisions, repair/filter interaction,
|
||||
later failure, and an overridden validator class. Update the
|
||||
[output contract](../integrations/output-files.md) and diagnostics internals.
|
||||
|
||||
**Complete.** Reports project typed operation, processing, and delivery
|
||||
facts, with separate attribution for the primary failure, interrupted peers,
|
||||
and file failures. The ledger joins each candidate's evidence once, uses
|
||||
recorded validator classes, and preserves literal empty replacements and
|
||||
repair lineage. Apply-time filtering reuses original section lookups and
|
||||
groups skip evidence by its originating operation. The full offline Go suite
|
||||
passed, including mixed YAML and Markdown counts, ledger class overrides,
|
||||
and the repair/filter/later-failure subprocess fixture.
|
||||
|
||||
## Stage 18 — Retain evidence according to selected outputs
|
||||
|
||||
Audit #32 and supporting work for #30, #33, and #44.
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/preflight"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
@@ -71,7 +72,7 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
|
||||
warn(debug.WriteNormalizationSummary(summary))
|
||||
}
|
||||
if err := inv.Context.Err(); err != nil {
|
||||
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, err)
|
||||
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, &runner.ExecutionFailure{Phase: "normalization", Cause: err})
|
||||
}
|
||||
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
|
||||
if runErr != nil {
|
||||
@@ -79,14 +80,14 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
|
||||
}
|
||||
encoder, err := outputschema.Resolve(cfg.Output.Schema)
|
||||
if err != nil {
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, &runner.ExecutionFailure{Phase: "encoding", Cause: err})
|
||||
}
|
||||
if result.Transcript == nil {
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, &runner.ExecutionFailure{Phase: "encoding", Cause: fmt.Errorf("processing returned no transcript")})
|
||||
}
|
||||
payload, err := encoder.Encoder(result.Transcript)
|
||||
if err != nil {
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, &runner.ExecutionFailure{Phase: "encoding", Cause: err})
|
||||
}
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, payload, nil)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/preflight"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/processreport"
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
@@ -116,10 +117,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
|
||||
failedSnapshot := snapshot
|
||||
completed := time.Now().UTC()
|
||||
failedSnapshot.CompletedAt = &completed
|
||||
if primary != nil {
|
||||
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
|
||||
}
|
||||
return renderReportBundle(failedOutputSnapshot(failedSnapshot, failures))
|
||||
return renderReportBundle(processreport.Failed(failedSnapshot, primary, "initialization", failures))
|
||||
}
|
||||
}
|
||||
var debug *preflight.DebugInput
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
@@ -9,7 +8,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/processreport"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
@@ -28,11 +27,12 @@ func finishProcess(inv commandkit.Invocation, input processInput, prepared *prep
|
||||
if len(reportPaths) > 0 {
|
||||
render = func(cause error, failures []diagnostics.FileFailure) (diagnostics.ReportBundle, error) {
|
||||
report := snapshot
|
||||
if cause != nil {
|
||||
report = failedProcessingSnapshot(report, cause, inv.Context.Err())
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
report = failedOutputSnapshot(report, failures)
|
||||
if cause != nil || len(failures) > 0 {
|
||||
phase := "processing"
|
||||
if primary == nil {
|
||||
phase = "publication"
|
||||
}
|
||||
report = processreport.Failed(snapshot, cause, phase, failures)
|
||||
}
|
||||
return renderReportBundle(report)
|
||||
}
|
||||
@@ -55,43 +55,3 @@ func finishProcess(inv commandkit.Invocation, input processInput, prepared *prep
|
||||
}
|
||||
return commandkit.Success()
|
||||
}
|
||||
|
||||
func failedProcessingSnapshot(snapshot reporting.Snapshot, primary, contextErr error) reporting.Snapshot {
|
||||
for _, record := range snapshot.Errors {
|
||||
if record.Scope == "run" {
|
||||
return failedOutputSnapshot(snapshot, nil)
|
||||
}
|
||||
}
|
||||
record := reporting.ErrorRecord{Scope: "run", Reason: "processing_failure", Message: "Processing failed"}
|
||||
if errors.Is(primary, context.Canceled) || (contextErr != nil && errors.Is(primary, contextErr)) {
|
||||
record.Phase = "processing"
|
||||
record.Reason = string(outcomes.ReasonCanceled)
|
||||
record.Message = "Processing was canceled."
|
||||
}
|
||||
snapshot.Errors = append(snapshot.Errors, record)
|
||||
return failedOutputSnapshot(snapshot, nil)
|
||||
}
|
||||
|
||||
// failedOutputSnapshot derives the count from the same run records sent to each report.
|
||||
func failedOutputSnapshot(snapshot reporting.Snapshot, failures []diagnostics.FileFailure) reporting.Snapshot {
|
||||
snapshot.Status = reporting.StatusFailed
|
||||
if len(failures) > 0 {
|
||||
snapshot.Errors = append([]reporting.ErrorRecord(nil), snapshot.Errors...)
|
||||
}
|
||||
for _, failure := range failures {
|
||||
record := reporting.ErrorRecord{Scope: "run", Reason: "output_failure", Message: "Required output delivery failed", OutputPath: failure.Path}
|
||||
if failure.Kind == "report initialization" {
|
||||
record.Phase = "initialization"
|
||||
record.Message = "Required report initialization failed"
|
||||
}
|
||||
snapshot.Errors = append(snapshot.Errors, record)
|
||||
}
|
||||
count := 0
|
||||
for _, record := range snapshot.Errors {
|
||||
if record.Scope == "run" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
snapshot.Summary.RunErrors = &count
|
||||
return snapshot
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
@@ -28,19 +27,24 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
}
|
||||
for i := range specs {
|
||||
item := &specs[i]
|
||||
setupFailure := func(role string, cause error) error {
|
||||
return &runner.ExecutionFailure{Phase: "setup", Origin: &outcomes.Reference{
|
||||
StageIndex: &item.Index, StageID: item.ID, ModuleKey: item.ModuleKey, ModuleInstance: item.InstanceName,
|
||||
}, Cause: fmt.Errorf("%s %s client: %w", item.InstanceName, role, cause)}
|
||||
}
|
||||
proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, hooks)
|
||||
if err != nil {
|
||||
return processResult{}, fmt.Errorf("%s proposal client: %w", item.InstanceName, err)
|
||||
return processResult{}, setupFailure("proposal", err)
|
||||
}
|
||||
validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, hooks)
|
||||
if err != nil {
|
||||
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
|
||||
return processResult{}, setupFailure("validation", err)
|
||||
}
|
||||
item.ProposalClient, item.ValidationClient = proposal, validation
|
||||
if item.Settings.LLM.CorrectionRepairAttempts > 0 {
|
||||
repair, err := loaded.Runtime.NewRoleClient("correction_repair", *item.Settings.LLM.CorrectionRepairProfile, item.Settings.LLM, hooks)
|
||||
if err != nil {
|
||||
return processResult{}, fmt.Errorf("%s correction repair client: %w", item.InstanceName, err)
|
||||
return processResult{}, setupFailure("correction repair", err)
|
||||
}
|
||||
item.RepairCoordinator = repair
|
||||
}
|
||||
@@ -63,27 +67,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observations)))
|
||||
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
||||
}
|
||||
var failures []processreport.RunFailure
|
||||
if runErr != nil {
|
||||
failure := processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure}
|
||||
var cause *structuredoutput.Failure
|
||||
if errors.As(runErr, &cause) {
|
||||
if cause.Reason != "" {
|
||||
failure.Reason = cause.Reason
|
||||
}
|
||||
} else if errors.Is(runErr, context.Canceled) || (ctx.Err() != nil && errors.Is(runErr, ctx.Err())) {
|
||||
failure.Reason = outcomes.ReasonCanceled
|
||||
}
|
||||
if len(output.ModuleResults) > 0 {
|
||||
last := output.ModuleResults[len(output.ModuleResults)-1]
|
||||
if last.Status == runner.ModuleStatusFailed {
|
||||
index := last.StageIndex
|
||||
failure.Origin = &outcomes.Reference{StageIndex: &index, StageID: last.StageID, ModuleKey: last.ModuleKey, ModuleInstance: last.ModuleInstance}
|
||||
}
|
||||
}
|
||||
failures = append(failures, failure)
|
||||
}
|
||||
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
|
||||
ModuleSpecs: specs, RunOutput: &output, RunFailures: failures, SummaryOnly: !cfg.Reporting.Enabled})
|
||||
ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !cfg.Reporting.Enabled})
|
||||
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
|
||||
}
|
||||
|
||||
@@ -34,10 +34,20 @@ type ReportBundle struct {
|
||||
Markdown []byte
|
||||
}
|
||||
|
||||
type OutputPhase string
|
||||
|
||||
const (
|
||||
OutputInitialization OutputPhase = "initialization"
|
||||
OutputTranscriptWrite OutputPhase = "transcript_write"
|
||||
OutputReportWrite OutputPhase = "report_write"
|
||||
OutputCleanup OutputPhase = "output_cleanup"
|
||||
)
|
||||
|
||||
type FileFailure struct {
|
||||
Path string
|
||||
Kind string
|
||||
Err error
|
||||
Phase OutputPhase
|
||||
Path string
|
||||
Kind string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (f FileFailure) Error() string { return fmt.Sprintf("%s %q: %v", f.Kind, f.Path, f.Err) }
|
||||
@@ -273,16 +283,16 @@ func (s *OutputSession) Finish(ctx context.Context, primary error, transcript []
|
||||
|
||||
func (s *OutputSession) finalize(ctx context.Context, primary error, transcript []byte, render ReportRenderer) []FileFailure {
|
||||
if s == nil {
|
||||
return []FileFailure{{Kind: "output finalization", Err: fmt.Errorf("output session is required")}}
|
||||
return []FileFailure{{Phase: OutputCleanup, Kind: "output finalization", Err: fmt.Errorf("output session is required")}}
|
||||
}
|
||||
var failures []FileFailure
|
||||
if primary == nil && ctx.Err() == nil && transcript != nil {
|
||||
if err := s.writeTranscript(ctx, transcript); err != nil {
|
||||
failures = append(failures, FileFailure{Path: s.transcript, Kind: "transcript write", Err: err})
|
||||
failures = append(failures, FileFailure{Path: s.transcript, Phase: OutputTranscriptWrite, Kind: "transcript write", Err: err})
|
||||
}
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
failures = append(failures, FileFailure{Path: s.transcript, Kind: "output cleanup", Err: err})
|
||||
failures = append(failures, FileFailure{Path: s.transcript, Phase: OutputCleanup, Kind: "output cleanup", Err: err})
|
||||
}
|
||||
if !s.enabled || render == nil {
|
||||
return failures
|
||||
@@ -293,7 +303,7 @@ func (s *OutputSession) finalize(ctx context.Context, primary error, transcript
|
||||
}
|
||||
reports, err := render(nil, nil)
|
||||
if err != nil {
|
||||
failures = append(failures, FileFailure{Kind: "report rendering", Err: err})
|
||||
failures = append(failures, FileFailure{Phase: OutputReportWrite, Kind: "report rendering", Err: err})
|
||||
}
|
||||
return s.publishReports(ctx, reports, primary, failures, "report write", render)
|
||||
}
|
||||
@@ -301,10 +311,14 @@ func (s *OutputSession) finalize(ctx context.Context, primary error, transcript
|
||||
func (s *OutputSession) publishReports(ctx context.Context, reports ReportBundle, primary error, failures []FileFailure, action string, render ReportRenderer) (result []FileFailure) {
|
||||
defer func() {
|
||||
if err := s.cleanupReports(); err != nil {
|
||||
result = append(result, FileFailure{Kind: "output cleanup", Err: err})
|
||||
result = append(result, FileFailure{Phase: OutputCleanup, Kind: "output cleanup", Err: err})
|
||||
}
|
||||
}()
|
||||
primary = includeCancellation(ctx, primary)
|
||||
phase := OutputReportWrite
|
||||
if action == "report initialization" {
|
||||
phase = OutputInitialization
|
||||
}
|
||||
failed := false
|
||||
var failedWritten [3]bool
|
||||
write := func(i int, payload []byte) {
|
||||
@@ -315,7 +329,7 @@ func (s *OutputSession) publishReports(ctx context.Context, reports ReportBundle
|
||||
kind = "failed " + action
|
||||
}
|
||||
if err := s.replace(writeCtx, s.reportDestinations[i], payload); err != nil {
|
||||
failures = append(failures, FileFailure{Path: s.reports[i], Kind: kind, Err: err})
|
||||
failures = append(failures, FileFailure{Path: s.reports[i], Phase: phase, Kind: kind, Err: err})
|
||||
} else if failed {
|
||||
failedWritten[i] = true
|
||||
}
|
||||
@@ -323,7 +337,7 @@ func (s *OutputSession) publishReports(ctx context.Context, reports ReportBundle
|
||||
renderFailed := func() bool {
|
||||
if action == "report initialization" {
|
||||
if err := s.Close(); err != nil {
|
||||
failures = append(failures, FileFailure{Path: s.transcript, Kind: "output cleanup", Err: err})
|
||||
failures = append(failures, FileFailure{Path: s.transcript, Phase: OutputCleanup, Kind: "output cleanup", Err: err})
|
||||
}
|
||||
}
|
||||
if render == nil {
|
||||
@@ -332,7 +346,7 @@ func (s *OutputSession) publishReports(ctx context.Context, reports ReportBundle
|
||||
var err error
|
||||
reports, err = render(includeCancellation(ctx, primary), failures)
|
||||
if err != nil {
|
||||
failures = append(failures, FileFailure{Kind: "failed report rendering", Err: err})
|
||||
failures = append(failures, FileFailure{Phase: phase, Kind: "failed report rendering", Err: err})
|
||||
return false
|
||||
}
|
||||
failed = true
|
||||
|
||||
@@ -65,6 +65,8 @@ type ErrorRecord struct {
|
||||
Operation string `yaml:"operation,omitempty"`
|
||||
Validator string `yaml:"validator,omitempty"`
|
||||
BatchIndex *int `yaml:"batch_index,omitempty"`
|
||||
ProposalIndex *int `yaml:"proposal_index,omitempty"`
|
||||
RepairAttempt *int `yaml:"repair_attempt,omitempty"`
|
||||
OutputPath string `yaml:"output_path,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestUnavailableCandidateAndFailureKeepIdentity(t *testing.T) {
|
||||
index := 1
|
||||
output := &runner.RunOutput{InventoryAvailable: true, ModuleResults: []runner.ModuleResult{
|
||||
{StageIndex: 1, StageID: &id, ModuleKey: "grammar", ModuleInstance: "grammar_2", ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}},
|
||||
}, WorkUnits: []runner.WorkUnitOutcome{{ModuleIndex: 1, StageIndex: 1, StageID: &id, ModuleKey: "grammar", ModuleInstance: "grammar_2", State: outcomes.WorkCompleted, Operations: []outcomes.Operation{{Reference: outcomes.Reference{StageIndex: &index, StageID: &id, Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed}}}}}
|
||||
}, WorkUnits: []runner.WorkUnitOutcome{{ModuleIndex: 1, StageIndex: 1, StageID: &id, ModuleKey: "grammar", ModuleInstance: "grammar_2", State: outcomes.WorkCompleted, Operations: []outcomes.Operation{{Reference: outcomes.Reference{StageIndex: &index, StageID: &id, Kind: outcomes.OperationValidation}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonProviderFailed}}}}}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
if len(ledger) != 1 || ledger[0].Disposition != "unavailable" || ledger[0].StageIndex != 1 || ledger[0].StageID == nil || *ledger[0].StageID != id {
|
||||
t.Fatalf("ledger=%+v", ledger)
|
||||
|
||||
@@ -67,9 +67,24 @@ type LedgerValidatorDecisionRecord struct {
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type proposalIdentity struct {
|
||||
instance string
|
||||
index int
|
||||
}
|
||||
|
||||
type candidateEvidence struct {
|
||||
candidate *proposals.EnrichedCorrectionProposal
|
||||
applied *proposals.AppliedChange
|
||||
skipped *proposals.SkippedChange
|
||||
rejection *runner.ValidatorRejectedChange
|
||||
repair *runner.RepairOutcome
|
||||
deterministic []LedgerValidatorDecisionRecord
|
||||
llm []LedgerValidatorDecisionRecord
|
||||
}
|
||||
|
||||
func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry {
|
||||
runOutput := input.RunOutput
|
||||
if runOutput == nil || len(runOutput.ModuleResults) == 0 {
|
||||
output := input.RunOutput
|
||||
if output == nil || len(output.ModuleResults) == 0 {
|
||||
return nil
|
||||
}
|
||||
runID := ""
|
||||
@@ -78,190 +93,150 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
|
||||
}
|
||||
|
||||
entries := make([]CorrectionLedgerEntry, 0)
|
||||
type proposalIdentity struct {
|
||||
instance string
|
||||
index int
|
||||
modules := make(map[string]*runner.ModuleResult, len(output.ModuleResults))
|
||||
evidence := make(map[proposalIdentity]*candidateEvidence, len(output.Candidates))
|
||||
get := func(instance string, index int) *candidateEvidence {
|
||||
key := proposalIdentity{instance, index}
|
||||
item := evidence[key]
|
||||
if item == nil {
|
||||
item = &candidateEvidence{}
|
||||
evidence[key] = item
|
||||
}
|
||||
return item
|
||||
}
|
||||
type stageEvidence struct {
|
||||
index int
|
||||
id *string
|
||||
policy string
|
||||
decisions map[int][]runner.ValidatorDecisionRecord
|
||||
for i := range output.Candidates {
|
||||
candidate := &output.Candidates[i]
|
||||
get(candidate.ModuleInstance, candidate.ProposalIndex).candidate = candidate
|
||||
}
|
||||
repairs := make(map[proposalIdentity]runner.RepairOutcome)
|
||||
for _, unit := range runOutput.WorkUnits {
|
||||
for _, repair := range unit.Repairs {
|
||||
repairs[proposalIdentity{unit.ModuleInstance, repair.ProposalIndex}] = repair
|
||||
for _, unit := range output.WorkUnits {
|
||||
for i := range unit.Repairs {
|
||||
repair := &unit.Repairs[i]
|
||||
get(unit.ModuleInstance, repair.ProposalIndex).repair = repair
|
||||
}
|
||||
}
|
||||
seen := make(map[proposalIdentity]bool)
|
||||
repairSkips := make(map[proposalIdentity]proposals.SkippedChange)
|
||||
stages := make(map[string]stageEvidence, len(runOutput.ModuleResults))
|
||||
for _, module := range runOutput.ModuleResults {
|
||||
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
|
||||
for i := range output.ModuleResults {
|
||||
module := &output.ModuleResults[i]
|
||||
instance := module.ModuleInstance
|
||||
modules[instance] = module
|
||||
for i := range module.AppliedChanges {
|
||||
change := &module.AppliedChanges[i]
|
||||
get(instance, change.ProposalIndex).applied = change
|
||||
}
|
||||
for i := range module.SkippedChanges {
|
||||
change := &module.SkippedChanges[i]
|
||||
get(instance, change.ProposalIndex).skipped = change
|
||||
}
|
||||
for i := range module.ValidatorRejected {
|
||||
rejection := &module.ValidatorRejected[i]
|
||||
item := get(instance, rejection.ProposalIndex)
|
||||
if item.rejection == nil || (item.rejection.Disposition == validators.DispositionUnavailable && rejection.Disposition == validators.DispositionRejected) {
|
||||
item.rejection = rejection
|
||||
}
|
||||
}
|
||||
for _, decision := range module.ValidatorDecisions {
|
||||
decisionsByProposal[decision.ProposalIndex] = append(decisionsByProposal[decision.ProposalIndex], decision)
|
||||
}
|
||||
if _, exists := stages[module.ModuleInstance]; !exists {
|
||||
stages[module.ModuleInstance] = stageEvidence{
|
||||
index: module.StageIndex, id: module.StageID, policy: string(module.ReplacementPolicy), decisions: decisionsByProposal,
|
||||
}
|
||||
}
|
||||
|
||||
for _, change := range module.AppliedChanges {
|
||||
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
|
||||
continue
|
||||
}
|
||||
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
StageIndex: module.StageIndex, StageID: module.StageID,
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
ProposalIndex: change.ProposalIndex,
|
||||
SegmentID: change.TargetSegmentID,
|
||||
OriginalText: change.OriginalText,
|
||||
ProposedCorrectedText: &change.CorrectedText,
|
||||
AppliedCorrectedText: &change.CorrectedText,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
Disposition: correctionDispositionApplied,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
|
||||
})
|
||||
}
|
||||
for _, change := range module.SkippedChanges {
|
||||
identity := proposalIdentity{module.ModuleInstance, change.ProposalIndex}
|
||||
if _, ok := repairs[identity]; ok {
|
||||
repairSkips[identity] = change
|
||||
continue
|
||||
}
|
||||
seen[identity] = true
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
StageIndex: module.StageIndex, StageID: module.StageID,
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
ProposalIndex: change.ProposalIndex,
|
||||
SegmentID: change.TargetSegmentID,
|
||||
OriginalText: change.OriginalText,
|
||||
ProposedCorrectedText: &change.CorrectedText,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
Disposition: correctionDispositionSkipped,
|
||||
DispositionReasonCode: string(change.SkipReason),
|
||||
DispositionMessage: change.Message,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
|
||||
})
|
||||
}
|
||||
for _, rejection := range module.ValidatorRejected {
|
||||
if seen[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}] {
|
||||
continue
|
||||
}
|
||||
if _, ok := repairs[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}]; ok {
|
||||
continue
|
||||
}
|
||||
disposition := correctionDispositionRejected
|
||||
if rejection.Disposition == validators.DispositionUnavailable {
|
||||
disposition = correctionDispositionUnavailable
|
||||
}
|
||||
seen[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}] = true
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
StageIndex: module.StageIndex, StageID: module.StageID,
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
ProposalIndex: rejection.ProposalIndex,
|
||||
SegmentID: rejection.TargetSegmentID,
|
||||
OriginalText: rejection.OriginalText,
|
||||
ProposedCorrectedText: &rejection.CorrectedText,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
Disposition: disposition,
|
||||
DispositionReasonCode: rejection.ReasonCode,
|
||||
DispositionMessage: rejection.Message,
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true),
|
||||
})
|
||||
item := get(instance, decision.ProposalIndex)
|
||||
recordLedgerDecision(decision, &item.deterministic, &item.llm)
|
||||
}
|
||||
if module.Status == runner.ModuleStatusFailed {
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
StageIndex: module.StageIndex, StageID: module.StageID,
|
||||
RunID: runID,
|
||||
ModuleKey: module.ModuleKey,
|
||||
ModuleInstance: module.ModuleInstance,
|
||||
Disposition: correctionDispositionFailed,
|
||||
DispositionReasonCode: "module_failed",
|
||||
DispositionMessage: module.ErrorMessage,
|
||||
StageIndex: module.StageIndex, StageID: module.StageID, RunID: runID,
|
||||
ModuleKey: module.ModuleKey, ModuleInstance: instance,
|
||||
Disposition: correctionDispositionFailed, DispositionReasonCode: "module_failed", DispositionMessage: module.ErrorMessage,
|
||||
})
|
||||
}
|
||||
}
|
||||
for identity, repair := range repairs {
|
||||
stage := stages[identity.instance]
|
||||
original := repair.Original
|
||||
entry := CorrectionLedgerEntry{StageIndex: stage.index, StageID: stage.id, RunID: runID, ModuleKey: original.ModuleKey, ModuleInstance: identity.instance, ProposalIndex: identity.index, SegmentID: original.TargetSegmentID, OriginalText: original.OriginalText, ProposedCorrectedText: &original.CorrectedText, ReplacementPolicy: stage.policy, Disposition: repair.Outcome, DeterministicValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], false), LLMValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], true)}
|
||||
for i, attempt := range repair.Attempts {
|
||||
item := LedgerRepairAttempt{Attempt: i + 1, Revision: attempt.Revision, FailureReason: string(attempt.FailureReason), RequestMetadataPath: attempt.Artifacts.RequestMetadataPath, RequestPayloadPath: attempt.Artifacts.RequestPayloadPath, ResponsePayloadPath: attempt.Artifacts.ResponsePayloadPath, ErrorPayloadPath: attempt.Artifacts.ErrorPayloadPath, DeterministicValidatorResults: filterLedgerDecisions(attempt.Decisions, false), LLMValidatorResults: filterLedgerDecisions(attempt.Decisions, true)}
|
||||
entry.RepairAttempts = append(entry.RepairAttempts, item)
|
||||
}
|
||||
if repair.Outcome == "applied" {
|
||||
segment, before, after := repair.Final.TargetSegmentID, repair.Final.OriginalText, repair.Final.CorrectedText
|
||||
entry.AppliedSegmentID = &segment
|
||||
entry.AppliedOriginalText = &before
|
||||
entry.AppliedCorrectedText = &after
|
||||
} else if skipped, ok := repairSkips[identity]; ok && repair.Outcome == "approved_skipped" {
|
||||
entry.DispositionReasonCode = string(skipped.SkipReason)
|
||||
entry.DispositionMessage = skipped.Message
|
||||
}
|
||||
seen[identity] = true
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
for _, candidate := range runOutput.Candidates {
|
||||
if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] {
|
||||
for identity, item := range evidence {
|
||||
module := modules[identity.instance]
|
||||
if module == nil {
|
||||
continue
|
||||
}
|
||||
stage := stages[candidate.ModuleInstance]
|
||||
decisions := stage.decisions[candidate.ProposalIndex]
|
||||
entries = append(entries, CorrectionLedgerEntry{
|
||||
StageIndex: stage.index, StageID: stage.id, RunID: runID, ModuleKey: candidate.ModuleKey, ModuleInstance: candidate.ModuleInstance,
|
||||
ProposalIndex: candidate.ProposalIndex, SegmentID: candidate.TargetSegmentID,
|
||||
OriginalText: candidate.OriginalText, ProposedCorrectedText: &candidate.CorrectedText,
|
||||
ReplacementPolicy: stage.policy, Disposition: correctionDispositionInterrupted,
|
||||
DispositionReasonCode: "not_applied", DispositionMessage: "candidate did not reach application",
|
||||
DeterministicValidatorResults: filterLedgerDecisions(decisions, false),
|
||||
LLMValidatorResults: filterLedgerDecisions(decisions, true),
|
||||
})
|
||||
entry := CorrectionLedgerEntry{
|
||||
StageIndex: module.StageIndex, StageID: module.StageID, RunID: runID,
|
||||
ModuleKey: module.ModuleKey, ModuleInstance: identity.instance, ProposalIndex: identity.index,
|
||||
ReplacementPolicy: string(module.ReplacementPolicy),
|
||||
DeterministicValidatorResults: item.deterministic, LLMValidatorResults: item.llm,
|
||||
}
|
||||
if item.project(&entry) {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
}
|
||||
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
if entries[i].StageIndex != entries[j].StageIndex {
|
||||
return entries[i].StageIndex < entries[j].StageIndex
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
a, b := entries[i], entries[j]
|
||||
if a.StageIndex != b.StageIndex {
|
||||
return a.StageIndex < b.StageIndex
|
||||
}
|
||||
if entries[i].ProposalIndex != entries[j].ProposalIndex {
|
||||
return entries[i].ProposalIndex < entries[j].ProposalIndex
|
||||
if a.ProposalIndex != b.ProposalIndex {
|
||||
return a.ProposalIndex < b.ProposalIndex
|
||||
}
|
||||
return entries[i].Disposition < entries[j].Disposition
|
||||
if a.Disposition != b.Disposition {
|
||||
return a.Disposition < b.Disposition
|
||||
}
|
||||
return a.ModuleInstance < b.ModuleInstance
|
||||
})
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []LedgerValidatorDecisionRecord {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]LedgerValidatorDecisionRecord, 0, len(in))
|
||||
for _, decision := range in {
|
||||
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
|
||||
if isLLMBacked != wantLLM {
|
||||
continue
|
||||
func (item candidateEvidence) project(entry *CorrectionLedgerEntry) bool {
|
||||
switch {
|
||||
case item.repair != nil:
|
||||
repair := item.repair
|
||||
entry.SegmentID, entry.OriginalText = repair.Original.TargetSegmentID, repair.Original.OriginalText
|
||||
entry.ProposedCorrectedText = &repair.Original.CorrectedText
|
||||
entry.Disposition = repair.Outcome
|
||||
for i, attempt := range repair.Attempts {
|
||||
record := LedgerRepairAttempt{
|
||||
Attempt: i + 1, Revision: attempt.Revision, FailureReason: string(attempt.FailureReason),
|
||||
RequestMetadataPath: attempt.Artifacts.RequestMetadataPath, RequestPayloadPath: attempt.Artifacts.RequestPayloadPath,
|
||||
ResponsePayloadPath: attempt.Artifacts.ResponsePayloadPath, ErrorPayloadPath: attempt.Artifacts.ErrorPayloadPath,
|
||||
}
|
||||
for _, decision := range attempt.Decisions {
|
||||
recordLedgerDecision(decision, &record.DeterministicValidatorResults, &record.LLMValidatorResults)
|
||||
}
|
||||
entry.RepairAttempts = append(entry.RepairAttempts, record)
|
||||
}
|
||||
out = append(out, LedgerValidatorDecisionRecord{
|
||||
ArtifactPath: decision.DiagnosticArtifactPath,
|
||||
ValidatorKey: decision.ValidatorName,
|
||||
Approved: decision.Approved,
|
||||
Disposition: decision.Disposition,
|
||||
ReasonCode: decision.ReasonCode,
|
||||
Message: decision.Message,
|
||||
})
|
||||
if repair.Outcome == correctionDispositionApplied {
|
||||
entry.AppliedSegmentID, entry.AppliedOriginalText = &repair.Final.TargetSegmentID, &repair.Final.OriginalText
|
||||
entry.AppliedCorrectedText = &repair.Final.CorrectedText
|
||||
if applied := item.applied; applied != nil {
|
||||
entry.AppliedSegmentID, entry.AppliedOriginalText = &applied.TargetSegmentID, &applied.OriginalText
|
||||
entry.AppliedCorrectedText = &applied.CorrectedText
|
||||
}
|
||||
} else if skip := item.skipped; skip != nil && repair.Outcome == "approved_skipped" {
|
||||
entry.DispositionReasonCode, entry.DispositionMessage = string(skip.SkipReason), skip.Message
|
||||
}
|
||||
case item.applied != nil:
|
||||
change := item.applied
|
||||
entry.SegmentID, entry.OriginalText, entry.ProposedCorrectedText = change.TargetSegmentID, change.OriginalText, &change.CorrectedText
|
||||
entry.Disposition, entry.AppliedCorrectedText = correctionDispositionApplied, &change.CorrectedText
|
||||
case item.skipped != nil:
|
||||
change := item.skipped
|
||||
entry.SegmentID, entry.OriginalText, entry.ProposedCorrectedText = change.TargetSegmentID, change.OriginalText, &change.CorrectedText
|
||||
entry.Disposition, entry.DispositionReasonCode, entry.DispositionMessage = correctionDispositionSkipped, string(change.SkipReason), change.Message
|
||||
case item.rejection != nil:
|
||||
rejection := item.rejection
|
||||
entry.SegmentID, entry.OriginalText, entry.ProposedCorrectedText = rejection.TargetSegmentID, rejection.OriginalText, &rejection.CorrectedText
|
||||
entry.Disposition, entry.DispositionReasonCode, entry.DispositionMessage = correctionDispositionRejected, rejection.ReasonCode, rejection.Message
|
||||
if rejection.Disposition == validators.DispositionUnavailable {
|
||||
entry.Disposition = correctionDispositionUnavailable
|
||||
}
|
||||
case item.candidate != nil:
|
||||
candidate := item.candidate
|
||||
entry.SegmentID, entry.OriginalText, entry.ProposedCorrectedText = candidate.TargetSegmentID, candidate.OriginalText, &candidate.CorrectedText
|
||||
entry.Disposition, entry.DispositionReasonCode, entry.DispositionMessage = correctionDispositionInterrupted, "not_applied", "candidate did not reach application"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func recordLedgerDecision(decision runner.ValidatorDecisionRecord, deterministic, llm *[]LedgerValidatorDecisionRecord) {
|
||||
record := LedgerValidatorDecisionRecord{
|
||||
ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName,
|
||||
Approved: decision.Approved, Disposition: decision.Disposition, ReasonCode: decision.ReasonCode, Message: decision.Message,
|
||||
}
|
||||
if decision.ExecutionClass == validatormetadata.ExecutionClassLLMBacked {
|
||||
*llm = append(*llm, record)
|
||||
} else {
|
||||
*deterministic = append(*deterministic, record)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -7,9 +7,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
|
||||
func TestBuildCorrectionLedgerUsesRecordedValidatorExecutionClass(t *testing.T) {
|
||||
output := &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{
|
||||
@@ -17,8 +18,8 @@ func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(
|
||||
ModuleInstance: "glossary",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, Disposition: validators.DispositionApproved, ReasonCode: "approved"},
|
||||
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, Disposition: validators.DispositionApproved, ReasonCode: "approved"},
|
||||
{ExecutionClass: validatormetadata.ExecutionClassLLMBacked, ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, Disposition: validators.DispositionApproved, ReasonCode: "approved"},
|
||||
{ExecutionClass: validatormetadata.ExecutionClassDeterministic, ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, Disposition: validators.DispositionApproved, ReasonCode: "approved"},
|
||||
},
|
||||
AppliedChanges: []proposals.AppliedChange{
|
||||
{
|
||||
@@ -45,10 +46,10 @@ func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(
|
||||
if entry.RunID != "audita-run-id" || entry.Disposition != "applied" || entry.AppliedCorrectedText == nil || *entry.AppliedCorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected applied ledger entry: %+v", entry)
|
||||
}
|
||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" || entry.DeterministicValidatorResults[0].Disposition != validators.DispositionApproved {
|
||||
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "spoken_form_plausibility" || entry.DeterministicValidatorResults[0].Disposition != validators.DispositionApproved {
|
||||
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
|
||||
}
|
||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" || entry.LLMValidatorResults[0].Disposition != validators.DispositionApproved {
|
||||
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "proposal_shape" || entry.LLMValidatorResults[0].Disposition != validators.DispositionApproved {
|
||||
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
|
||||
}
|
||||
}
|
||||
@@ -125,10 +126,10 @@ func TestCorrectionLedgerIncludesUnappliedCandidateAfterFatalValidation(t *testi
|
||||
ModuleResults: []runner.ModuleResult{
|
||||
{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess,
|
||||
AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 2, TargetSegmentID: 7, OriginalText: "before", CorrectedText: "old"}},
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "editorial_review", ProposalIndex: 2, Approved: true, Disposition: validators.DispositionApproved}}},
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{{ExecutionClass: validatormetadata.ExecutionClassLLMBacked, ValidatorName: "editorial_review", ProposalIndex: 2, Approved: true, Disposition: validators.DispositionApproved}}},
|
||||
{StageIndex: 1, StageID: &stageID, ModuleKey: "grammar", ModuleInstance: "grammar_2",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Status: runner.ModuleStatusFailed,
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "spoken_form_plausibility", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "transport_failed"}}},
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{{ExecutionClass: validatormetadata.ExecutionClassLLMBacked, ValidatorName: "spoken_form_plausibility", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "transport_failed"}}},
|
||||
},
|
||||
}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
@@ -158,8 +159,8 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
|
||||
{ProposalIndex: 4, TargetSegmentID: 5, OriginalText: "uh ", CorrectedText: "", Disposition: validators.DispositionUnavailable, ReasonCode: validators.ReasonMissingValidationDecision},
|
||||
},
|
||||
ValidatorDecisions: []runner.ValidatorDecisionRecord{
|
||||
{ValidatorName: "editorial_review", ProposalIndex: 1, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected"},
|
||||
{ValidatorName: "editorial_review", ProposalIndex: 4, Disposition: validators.DispositionUnavailable, ReasonCode: validators.ReasonMissingValidationDecision},
|
||||
{ExecutionClass: validatormetadata.ExecutionClassLLMBacked, ValidatorName: "editorial_review", ProposalIndex: 1, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected"},
|
||||
{ExecutionClass: validatormetadata.ExecutionClassLLMBacked, ValidatorName: "editorial_review", ProposalIndex: 4, Disposition: validators.DispositionUnavailable, ReasonCode: validators.ReasonMissingValidationDecision},
|
||||
},
|
||||
SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 2, TargetSegmentID: 3, OriginalText: "uh ", CorrectedText: "", SkipReason: proposals.SkipReasonEmptyResultingText}},
|
||||
},
|
||||
@@ -215,7 +216,7 @@ func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
|
||||
revised.TargetSegmentID = 2
|
||||
revised.OriginalText = "uh "
|
||||
revised.CorrectedText = ""
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "original_text_presence", ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "llm_rejected"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "original_text_presence", ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "llm_rejected"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ExecutionClass: validatormetadata.ExecutionClassLLMBacked, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
if len(ledger) != 1 {
|
||||
t.Fatalf("ledger: %+v", ledger)
|
||||
@@ -258,7 +259,7 @@ func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
|
||||
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
|
||||
revision := original
|
||||
revision.CorrectedText = "fox"
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "original_text_presence", ProposalIndex: 1, ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "original_text_presence", ProposalIndex: 1, ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ExecutionClass: validatormetadata.ExecutionClassLLMBacked, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
var found *CorrectionLedgerEntry
|
||||
for i := range ledger {
|
||||
|
||||
64
internal/framework/processreport/failures.go
Normal file
64
internal/framework/processreport/failures.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
func processingFailure(cause error, phase string) RunFailure {
|
||||
failure := RunFailure{Phase: phase, Reason: outcomes.ReasonInternalFailure}
|
||||
var execution *runner.ExecutionFailure
|
||||
if errors.As(cause, &execution) {
|
||||
failure.Phase = execution.Phase
|
||||
failure.Origin = execution.Origin
|
||||
failure.HasSection = execution.HasSection
|
||||
}
|
||||
var model *structuredoutput.Failure
|
||||
var infeasible *chunking.InfeasibleTargetError
|
||||
switch {
|
||||
case errors.As(cause, &model) && model.Reason != "":
|
||||
failure.Reason = model.Reason
|
||||
case errors.Is(cause, context.Canceled), errors.Is(cause, context.DeadlineExceeded):
|
||||
failure.Reason = outcomes.ReasonCanceled
|
||||
case errors.As(cause, &infeasible):
|
||||
failure.Reason = outcomes.ReasonInputInvalid
|
||||
case failure.Phase == "setup":
|
||||
failure.Reason = outcomes.ReasonSetupFailure
|
||||
}
|
||||
return failure
|
||||
}
|
||||
|
||||
// Failed publishes the primary processing failure once and adds delivery failures
|
||||
// observed separately by the output session.
|
||||
func Failed(snapshot reporting.Snapshot, primary error, phase string, files []diagnostics.FileFailure) reporting.Snapshot {
|
||||
snapshot.Status = reporting.StatusFailed
|
||||
snapshot.Errors = slices.Clone(snapshot.Errors)
|
||||
count := 0
|
||||
for _, record := range snapshot.Errors {
|
||||
if record.Scope == "run" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if primary != nil && count == 0 {
|
||||
snapshot.Errors = append(snapshot.Errors, failureRecord(processingFailure(primary, phase)))
|
||||
count++
|
||||
}
|
||||
for _, file := range files {
|
||||
record := failureRecord(RunFailure{Phase: string(file.Phase), Reason: "output_failure", OutputPath: file.Path})
|
||||
if file.Phase == diagnostics.OutputInitialization {
|
||||
record.Message = "Required report initialization failed."
|
||||
}
|
||||
snapshot.Errors = append(snapshot.Errors, record)
|
||||
count++
|
||||
}
|
||||
snapshot.Summary.RunErrors = &count
|
||||
return snapshot
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
@@ -17,19 +18,21 @@ type RunFailure struct {
|
||||
OutputPath string
|
||||
Origin *outcomes.Reference
|
||||
StageIndex *int
|
||||
HasSection bool
|
||||
}
|
||||
|
||||
type BuildInput struct {
|
||||
RunID string
|
||||
Status string
|
||||
StartedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
TranscriptPath string
|
||||
OutputPath string
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
RunOutput *runner.RunOutput
|
||||
RunFailures []RunFailure
|
||||
SummaryOnly bool
|
||||
RunID string
|
||||
Status string
|
||||
StartedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
TranscriptPath string
|
||||
OutputPath string
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
RunOutput *runner.RunOutput
|
||||
ProcessingError error
|
||||
RunFailures []RunFailure
|
||||
SummaryOnly bool
|
||||
}
|
||||
|
||||
// Build maps processing facts once into an immutable presentation snapshot.
|
||||
@@ -38,6 +41,10 @@ func Build(input BuildInput) reporting.Snapshot {
|
||||
RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt,
|
||||
TranscriptPath: input.TranscriptPath, OutputPath: input.OutputPath,
|
||||
}
|
||||
failures := input.RunFailures
|
||||
if input.ProcessingError != nil {
|
||||
failures = append([]RunFailure{processingFailure(input.ProcessingError, "processing")}, failures...)
|
||||
}
|
||||
details := !input.SummaryOnly
|
||||
if details {
|
||||
snapshot.StageNames = make([]string, len(input.ModuleSpecs))
|
||||
@@ -99,7 +106,7 @@ func Build(input BuildInput) reporting.Snapshot {
|
||||
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
|
||||
}
|
||||
}
|
||||
if operation.FailureReason != "" && isRecoverableReason(operation.FailureReason) {
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
unitHasError = true
|
||||
if details {
|
||||
failures = append(failures, mapFailure(operation))
|
||||
@@ -122,6 +129,11 @@ func Build(input BuildInput) reporting.Snapshot {
|
||||
ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, ChunkIndex: &chunk,
|
||||
StartSegmentID: &start, EndSegmentID: &end, CorrectionsApplied: &applied, Failures: failures})
|
||||
}
|
||||
if unit.Failure != nil && !errors.Is(input.ProcessingError, unit.Failure) {
|
||||
record := failureRecord(processingFailure(unit.Failure, "processing"))
|
||||
record.Scope = "interrupted"
|
||||
snapshot.Errors = append(snapshot.Errors, record)
|
||||
}
|
||||
snapshot.WorkUnits = append(snapshot.WorkUnits, row)
|
||||
for _, repair := range unit.Repairs {
|
||||
if repair.AttemptCount == 0 {
|
||||
@@ -140,7 +152,7 @@ func Build(input BuildInput) reporting.Snapshot {
|
||||
}
|
||||
return a.ProposalIndex < b.ProposalIndex
|
||||
})
|
||||
runErrors := len(input.RunFailures)
|
||||
runErrors := len(failures)
|
||||
snapshot.Summary = reporting.Summary{
|
||||
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
|
||||
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &withErrors,
|
||||
@@ -148,32 +160,38 @@ func Build(input BuildInput) reporting.Snapshot {
|
||||
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount,
|
||||
ProposalResponsesWithFiltering: &filteringResponses,
|
||||
}
|
||||
} else if len(input.RunFailures) > 0 {
|
||||
runErrors := len(input.RunFailures)
|
||||
} else if len(failures) > 0 {
|
||||
runErrors := len(failures)
|
||||
snapshot.Summary.RunErrors = &runErrors
|
||||
}
|
||||
for _, failure := range sortedRunFailures(input.RunFailures) {
|
||||
record := reporting.ErrorRecord{Scope: "run", Phase: failure.Phase, Reason: string(failure.Reason),
|
||||
Message: safeMessage(failure.Reason), OutputPath: failure.OutputPath, StageIndex: failure.StageIndex}
|
||||
if failure.Origin != nil {
|
||||
record.ModuleKey, record.ModuleInstance = failure.Origin.ModuleKey, failure.Origin.ModuleInstance
|
||||
record.StageID = failure.Origin.StageID
|
||||
if record.StageIndex == nil {
|
||||
record.StageIndex = failure.Origin.StageIndex
|
||||
}
|
||||
if failure.Origin.Kind != "" {
|
||||
chunk := failure.Origin.SectionIndex
|
||||
record.ChunkIndex = &chunk
|
||||
}
|
||||
record.Operation = string(failure.Origin.Kind)
|
||||
record.Validator = failure.Origin.ValidatorName
|
||||
record.BatchIndex = failure.Origin.BatchIndex
|
||||
}
|
||||
snapshot.Errors = append(snapshot.Errors, record)
|
||||
for _, failure := range sortedRunFailures(failures) {
|
||||
snapshot.Errors = append(snapshot.Errors, failureRecord(failure))
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func failureRecord(failure RunFailure) reporting.ErrorRecord {
|
||||
record := reporting.ErrorRecord{Scope: "run", Phase: failure.Phase, Reason: string(failure.Reason),
|
||||
Message: safeMessage(failure.Reason), OutputPath: failure.OutputPath, StageIndex: failure.StageIndex}
|
||||
if failure.Origin != nil {
|
||||
record.ModuleKey, record.ModuleInstance = failure.Origin.ModuleKey, failure.Origin.ModuleInstance
|
||||
record.StageID = failure.Origin.StageID
|
||||
if record.StageIndex == nil {
|
||||
record.StageIndex = failure.Origin.StageIndex
|
||||
}
|
||||
if failure.HasSection {
|
||||
chunk := failure.Origin.SectionIndex
|
||||
record.ChunkIndex = &chunk
|
||||
}
|
||||
record.Operation = string(failure.Origin.Kind)
|
||||
record.Validator = failure.Origin.ValidatorName
|
||||
record.BatchIndex = failure.Origin.BatchIndex
|
||||
record.ProposalIndex = failure.Origin.ProposalIndex
|
||||
record.RepairAttempt = failure.Origin.RepairAttempt
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func mapFailure(operation outcomes.Operation) reporting.Failure {
|
||||
indices := append([]int(nil), operation.ProposalIndices...)
|
||||
sort.Ints(indices)
|
||||
@@ -209,18 +227,6 @@ func mapCounts(counts outcomes.ExecutionCounts) reporting.ExecutionCounts {
|
||||
StructuralRepairs: &repairs, RetriesExhausted: &exhausted, HTTPStatus: counts.HTTPStatus}
|
||||
}
|
||||
|
||||
func isRecoverableReason(reason outcomes.ReasonCode) bool {
|
||||
switch reason {
|
||||
case outcomes.ReasonOperationTimeout, outcomes.ReasonCapacityExceeded, outcomes.ReasonTransportFailed,
|
||||
outcomes.ReasonProviderResponseInvalid, outcomes.ReasonProviderFailed,
|
||||
outcomes.ReasonInvalidOutputExhausted, outcomes.ReasonInvalidOutput,
|
||||
outcomes.ReasonValidationInputTooLarge, outcomes.ReasonValidationDecisionsInvalid:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func safeMessage(reason outcomes.ReasonCode) string {
|
||||
switch reason {
|
||||
case outcomes.ReasonOperationTimeout:
|
||||
@@ -247,6 +253,8 @@ func safeMessage(reason outcomes.ReasonCode) string {
|
||||
return "Processing setup failed."
|
||||
case outcomes.ReasonInputInvalid:
|
||||
return "Input was invalid."
|
||||
case "output_failure":
|
||||
return "Required output delivery failed."
|
||||
default:
|
||||
return "Processing failed."
|
||||
}
|
||||
@@ -296,7 +304,7 @@ func operationLess(a, b outcomes.Operation) bool {
|
||||
|
||||
func sortedRunFailures(in []RunFailure) []RunFailure {
|
||||
out := append([]RunFailure(nil), in...)
|
||||
phase := map[string]int{"configuration": 0, "destination": 1, "initialization": 2, "input": 3, "normalization": 4, "chunking": 5, "processing": 6, "encoding": 7, "transcript_write": 8, "report_write": 9, "debug": 10}
|
||||
phase := map[string]int{"configuration": 0, "destination": 1, "initialization": 2, "input": 3, "normalization": 4, "chunking": 5, "processing": 6, "setup": 6, "proposal": 6, "validation": 6, "correction_repair": 6, "application": 6, "encoding": 7, "transcript_write": 8, "report_write": 9, "debug": 10}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
a, aok := phase[out[i].Phase]
|
||||
b, bok := phase[out[j].Phase]
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package processreport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
@@ -22,8 +25,8 @@ func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
|
||||
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered, outcomes.RecoveryOutputRepaired, outcomes.RecoveryProposalsFiltered},
|
||||
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 4, Reason: outcomes.ReasonInvalidOutput, Count: 1}, {ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}}
|
||||
batch := 1
|
||||
validation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "review", ValidatorOrder: intPtr(1), BatchIndex: &batch}, FailureReason: outcomes.ReasonTransportFailed, ProposalIndices: []int{9, 3}}
|
||||
inputFailure := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidationInput, ValidatorName: "review", ValidatorOrder: intPtr(1)}, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: []int{7}}
|
||||
validation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "review", ValidatorOrder: intPtr(1), BatchIndex: &batch}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonTransportFailed, ProposalIndices: []int{9, 3}}
|
||||
inputFailure := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationValidationInput, ValidatorName: "review", ValidatorOrder: intPtr(1)}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: []int{7}}
|
||||
output := &runner.RunOutput{InventoryAvailable: true, WorkUnits: []runner.WorkUnitOutcome{
|
||||
{ModuleIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkNotStarted},
|
||||
{ModuleIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 2}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{validation, proposal, inputFailure}},
|
||||
@@ -72,7 +75,7 @@ func TestBuildReportsUnavailableDecisionsAndFilteredExtras(t *testing.T) {
|
||||
ModuleKey: "grammar", ModuleInstance: "grammar", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 1},
|
||||
State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &failedBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, FailureReason: outcomes.ReasonValidationDecisionsInvalid, ProposalIndices: []int{3, 1}},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &failedBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonValidationDecisionsInvalid, ProposalIndices: []int{3, 1}},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &cleanBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryValidationDecisionsFiltered}},
|
||||
},
|
||||
}}}
|
||||
@@ -145,7 +148,7 @@ func TestRepairOperationsAndCompactOutcomes(t *testing.T) {
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ProposalIndex: &two, RepairAttempt: &one, ValidatorOrder: &zero}, Succeeded: true},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &two, RepairAttempt: &one}, Succeeded: true, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired, outcomes.RecoveryCorrectionRevised}},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorOrder: &zero}, Succeeded: true},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationCorrectionRepair, ProposalIndex: &one, RepairAttempt: &one}, FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonProviderFailed, ProposalIndices: []int{1}},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
|
||||
}
|
||||
units := []runner.WorkUnitOutcome{{ModuleIndex: 0, Section: contracts.SectionMetadata{Index: 0}, State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: operations}}
|
||||
@@ -189,3 +192,63 @@ func TestRepairOperationOrderingKeepsAttemptsTogether(t *testing.T) {
|
||||
t.Fatalf("order: %+v", operations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportsDistinguishFatalPeerAndDeliveryFailures(t *testing.T) {
|
||||
defect := errors.New("private provider detail")
|
||||
origin := outcomes.Reference{StageIndex: intPtr(1), ModuleKey: "grammar", ModuleInstance: "grammar_2",
|
||||
SectionIndex: 2, Kind: outcomes.OperationValidation, ValidatorName: "review", BatchIndex: intPtr(0),
|
||||
ProposalIndex: intPtr(3), RepairAttempt: intPtr(1)}
|
||||
primary := &runner.ExecutionFailure{Phase: "validation", Origin: &origin, HasSection: true, Cause: defect}
|
||||
peerOrigin := origin
|
||||
peerOrigin.SectionIndex, peerOrigin.ProposalIndex, peerOrigin.RepairAttempt = 3, nil, nil
|
||||
peer := &runner.ExecutionFailure{Phase: "validation", Origin: &peerOrigin, HasSection: true, Cause: context.Canceled}
|
||||
work := []runner.WorkUnitOutcome{
|
||||
{ModuleIndex: 0, ModuleInstance: "grammar_1", State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true,
|
||||
RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryProposalsFiltered},
|
||||
Filtered: []outcomes.FilteredProposal{{ProposalIndex: 2, Reason: outcomes.ReasonInvalidOutput, Count: 1}}},
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureClass: outcomes.FailureRecoverable,
|
||||
FailureReason: outcomes.ReasonValidationDecisionsInvalid, ProposalIndices: []int{4},
|
||||
Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}},
|
||||
}},
|
||||
{ModuleIndex: 1, ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 2}, State: outcomes.WorkInterrupted, Failure: primary, Operations: []outcomes.Operation{
|
||||
{Reference: origin, FailureClass: outcomes.FailureFatal, Cause: defect, FailureReason: outcomes.ReasonProviderFailed},
|
||||
}},
|
||||
{ModuleIndex: 1, ModuleInstance: "grammar_2", Section: contracts.SectionMetadata{Index: 3}, State: outcomes.WorkInterrupted, Failure: peer},
|
||||
}
|
||||
snapshot := Build(BuildInput{
|
||||
RunID: "mixed", StartedAt: time.Unix(1, 0),
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar"}, {Index: 1, ModuleKey: "grammar"}},
|
||||
RunOutput: &runner.RunOutput{InventoryAvailable: true, WorkUnits: work}, ProcessingError: primary,
|
||||
})
|
||||
snapshot = Failed(snapshot, primary, "processing", []diagnostics.FileFailure{
|
||||
{Phase: diagnostics.OutputReportWrite, Path: "recoveries.yml", Err: errors.New("disk failure")},
|
||||
})
|
||||
raw, err := reporting.YAML(snapshot.ErrorsDocument())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc reporting.ErrorsDocument
|
||||
if err := yaml.Unmarshal(raw, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if doc.Status != reporting.StatusFailed || *doc.Summary.WorkUnitsWithErrors != 1 || *doc.Summary.RunErrors != 2 ||
|
||||
*doc.Summary.WorkUnitsInterrupted != 2 || *doc.Summary.Recoveries != 1 || len(doc.Errors) != 4 {
|
||||
t.Fatalf("unexpected report: %s", raw)
|
||||
}
|
||||
unit, secondary, initiating, delivery := doc.Errors[0], doc.Errors[1], doc.Errors[2], doc.Errors[3]
|
||||
if unit.Scope != "work_unit" || len(unit.Failures) != 1 || !reflect.DeepEqual(unit.Failures[0].ProposalIndices, []int{4}) ||
|
||||
unit.Failures[0].ProviderAttempts == nil || *unit.Failures[0].ProviderAttempts != 1 ||
|
||||
secondary.Scope != "interrupted" || !reflect.DeepEqual(secondary.ChunkIndex, intPtr(3)) || secondary.Reason != "canceled" ||
|
||||
initiating.Scope != "run" || initiating.Phase != "validation" || initiating.Validator != "review" ||
|
||||
!reflect.DeepEqual(initiating.ChunkIndex, intPtr(2)) || !reflect.DeepEqual(initiating.ProposalIndex, intPtr(3)) ||
|
||||
!reflect.DeepEqual(initiating.RepairAttempt, intPtr(1)) ||
|
||||
delivery.Scope != "run" || delivery.Phase != "report_write" || delivery.OutputPath != "recoveries.yml" ||
|
||||
strings.Contains(string(raw), defect.Error()) {
|
||||
t.Fatalf("failure facts were merged or lost: %s", raw)
|
||||
}
|
||||
if markdown := string(reporting.Markdown(snapshot)); !strings.Contains(markdown, "Errors: 1 work units; 2 run errors") ||
|
||||
!strings.Contains(markdown, "Recoveries: 1") {
|
||||
t.Fatalf("Markdown totals differ from YAML: %s", markdown)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
@@ -844,46 +845,42 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
// mergeFilteredProposals attributes defective candidates to their originating
|
||||
// proposal response. Each response has at most one filtering recovery reason.
|
||||
func mergeFilteredProposals(unit *WorkUnitOutcome, filtered []outcomes.FilteredProposal) {
|
||||
if unit == nil || len(filtered) == 0 {
|
||||
if len(filtered) == 0 {
|
||||
return
|
||||
}
|
||||
for i := range unit.Operations {
|
||||
operation := &unit.Operations[i]
|
||||
if operation.Reference.Kind != outcomes.OperationProposal || !operation.Succeeded {
|
||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||
mergeOperationFiltering(operation, filtered)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mergeOperationFiltering(operation *outcomes.Operation, filtered []outcomes.FilteredProposal) {
|
||||
seen := make(map[int]struct{}, len(operation.Filtered)+len(filtered))
|
||||
for _, existing := range operation.Filtered {
|
||||
seen[existing.ProposalIndex] = struct{}{}
|
||||
}
|
||||
for _, item := range filtered {
|
||||
if _, ok := seen[item.ProposalIndex]; ok {
|
||||
continue
|
||||
}
|
||||
seen := make(map[int]struct{}, len(operation.Filtered))
|
||||
for _, existing := range operation.Filtered {
|
||||
seen[existing.ProposalIndex] = struct{}{}
|
||||
}
|
||||
for _, item := range filtered {
|
||||
if _, ok := seen[item.ProposalIndex]; ok {
|
||||
continue
|
||||
}
|
||||
seen[item.ProposalIndex] = struct{}{}
|
||||
operation.Filtered = append(operation.Filtered, item)
|
||||
}
|
||||
if len(operation.Filtered) == 0 {
|
||||
continue
|
||||
}
|
||||
sort.Slice(operation.Filtered, func(a, b int) bool {
|
||||
return operation.Filtered[a].ProposalIndex < operation.Filtered[b].ProposalIndex
|
||||
})
|
||||
found := false
|
||||
for _, reason := range operation.RecoveryReasons {
|
||||
if reason == outcomes.RecoveryProposalsFiltered {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
|
||||
}
|
||||
return
|
||||
seen[item.ProposalIndex] = struct{}{}
|
||||
operation.Filtered = append(operation.Filtered, item)
|
||||
}
|
||||
sort.Slice(operation.Filtered, func(a, b int) bool {
|
||||
return operation.Filtered[a].ProposalIndex < operation.Filtered[b].ProposalIndex
|
||||
})
|
||||
if !slices.Contains(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered) {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
|
||||
}
|
||||
}
|
||||
|
||||
func attributeApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy, units []WorkUnitOutcome) {
|
||||
if len(skipped) == 0 {
|
||||
return
|
||||
}
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved))
|
||||
sectionPosition := make(map[int]int, len(sections))
|
||||
for pos, section := range sections {
|
||||
@@ -892,30 +889,52 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
|
||||
for _, proposal := range approved {
|
||||
byIndex[proposal.ProposalIndex] = proposal
|
||||
}
|
||||
lookups := make(map[int]*validators.SectionLookup)
|
||||
filtered := make(map[int][]outcomes.FilteredProposal)
|
||||
for _, skip := range skipped {
|
||||
reason, defective := validators.DefectiveProposalReason(string(skip.SkipReason))
|
||||
proposal, ok := byIndex[skip.ProposalIndex]
|
||||
if !ok || proposal.SectionIndex == nil {
|
||||
if !defective || !ok || proposal.SectionIndex == nil {
|
||||
continue
|
||||
}
|
||||
pos, ok := sectionPosition[*proposal.SectionIndex]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var segment *schema.Segment
|
||||
for i := range sections[pos].Segments {
|
||||
if sections[pos].Segments[i].ID == proposal.TargetSegmentID {
|
||||
segment = §ions[pos].Segments[i]
|
||||
break
|
||||
}
|
||||
lookup := lookups[pos]
|
||||
if lookup == nil {
|
||||
lookup = validators.NewSectionLookup(transcriptFromSection(sections[pos]))
|
||||
lookups[pos] = lookup
|
||||
}
|
||||
// A proposal applicable to the input was made stale by a previously
|
||||
// accepted correction. That conflict is not a response defect.
|
||||
var segment *schema.Segment
|
||||
if i, found := lookup.Position(proposal.TargetSegmentID); found {
|
||||
segment = §ions[pos].Segments[i]
|
||||
}
|
||||
// A previously applicable proposal can become stale during application.
|
||||
if proposals.PreviewProposalForSegment(segment, proposal.CorrectionProposal, policy).Applicable {
|
||||
continue
|
||||
}
|
||||
if reason, ok := validators.DefectiveProposalReason(string(skip.SkipReason)); ok {
|
||||
mergeFilteredProposals(&units[pos], []outcomes.FilteredProposal{{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1}})
|
||||
filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1})
|
||||
}
|
||||
for pos, items := range filtered {
|
||||
unit := &units[pos]
|
||||
revisions := make(map[int]*outcomes.Operation)
|
||||
for i := range unit.Operations {
|
||||
op := &unit.Operations[i]
|
||||
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Succeeded && op.Reference.ProposalIndex != nil &&
|
||||
slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
|
||||
revisions[*op.Reference.ProposalIndex] = op
|
||||
}
|
||||
}
|
||||
original := items[:0]
|
||||
for _, item := range items {
|
||||
if op := revisions[item.ProposalIndex]; op != nil {
|
||||
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
|
||||
} else {
|
||||
original = append(original, item)
|
||||
}
|
||||
}
|
||||
mergeFilteredProposals(unit, original)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user