From 1c43264f6f065d9fa5a88fc55144550ab3b223f0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 14:42:24 +0000 Subject: [PATCH] Commit module changes only after successful execution --- docs/internal/pipeline.md | 11 +- docs/roadmap/implementation.md | 9 + .../framework/runner/cancellation_test.go | 107 +++++ internal/framework/runner/failure.go | 51 +++ internal/framework/runner/repair.go | 7 + internal/framework/runner/runner.go | 383 +++++++++--------- internal/framework/runner/runner_test.go | 11 +- 7 files changed, 388 insertions(+), 191 deletions(-) create mode 100644 internal/framework/runner/failure.go diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index d98fef5..09b5fca 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -40,9 +40,14 @@ explicit `not_started`/`interrupted`/`completed` work units, compact model operation outcomes, and per-module results. Candidate payloads are retained in the returned output only when debug capture is requested. A work unit can complete with model errors and applied corrections. Recoverable model failures -never trigger shared cancellation. Fatal setup, internal, application, or -parent-cancellation errors cancel peers, join workers, and preserve earlier -committed module text for partial reports; no completed transcript is published. +never trigger shared cancellation. Fatal failures and parent cancellation stop +further dispatch and join active workers. The initiating error carries its +processing phase and available origin. Work-unit evidence preserves each peer's +interruption or error. Usable proposal results from later sections are indexed +in order after joining, even when the runner could not validate them. The +working transcript advances only at the module's successful commit boundary. +Earlier committed text remains available to an interrupted run, and transcript +delivery requires successful processing. Required transcript/report write failures are fatal. Optional debug writes report safely to stderr without changing decisions or exit status. See diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 66cc41e..17c5801 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -543,6 +543,15 @@ earlier chunk, and a fatal stage after earlier committed text. Assert causal origin and preserved evidence as well as returned failure. Leave report/ledger projection to Stage 17. +**Complete.** A single module completion path advances the transcript only +after a successful application and context check. The runner returns the +initiating error with its phase and available origin, and keeps peer failures +on their work units. After joining, it assigns deterministic indices to usable +proposal responses that never reached validation. Tests exercised canceled +entry, cancellation after approval, a later response behind an earlier +failure, and preservation of earlier committed text. The CLI build, full +offline suite, and targeted race checks passed. + ## Stage 16 — Bound local preparation and actively drain results Audit #15 (dispatch) and #30, under D3. diff --git a/internal/framework/runner/cancellation_test.go b/internal/framework/runner/cancellation_test.go index 203ea86..d391804 100644 --- a/internal/framework/runner/cancellation_test.go +++ b/internal/framework/runner/cancellation_test.go @@ -15,6 +15,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" @@ -128,6 +129,17 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { if len(outcome.result.WorkUnits) != 3 { t.Fatalf("lost work inventory: %+v", outcome.result.WorkUnits) } + var failure *ExecutionFailure + if !errors.As(outcome.err, &failure) || failure.Phase != "validation" || !failure.HasSection || + failure.Origin.SectionIndex != 0 || failure.Origin.ValidatorName != validator.Name() || + outcome.result.WorkUnits[0].Failure != failure { + t.Fatalf("lost primary validation owner: failure=%+v units=%+v", failure, outcome.result.WorkUnits) + } + for _, peer := range outcome.result.WorkUnits[1:] { + if peer.State != outcomes.WorkInterrupted || peer.Failure == nil || !errors.Is(peer.Failure, context.Canceled) || peer.Failure == failure { + t.Fatalf("lost peer interruption: %+v", peer) + } + } if len(outcome.result.Approved) != 0 { t.Fatalf("fatal validation returned unchecked candidates: %+v", outcome.result.Approved) } @@ -150,3 +162,98 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { t.Fatalf("attempt count: %+v", got) } } + +func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out, err := New(fakeFactory{}).Run(ctx, RunInput{ + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original"}}}, + }) + if !errors.Is(err, context.Canceled) || out.FinalTranscript == nil || out.FinalTranscript.Segments[0].Text != "original" { + t.Fatalf("canceled entry: result=%+v error=%v", out, err) + } +} + +func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) { + origin := errors.New("earlier proposal failed") + laterCompleted := make(chan struct{}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var validations atomic.Int32 + module := outcomeModule{ + fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{ + contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) { + validations.Add(1) + return validators.Result{}, errors.New("validation should not be submitted") + }}, + }}, + proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { + if req.Section.Index == 0 { + select { + case <-laterCompleted: + return contracts.ProposalResult{}, origin + case <-ctx.Done(): + return contracts.ProposalResult{}, ctx.Err() + } + } + close(laterCompleted) + return contracts.ProposalResult{ + Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}}, + SegmentReferences: []int{1}, + Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}, + }, nil + }, + } + cfg := runnerTestConfig() + cfg.Chunking.MaxSectionTokens = 3 + cfg.Chunking.MinSectionTokens = 0 + out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ + Config: &cfg, CaptureCandidates: true, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, + }) + if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 { + t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err) + } + var failure *ExecutionFailure + if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection || + failure.Origin.SectionIndex != 0 || failure.Origin.Kind != outcomes.OperationProposal || out.WorkUnits[0].Failure != failure { + t.Fatalf("lost originating proposal failure: %+v", failure) + } + candidate := out.Candidates[0] + later := out.WorkUnits[1] + if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || + len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex || + out.FinalTranscript.Segments[1].Text != "other text" { + t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later) + } +} + +func TestRunnerCancellationAfterApprovalKeepsCommittedModuleText(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + propose := func(original, replacement string) func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: replacement}}, nil + } + } + last := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) { + cancel() + return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil + }} + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ + "first": fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("teh", "the")}, + "second": fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("cat", "dog"), validators: []validators.Validator{last}}, + }}) + out, err := r.Run(ctx, RunInput{ + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {Index: 1, ModuleKey: "second", InstanceName: "second"}}, + }) + var failure *ExecutionFailure + if !errors.Is(err, context.Canceled) || !errors.As(err, &failure) || failure.Origin == nil || *failure.Origin.StageIndex != 1 || + len(out.ModuleResults) != 2 || out.ModuleResults[0].Status != ModuleStatusSuccess || out.ModuleResults[1].Status != ModuleStatusFailed || + len(out.ModuleResults[1].AppliedChanges) != 0 || len(out.ModuleResults[1].ValidatorDecisions) != 1 || + !out.ModuleResults[1].ValidatorDecisions[0].Approved || out.FinalTranscript.Segments[0].Text != "the cat" { + t.Fatalf("commit boundary lost earlier text or approval evidence: result=%+v error=%+v", out, failure) + } +} diff --git a/internal/framework/runner/failure.go b/internal/framework/runner/failure.go new file mode 100644 index 0000000..0167ca8 --- /dev/null +++ b/internal/framework/runner/failure.go @@ -0,0 +1,51 @@ +package runner + +import ( + "errors" + + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" +) + +// ExecutionFailure associates the initiating cause with its processing owner. +// The section is optional, for example when module setup fails. +type ExecutionFailure struct { + Phase string + Origin *outcomes.Reference + HasSection bool + Cause error +} + +func (f *ExecutionFailure) Error() string { return f.Cause.Error() } +func (f *ExecutionFailure) Unwrap() error { return f.Cause } + +func locateFailure(cause error, phase string, origin outcomes.Reference, hasSection bool, operations []outcomes.Operation) *ExecutionFailure { + for _, operation := range operations { + if operation.Cause == nil || !errors.Is(cause, operation.Cause) { + continue + } + ref := operation.Reference + if ref.StageIndex == nil { + ref.StageIndex, ref.StageID = origin.StageIndex, origin.StageID + } + if ref.ProposalIndex == nil { + ref.ProposalIndex = origin.ProposalIndex + } + if ref.RepairAttempt == nil { + ref.RepairAttempt = origin.RepairAttempt + } + return &ExecutionFailure{Phase: phase, Origin: &ref, HasSection: hasSection, Cause: cause} + } + return &ExecutionFailure{Phase: phase, Origin: &origin, HasSection: hasSection, Cause: cause} +} + +func failureReference(spec contracts.ModuleRunSpec, section *int, kind outcomes.OperationKind) outcomes.Reference { + ref := outcomes.Reference{ + StageIndex: &spec.Index, StageID: spec.ID, + ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, Kind: kind, + } + if section != nil { + ref.SectionIndex = *section + } + return ref +} diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index fa69659..d5ab68f 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -2,6 +2,7 @@ package runner import ( "context" + "errors" "fmt" "slices" @@ -278,6 +279,12 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair if coordinationErr != nil { branch.Outcome = "interrupted" result.approved = nil + var failure *ExecutionFailure + if !errors.As(coordinationErr, &failure) { + ref := failureReference(input.Spec, input.SectionIndex, outcomes.OperationCorrectionRepair) + ref.ProposalIndex = &original.ProposalIndex + coordinationErr = locateFailure(coordinationErr, "correction_repair", ref, input.SectionIndex != nil, result.operations) + } } else if exhausted { branch.Outcome = "exhausted" } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index b47c9ce..9421d47 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -63,6 +63,7 @@ type WorkUnitOutcome struct { ModuleInstance string `json:"module_instance"` Section contracts.SectionMetadata `json:"section"` State outcomes.WorkState `json:"state"` + Failure *ExecutionFailure `json:"-"` CorrectionsApplied int `json:"corrections_applied"` Repairs []RepairOutcome `json:"-"` Operations []outcomes.Operation `json:"operations,omitempty"` @@ -116,15 +117,18 @@ func New(factory ModuleFactory) *Runner { } func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { + if err := ctx.Err(); err != nil { + return RunOutput{FinalTranscript: input.Transcript.Clone()}, &ExecutionFailure{Phase: "processing", Cause: err} + } if r == nil || r.factory == nil { - return RunOutput{}, fmt.Errorf("runner module factory is required") + return RunOutput{}, &ExecutionFailure{Phase: "setup", Cause: errors.New("runner module factory is required")} } working := input.Transcript.Clone() results := make([]ModuleResult, 0, len(input.ModuleSpecs)) sections, err := chunkWorkingTranscript(input.Config, working) if err != nil { - return RunOutput{FinalTranscript: working}, fmt.Errorf("chunk normalized transcript: %w", err) + return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)} } inventory := make([]contracts.SectionMetadata, len(sections)) workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections)) @@ -149,135 +153,124 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { } for moduleIndex, spec := range input.ModuleSpecs { - module, err := r.factory.ModuleForSpec(spec) - if err != nil { - failed := ModuleResult{ - StageIndex: spec.Index, StageID: spec.ID, - ModuleKey: spec.ModuleKey, - ModuleInstance: spec.InstanceName, - Status: ModuleStatusFailed, - ErrorMessage: err.Error(), - } - results = append(results, failed) - return snapshot(), fmt.Errorf("module %q setup failed: %w", spec.InstanceName, err) + execution, err := r.executeModule(ctx, spec, sections, input.Interactions) + units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] + copy(units, execution.pipeline.WorkUnits) + if err == nil && ctx.Err() != nil { + err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil) + } + if err == nil { + previous := sections + sections = execution.updated + attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units) } - policy := module.ReplacementPolicy() - if err := spec.PrepareGlossary(); err != nil { - return snapshot(), fmt.Errorf("module %q glossary preparation: %w", spec.InstanceName, err) + var failure *ExecutionFailure + applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application" + appliedIndices := make(map[int]struct{}, len(execution.applied)) + for _, change := range execution.applied { + appliedIndices[change.ProposalIndex] = struct{}{} } - pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{ - Module: module, - Spec: spec, - Policy: policy, - Sections: sections, - ProposalClient: spec.ProposalClient, - Interactions: input.Interactions, - ValidationClient: spec.ValidationClient, - GlossaryVariables: spec.GlossaryVariables, - GlossaryView: spec.GlossaryView, - }) - if input.CaptureCandidates { - candidates = append(candidates, pipelineResult.Candidates...) - } - if pipelineErr == nil && ctx.Err() != nil { - pipelineErr = ctx.Err() - for i := range pipelineResult.WorkUnits { - for j := range pipelineResult.WorkUnits[i].Repairs { - if pipelineResult.WorkUnits[i].Repairs[j].Outcome == "approved" { - pipelineResult.WorkUnits[i].Repairs[j].Outcome = "interrupted" - } - } - } - } - if pipelineErr != nil { - failed := ModuleResult{ - StageIndex: spec.Index, StageID: spec.ID, - ModuleKey: spec.ModuleKey, - ModuleInstance: spec.InstanceName, - ReplacementPolicy: policy, - Status: ModuleStatusFailed, - ProposalCount: pipelineResult.ProposalCount, - Operations: pipelineResult.Operations, - ValidatorDecisions: pipelineResult.ValidatorDecisions, - ValidatorRejected: pipelineResult.ValidatorRejected, - ErrorMessage: pipelineErr.Error(), - } - results = append(results, failed) - copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits) - for sectionPos := range sections { - workUnits[moduleIndex*len(sections)+sectionPos].ModuleIndex = moduleIndex - } - return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr) - } - - updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord) - if applyErr != nil { - for i := range pipelineResult.WorkUnits { - unit := &pipelineResult.WorkUnits[i] - unit.ModuleIndex = moduleIndex - unit.State = outcomes.WorkInterrupted - for j := range unit.Repairs { - if unit.Repairs[j].Outcome == "approved" { - unit.Repairs[j].Outcome = "interrupted" - } - } - } - copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits) - results = append(results, ModuleResult{StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, ReplacementPolicy: policy, Status: ModuleStatusFailed, ProposalCount: pipelineResult.ProposalCount, Operations: pipelineResult.Operations, ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorRejected: pipelineResult.ValidatorRejected, ErrorMessage: applyErr.Error()}) - return snapshot(), fmt.Errorf("module %q application failed: %w", spec.InstanceName, applyErr) - } - copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits) - for sectionPos := range sections { - unit := &workUnits[moduleIndex*len(sections)+sectionPos] + var operations []outcomes.Operation + for i := range units { + unit := &units[i] unit.ModuleIndex = moduleIndex - unit.State = outcomes.WorkCompleted - unit.CorrectionsApplied = appliedBySection[sectionPos] - } - var appliedIndices map[int]struct{} - for i := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] { - unit := &workUnits[moduleIndex*len(sections)+i] + if err == nil { + unit.State = outcomes.WorkCompleted + unit.CorrectionsApplied = execution.appliedBySection[i] + } else if applicationInterrupted && unit.State != outcomes.WorkNotStarted { + unit.State = outcomes.WorkInterrupted + } for j := range unit.Repairs { branch := &unit.Repairs[j] if branch.Outcome != "approved" { continue } - if appliedIndices == nil { - appliedIndices = make(map[int]struct{}, len(applied)) - for _, change := range applied { - appliedIndices[change.ProposalIndex] = struct{}{} - } - } - branch.Outcome = "approved_skipped" - if _, ok := appliedIndices[branch.ProposalIndex]; ok { + if err != nil { + branch.Outcome = "interrupted" + } else if _, ok := appliedIndices[branch.ProposalIndex]; ok { branch.Outcome = "applied" + } else { + branch.Outcome = "approved_skipped" } } - } - attributeApplyFiltering(sections, pipelineResult.Approved, skipped, policy, workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)]) - sections = updated - moduleOperations := make([]outcomes.Operation, 0) - for _, unit := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] { - moduleOperations = append(moduleOperations, unit.Operations...) + operations = append(operations, unit.Operations...) } - results = append(results, ModuleResult{ - StageIndex: spec.Index, StageID: spec.ID, - ModuleKey: spec.ModuleKey, - ModuleInstance: spec.InstanceName, - ReplacementPolicy: policy, - Status: ModuleStatusSuccess, - ProposalCount: pipelineResult.ProposalCount, - Operations: moduleOperations, - ValidatorDecisions: pipelineResult.ValidatorDecisions, - ValidatorRejected: pipelineResult.ValidatorRejected, - AppliedChanges: applied, - SkippedChanges: skipped, - }) + if input.CaptureCandidates { + candidates = append(candidates, execution.pipeline.Candidates...) + } + result := ModuleResult{ + StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, + ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy, + ProposalCount: execution.pipeline.ProposalCount, Operations: operations, + ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected, + } + if err != nil { + result.Status = ModuleStatusFailed + result.ErrorMessage = err.Error() + } else { + result.Status = ModuleStatusSuccess + result.AppliedChanges = execution.applied + result.SkippedChanges = execution.skipped + } + results = append(results, result) + if err != nil { + return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, err) + } } - output := snapshot() - return output, nil + if err := ctx.Err(); err != nil { + return snapshot(), &ExecutionFailure{Phase: "processing", Cause: err} + } + return snapshot(), nil +} + +type moduleExecution struct { + policy proposals.ReplacementPolicy + pipeline modulePipelineResult + updated []chunking.Section + applied []proposals.AppliedChange + skipped []proposals.SkippedChange + appliedBySection []int +} + +func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) { + var execution moduleExecution + origin := failureReference(spec, nil, "") + if err := ctx.Err(); err != nil { + return execution, locateFailure(err, "processing", origin, false, nil) + } + module, err := r.factory.ModuleForSpec(spec) + if err != nil { + return execution, locateFailure(fmt.Errorf("setup failed: %w", err), "setup", origin, false, nil) + } + execution.policy = module.ReplacementPolicy() + if err := spec.PrepareGlossary(); err != nil { + return execution, locateFailure(fmt.Errorf("glossary preparation failed: %w", err), "setup", origin, false, nil) + } + execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{ + Module: module, + Spec: spec, + Policy: execution.policy, + Sections: sections, + ProposalClient: spec.ProposalClient, + Interactions: interactions, + ValidationClient: spec.ValidationClient, + GlossaryVariables: spec.GlossaryVariables, + GlossaryView: spec.GlossaryView, + }) + if err != nil { + return execution, err + } + if err := ctx.Err(); err != nil { + return execution, locateFailure(err, "application", origin, false, nil) + } + execution.updated, execution.applied, execution.skipped, execution.appliedBySection, err = applyWithinSections(sections, execution.pipeline.Approved, execution.policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord) + if err != nil { + return execution, locateFailure(fmt.Errorf("application failed: %w", err), "application", origin, false, nil) + } + return execution, nil } func transcriptFromSections(sections []chunking.Section) *schema.Transcript { @@ -345,7 +338,7 @@ type sectionProposalResult struct { section chunking.Section proposal contracts.ProposalResult started bool - err error + failure *ExecutionFailure } type sectionValidationResult struct { @@ -357,7 +350,7 @@ type sectionValidationResult struct { rejected []ValidatorRejectedChange operations []outcomes.Operation filtered []outcomes.FilteredProposal - err error + failure *ExecutionFailure } type modulePipelineResult struct { @@ -370,9 +363,8 @@ type modulePipelineResult struct { ValidatorRejected []ValidatorRejectedChange } -func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) { +func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult { results := make(chan sectionProposalResult, len(input.Sections)) - runCtx, cancel := context.WithCancel(ctx) go func() { defer close(results) @@ -383,12 +375,12 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI wg.Add(1) go func() { defer wg.Done() - if runCtx.Err() != nil { + if ctx.Err() != nil { results <- sectionProposalResult{sectionPos: sectionPos, section: section} return } meta := contracts.SectionMetadataFromSection(section) - proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{ + proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Settings: input.Spec.Settings, @@ -407,19 +399,24 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI }, LLMClient: input.ProposalClient, }) + var failure *ExecutionFailure + if err != nil { + failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations) + fail(failure) + } results <- sectionProposalResult{ sectionPos: sectionPos, section: section, proposal: proposalResult, started: true, - err: err, + failure: failure, } }() } wg.Wait() }() - return runCtx, results, cancel + return results } func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) { @@ -445,18 +442,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 { provider, ok := input.Module.(interface{ RepairPromptID() string }) if !ok || input.Spec.RepairCoordinator == nil { - return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"} + cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"} + return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil) } repairPromptID = provider.RepairPromptID() if repairPromptID == "" { - return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"} + cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"} + return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil) } input.Spec.CaptureProposalExchange = true } - runCtx, proposalResults, cancel := collectSectionProposals(ctx, input) - defer cancel() - pending := make(map[int]sectionProposalResult, len(input.Sections)) validationResults := make(chan sectionValidationResult, len(input.Sections)) validationBySection := make(map[int]sectionValidationResult, len(input.Sections)) @@ -466,26 +462,49 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) nextProposalIndex := 0 var ( - firstErr error - errMu sync.Mutex - vwg sync.WaitGroup + firstFailure *ExecutionFailure + errMu sync.Mutex + vwg sync.WaitGroup ) - setErr := func(err error) { - if err == nil { - return - } + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + setFailure := func(failure *ExecutionFailure) { errMu.Lock() defer errMu.Unlock() - if firstErr == nil { - firstErr = err + if firstFailure == nil { + firstFailure = failure cancel() } } - getErr := func() error { - errMu.Lock() - defer errMu.Unlock() - return firstErr + proposalResults := collectSectionProposals(runCtx, input, setFailure) + + collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal { + sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals)) + for i, corrected := range sectionResult.proposal.Proposals { + sectionIndex := index + sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{ + CorrectionProposal: corrected, + ProposalMetadata: proposals.ProposalMetadata{ + ProposalIndex: nextProposalIndex + i, + ModuleKey: input.Spec.ModuleKey, + ModuleInstance: input.Spec.InstanceName, + SectionIndex: §ionIndex, + }, + }) + } + nextProposalIndex += len(sectionEnriched) + out.ProposalCount += len(sectionEnriched) + out.Candidates = append(out.Candidates, sectionEnriched...) + for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations { + operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex] + if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { + for _, proposal := range sectionEnriched { + operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex) + } + } + } + return sectionEnriched } for result := range proposalResults { @@ -494,11 +513,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) unit.State = outcomes.WorkInterrupted } unit.Operations = append(unit.Operations, result.proposal.Operations...) - if result.err != nil { - setErr(result.err) - continue - } - if getErr() != nil { + if result.failure != nil { + unit.Failure = result.failure continue } if !result.started { @@ -507,7 +523,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) pending[result.sectionPos] = result for { - if getErr() != nil { + if runCtx.Err() != nil { break } sectionResult, ok := pending[nextSectionToProcess] @@ -516,30 +532,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } delete(pending, nextSectionToProcess) sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section) - sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals)) - for i, corrected := range sectionResult.proposal.Proposals { - sectionIndex := sectionMeta.Index - sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{ - CorrectionProposal: corrected, - ProposalMetadata: proposals.ProposalMetadata{ - ProposalIndex: nextProposalIndex + i, - ModuleKey: input.Spec.ModuleKey, - ModuleInstance: input.Spec.InstanceName, - SectionIndex: §ionIndex, - }, - }) - } - nextProposalIndex += len(sectionEnriched) - out.ProposalCount += len(sectionEnriched) - out.Candidates = append(out.Candidates, sectionEnriched...) - for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations { - operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex] - if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { - for _, proposal := range sectionEnriched { - operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex) - } - } - } + sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index) vwg.Add(1) go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) { @@ -561,10 +554,18 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } validated, err := validateSectionCandidates(runCtx, validationInput) var repairs []RepairOutcome + phase, kind := "validation", outcomes.OperationValidation if err == nil && repairPromptID != "" { + phase, kind = "correction_repair", outcomes.OperationCorrectionRepair validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID) } - setErr(err) + var failure *ExecutionFailure + if err != nil { + if !errors.As(err, &failure) { + failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations) + } + setFailure(failure) + } validationResults <- sectionValidationResult{ repairs: repairs, sectionPos: sectionPos, @@ -574,7 +575,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) rejected: validated.rejected, operations: validated.operations, filtered: validated.filtered, - err: err, + failure: failure, } }(nextSectionToProcess, sectionEnriched, sectionMeta) nextSectionToProcess++ @@ -584,11 +585,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) vwg.Wait() close(validationResults) + // A failed or canceled module can still have usable proposals from later + // sections. Assign their indices in order after preserving the dispatched + // prefix, without submitting new validation work. + for pos := nextSectionToProcess; pos < len(input.Sections); pos++ { + if result, ok := pending[pos]; ok { + collectCandidates(result, result.section.Index) + } + } + for v := range validationResults { validationBySection[v.sectionPos] = v - if v.err != nil { - setErr(v.err) - } } for i := range input.Sections { @@ -600,7 +607,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) unit.Operations = append(unit.Operations, res.operations...) unit.Repairs = res.repairs mergeFilteredProposals(unit, res.filtered) - if res.err == nil { + unit.Failure = res.failure + if res.failure == nil { unit.State = outcomes.WorkCompleted } else { unit.State = outcomes.WorkInterrupted @@ -625,11 +633,11 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName] }) - if ctx.Err() != nil { - setErr(ctx.Err()) + if err := ctx.Err(); err != nil { + setFailure(locateFailure(err, "processing", failureReference(input.Spec, nil, ""), false, nil)) } - if err := getErr(); err != nil { + if firstFailure != nil { for i := range out.WorkUnits { for j := range out.WorkUnits[i].Repairs { if out.WorkUnits[i].Repairs[j].Outcome == "approved" { @@ -637,7 +645,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } } } - return out, err + return out, firstFailure } return out, nil @@ -713,6 +721,13 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida }) operations = append(operations, vResult.Operations...) + locate := func(cause error) *ExecutionFailure { + ref := failureReference(input.Spec, input.SectionIndex, outcomes.OperationValidation) + ref.ValidatorName, ref.ValidatorOrder = validator.Name(), &validatorOrder + ref.ProposalIndex, ref.RepairAttempt = input.RepairProposalIndex, input.RepairAttempt + return locateFailure(cause, "validation", ref, input.SectionIndex != nil, vResult.Operations) + } + var decisionErr error if err != nil { decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions) @@ -722,7 +737,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida if decisionErr != nil { return validateSectionCandidatesResult{ decisions: decisions, rejected: rejected, operations: operations, filtered: filtered, - }, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr)) + }, locate(fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))) } nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted)) @@ -737,7 +752,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida if contractErr != nil { return validateSectionCandidatesResult{ decisions: decisions, rejected: rejected, operations: operations, filtered: filtered, - }, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr)) + }, locate(fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))) } dispositions[i] = disposition } @@ -789,7 +804,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida // approval to candidates that have not finished the full chain. return validateSectionCandidatesResult{ decisions: decisions, rejected: rejected, operations: operations, filtered: filtered, - }, fmt.Errorf("validator %q failed: %w", validator.Name(), err) + }, locate(fmt.Errorf("validator %q failed: %w", validator.Name(), err)) } eligible = nextEligible } diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index a982187..7cae0cf 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -634,19 +634,22 @@ func TestRunnerSkippedRecorded(t *testing.T) { } func TestRunnerFailureReturnsPartialProgress(t *testing.T) { + origin := errors.New("boom") transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { - return nil, errors.New("boom") + return nil, origin }}, }}) - out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}}) - if err == nil { - t.Fatal("expected error") + out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {Index: 1, ModuleKey: "m2", InstanceName: "m2"}}}) + var failure *ExecutionFailure + if !errors.Is(err, origin) || !errors.As(err, &failure) || failure.Phase != "proposal" || + !failure.HasSection || *failure.Origin.StageIndex != 1 || failure.Origin.ModuleInstance != "m2" || failure.Origin.SectionIndex != 0 { + t.Fatalf("lost initiating proposal failure: %+v", failure) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial transcript progress preserved, got %q", out.FinalTranscript.Segments[0].Text)