Commit module changes only after successful execution

This commit is contained in:
2026-11-16 14:42:24 +00:00
parent e2710d869e
commit 1c43264f6f
7 changed files with 388 additions and 191 deletions

View File

@@ -40,9 +40,14 @@ explicit `not_started`/`interrupted`/`completed` work units, compact model
operation outcomes, and per-module results. Candidate payloads are retained
in the returned output only when debug capture is requested. A work unit can
complete with model errors and applied corrections. Recoverable model failures
never trigger shared cancellation. Fatal setup, internal, application, or
parent-cancellation errors cancel peers, join workers, and preserve earlier
committed module text for partial reports; no completed transcript is published.
never trigger shared cancellation. Fatal failures and parent cancellation stop
further dispatch and join active workers. The initiating error carries its
processing phase and available origin. Work-unit evidence preserves each peer's
interruption or error. Usable proposal results from later sections are indexed
in order after joining, even when the runner could not validate them. The
working transcript advances only at the module's successful commit boundary.
Earlier committed text remains available to an interrupted run, and transcript
delivery requires successful processing.
Required transcript/report write failures are fatal. Optional debug writes
report safely to stderr without changing decisions or exit status. See

View File

@@ -543,6 +543,15 @@ earlier chunk, and a fatal stage after earlier committed text. Assert causal
origin and preserved evidence as well as returned failure. Leave report/ledger
projection to Stage 17.
**Complete.** A single module completion path advances the transcript only
after a successful application and context check. The runner returns the
initiating error with its phase and available origin, and keeps peer failures
on their work units. After joining, it assigns deterministic indices to usable
proposal responses that never reached validation. Tests exercised canceled
entry, cancellation after approval, a later response behind an earlier
failure, and preservation of earlier committed text. The CLI build, full
offline suite, and targeted race checks passed.
## Stage 16 — Bound local preparation and actively drain results
Audit #15 (dispatch) and #30, under D3.

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
@@ -128,6 +129,17 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
if len(outcome.result.WorkUnits) != 3 {
t.Fatalf("lost work inventory: %+v", outcome.result.WorkUnits)
}
var failure *ExecutionFailure
if !errors.As(outcome.err, &failure) || failure.Phase != "validation" || !failure.HasSection ||
failure.Origin.SectionIndex != 0 || failure.Origin.ValidatorName != validator.Name() ||
outcome.result.WorkUnits[0].Failure != failure {
t.Fatalf("lost primary validation owner: failure=%+v units=%+v", failure, outcome.result.WorkUnits)
}
for _, peer := range outcome.result.WorkUnits[1:] {
if peer.State != outcomes.WorkInterrupted || peer.Failure == nil || !errors.Is(peer.Failure, context.Canceled) || peer.Failure == failure {
t.Fatalf("lost peer interruption: %+v", peer)
}
}
if len(outcome.result.Approved) != 0 {
t.Fatalf("fatal validation returned unchecked candidates: %+v", outcome.result.Approved)
}
@@ -150,3 +162,98 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
t.Fatalf("attempt count: %+v", got)
}
}
func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
out, err := New(fakeFactory{}).Run(ctx, RunInput{
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original"}}},
})
if !errors.Is(err, context.Canceled) || out.FinalTranscript == nil || out.FinalTranscript.Segments[0].Text != "original" {
t.Fatalf("canceled entry: result=%+v error=%v", out, err)
}
}
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
origin := errors.New("earlier proposal failed")
laterCompleted := make(chan struct{})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var validations atomic.Int32
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) {
validations.Add(1)
return validators.Result{}, errors.New("validation should not be submitted")
}},
}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index == 0 {
select {
case <-laterCompleted:
return contracts.ProposalResult{}, origin
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
close(laterCompleted)
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
SegmentReferences: []int{1},
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
}, nil
},
}
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, CaptureCandidates: true,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
})
if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err)
}
var failure *ExecutionFailure
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
failure.Origin.SectionIndex != 0 || failure.Origin.Kind != outcomes.OperationProposal || out.WorkUnits[0].Failure != failure {
t.Fatalf("lost originating proposal failure: %+v", failure)
}
candidate := out.Candidates[0]
later := out.WorkUnits[1]
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted ||
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
out.FinalTranscript.Segments[1].Text != "other text" {
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)
}
}
func TestRunnerCancellationAfterApprovalKeepsCommittedModuleText(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
propose := func(original, replacement string) func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: replacement}}, nil
}
}
last := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
cancel()
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"first": fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("teh", "the")},
"second": fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("cat", "dog"), validators: []validators.Validator{last}},
}})
out, err := r.Run(ctx, RunInput{
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {Index: 1, ModuleKey: "second", InstanceName: "second"}},
})
var failure *ExecutionFailure
if !errors.Is(err, context.Canceled) || !errors.As(err, &failure) || failure.Origin == nil || *failure.Origin.StageIndex != 1 ||
len(out.ModuleResults) != 2 || out.ModuleResults[0].Status != ModuleStatusSuccess || out.ModuleResults[1].Status != ModuleStatusFailed ||
len(out.ModuleResults[1].AppliedChanges) != 0 || len(out.ModuleResults[1].ValidatorDecisions) != 1 ||
!out.ModuleResults[1].ValidatorDecisions[0].Approved || out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("commit boundary lost earlier text or approval evidence: result=%+v error=%+v", out, failure)
}
}

View File

@@ -0,0 +1,51 @@
package runner
import (
"errors"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
)
// ExecutionFailure associates the initiating cause with its processing owner.
// The section is optional, for example when module setup fails.
type ExecutionFailure struct {
Phase string
Origin *outcomes.Reference
HasSection bool
Cause error
}
func (f *ExecutionFailure) Error() string { return f.Cause.Error() }
func (f *ExecutionFailure) Unwrap() error { return f.Cause }
func locateFailure(cause error, phase string, origin outcomes.Reference, hasSection bool, operations []outcomes.Operation) *ExecutionFailure {
for _, operation := range operations {
if operation.Cause == nil || !errors.Is(cause, operation.Cause) {
continue
}
ref := operation.Reference
if ref.StageIndex == nil {
ref.StageIndex, ref.StageID = origin.StageIndex, origin.StageID
}
if ref.ProposalIndex == nil {
ref.ProposalIndex = origin.ProposalIndex
}
if ref.RepairAttempt == nil {
ref.RepairAttempt = origin.RepairAttempt
}
return &ExecutionFailure{Phase: phase, Origin: &ref, HasSection: hasSection, Cause: cause}
}
return &ExecutionFailure{Phase: phase, Origin: &origin, HasSection: hasSection, Cause: cause}
}
func failureReference(spec contracts.ModuleRunSpec, section *int, kind outcomes.OperationKind) outcomes.Reference {
ref := outcomes.Reference{
StageIndex: &spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, Kind: kind,
}
if section != nil {
ref.SectionIndex = *section
}
return ref
}

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
"errors"
"fmt"
"slices"
@@ -278,6 +279,12 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
if coordinationErr != nil {
branch.Outcome = "interrupted"
result.approved = nil
var failure *ExecutionFailure
if !errors.As(coordinationErr, &failure) {
ref := failureReference(input.Spec, input.SectionIndex, outcomes.OperationCorrectionRepair)
ref.ProposalIndex = &original.ProposalIndex
coordinationErr = locateFailure(coordinationErr, "correction_repair", ref, input.SectionIndex != nil, result.operations)
}
} else if exhausted {
branch.Outcome = "exhausted"
}

View File

@@ -63,6 +63,7 @@ type WorkUnitOutcome struct {
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
@@ -116,15 +117,18 @@ func New(factory ModuleFactory) *Runner {
}
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err := ctx.Err(); err != nil {
return RunOutput{FinalTranscript: input.Transcript.Clone()}, &ExecutionFailure{Phase: "processing", Cause: err}
}
if r == nil || r.factory == nil {
return RunOutput{}, fmt.Errorf("runner module factory is required")
return RunOutput{}, &ExecutionFailure{Phase: "setup", Cause: errors.New("runner module factory is required")}
}
working := input.Transcript.Clone()
results := make([]ModuleResult, 0, len(input.ModuleSpecs))
sections, err := chunkWorkingTranscript(input.Config, working)
if err != nil {
return RunOutput{FinalTranscript: working}, fmt.Errorf("chunk normalized transcript: %w", err)
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
}
inventory := make([]contracts.SectionMetadata, len(sections))
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
@@ -149,135 +153,124 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
for moduleIndex, spec := range input.ModuleSpecs {
module, err := r.factory.ModuleForSpec(spec)
if err != nil {
failed := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
Status: ModuleStatusFailed,
ErrorMessage: err.Error(),
}
results = append(results, failed)
return snapshot(), fmt.Errorf("module %q setup failed: %w", spec.InstanceName, err)
execution, err := r.executeModule(ctx, spec, sections, input.Interactions)
units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
copy(units, execution.pipeline.WorkUnits)
if err == nil && ctx.Err() != nil {
err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil)
}
if err == nil {
previous := sections
sections = execution.updated
attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units)
}
policy := module.ReplacementPolicy()
if err := spec.PrepareGlossary(); err != nil {
return snapshot(), fmt.Errorf("module %q glossary preparation: %w", spec.InstanceName, err)
var failure *ExecutionFailure
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
appliedIndices := make(map[int]struct{}, len(execution.applied))
for _, change := range execution.applied {
appliedIndices[change.ProposalIndex] = struct{}{}
}
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Spec: spec,
Policy: policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
Interactions: input.Interactions,
ValidationClient: spec.ValidationClient,
GlossaryVariables: spec.GlossaryVariables,
GlossaryView: spec.GlossaryView,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
}
if pipelineErr == nil && ctx.Err() != nil {
pipelineErr = ctx.Err()
for i := range pipelineResult.WorkUnits {
for j := range pipelineResult.WorkUnits[i].Repairs {
if pipelineResult.WorkUnits[i].Repairs[j].Outcome == "approved" {
pipelineResult.WorkUnits[i].Repairs[j].Outcome = "interrupted"
}
}
}
}
if pipelineErr != nil {
failed := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusFailed,
ProposalCount: pipelineResult.ProposalCount,
Operations: pipelineResult.Operations,
ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected,
ErrorMessage: pipelineErr.Error(),
}
results = append(results, failed)
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
for sectionPos := range sections {
workUnits[moduleIndex*len(sections)+sectionPos].ModuleIndex = moduleIndex
}
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
}
updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord)
if applyErr != nil {
for i := range pipelineResult.WorkUnits {
unit := &pipelineResult.WorkUnits[i]
unit.ModuleIndex = moduleIndex
unit.State = outcomes.WorkInterrupted
for j := range unit.Repairs {
if unit.Repairs[j].Outcome == "approved" {
unit.Repairs[j].Outcome = "interrupted"
}
}
}
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
results = append(results, ModuleResult{StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, ReplacementPolicy: policy, Status: ModuleStatusFailed, ProposalCount: pipelineResult.ProposalCount, Operations: pipelineResult.Operations, ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorRejected: pipelineResult.ValidatorRejected, ErrorMessage: applyErr.Error()})
return snapshot(), fmt.Errorf("module %q application failed: %w", spec.InstanceName, applyErr)
}
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
for sectionPos := range sections {
unit := &workUnits[moduleIndex*len(sections)+sectionPos]
var operations []outcomes.Operation
for i := range units {
unit := &units[i]
unit.ModuleIndex = moduleIndex
unit.State = outcomes.WorkCompleted
unit.CorrectionsApplied = appliedBySection[sectionPos]
}
var appliedIndices map[int]struct{}
for i := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] {
unit := &workUnits[moduleIndex*len(sections)+i]
if err == nil {
unit.State = outcomes.WorkCompleted
unit.CorrectionsApplied = execution.appliedBySection[i]
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
unit.State = outcomes.WorkInterrupted
}
for j := range unit.Repairs {
branch := &unit.Repairs[j]
if branch.Outcome != "approved" {
continue
}
if appliedIndices == nil {
appliedIndices = make(map[int]struct{}, len(applied))
for _, change := range applied {
appliedIndices[change.ProposalIndex] = struct{}{}
}
}
branch.Outcome = "approved_skipped"
if _, ok := appliedIndices[branch.ProposalIndex]; ok {
if err != nil {
branch.Outcome = "interrupted"
} else if _, ok := appliedIndices[branch.ProposalIndex]; ok {
branch.Outcome = "applied"
} else {
branch.Outcome = "approved_skipped"
}
}
}
attributeApplyFiltering(sections, pipelineResult.Approved, skipped, policy, workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)])
sections = updated
moduleOperations := make([]outcomes.Operation, 0)
for _, unit := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] {
moduleOperations = append(moduleOperations, unit.Operations...)
operations = append(operations, unit.Operations...)
}
results = append(results, ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusSuccess,
ProposalCount: pipelineResult.ProposalCount,
Operations: moduleOperations,
ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected,
AppliedChanges: applied,
SkippedChanges: skipped,
})
if input.CaptureCandidates {
candidates = append(candidates, execution.pipeline.Candidates...)
}
result := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
ProposalCount: execution.pipeline.ProposalCount, Operations: operations,
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
}
if err != nil {
result.Status = ModuleStatusFailed
result.ErrorMessage = err.Error()
} else {
result.Status = ModuleStatusSuccess
result.AppliedChanges = execution.applied
result.SkippedChanges = execution.skipped
}
results = append(results, result)
if err != nil {
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, err)
}
}
output := snapshot()
return output, nil
if err := ctx.Err(); err != nil {
return snapshot(), &ExecutionFailure{Phase: "processing", Cause: err}
}
return snapshot(), nil
}
type moduleExecution struct {
policy proposals.ReplacementPolicy
pipeline modulePipelineResult
updated []chunking.Section
applied []proposals.AppliedChange
skipped []proposals.SkippedChange
appliedBySection []int
}
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) {
var execution moduleExecution
origin := failureReference(spec, nil, "")
if err := ctx.Err(); err != nil {
return execution, locateFailure(err, "processing", origin, false, nil)
}
module, err := r.factory.ModuleForSpec(spec)
if err != nil {
return execution, locateFailure(fmt.Errorf("setup failed: %w", err), "setup", origin, false, nil)
}
execution.policy = module.ReplacementPolicy()
if err := spec.PrepareGlossary(); err != nil {
return execution, locateFailure(fmt.Errorf("glossary preparation failed: %w", err), "setup", origin, false, nil)
}
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Spec: spec,
Policy: execution.policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
Interactions: interactions,
ValidationClient: spec.ValidationClient,
GlossaryVariables: spec.GlossaryVariables,
GlossaryView: spec.GlossaryView,
})
if err != nil {
return execution, err
}
if err := ctx.Err(); err != nil {
return execution, locateFailure(err, "application", origin, false, nil)
}
execution.updated, execution.applied, execution.skipped, execution.appliedBySection, err = applyWithinSections(sections, execution.pipeline.Approved, execution.policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord)
if err != nil {
return execution, locateFailure(fmt.Errorf("application failed: %w", err), "application", origin, false, nil)
}
return execution, nil
}
func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
@@ -345,7 +338,7 @@ type sectionProposalResult struct {
section chunking.Section
proposal contracts.ProposalResult
started bool
err error
failure *ExecutionFailure
}
type sectionValidationResult struct {
@@ -357,7 +350,7 @@ type sectionValidationResult struct {
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
err error
failure *ExecutionFailure
}
type modulePipelineResult struct {
@@ -370,9 +363,8 @@ type modulePipelineResult struct {
ValidatorRejected []ValidatorRejectedChange
}
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) {
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
results := make(chan sectionProposalResult, len(input.Sections))
runCtx, cancel := context.WithCancel(ctx)
go func() {
defer close(results)
@@ -383,12 +375,12 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
wg.Add(1)
go func() {
defer wg.Done()
if runCtx.Err() != nil {
if ctx.Err() != nil {
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
return
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
@@ -407,19 +399,24 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
},
LLMClient: input.ProposalClient,
})
var failure *ExecutionFailure
if err != nil {
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
fail(failure)
}
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
proposal: proposalResult,
started: true,
err: err,
failure: failure,
}
}()
}
wg.Wait()
}()
return runCtx, results, cancel
return results
}
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
@@ -445,18 +442,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 {
provider, ok := input.Module.(interface{ RepairPromptID() string })
if !ok || input.Spec.RepairCoordinator == nil {
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"}
cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"}
return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil)
}
repairPromptID = provider.RepairPromptID()
if repairPromptID == "" {
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"}
cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"}
return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil)
}
input.Spec.CaptureProposalExchange = true
}
runCtx, proposalResults, cancel := collectSectionProposals(ctx, input)
defer cancel()
pending := make(map[int]sectionProposalResult, len(input.Sections))
validationResults := make(chan sectionValidationResult, len(input.Sections))
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
@@ -466,26 +462,49 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
nextProposalIndex := 0
var (
firstErr error
errMu sync.Mutex
vwg sync.WaitGroup
firstFailure *ExecutionFailure
errMu sync.Mutex
vwg sync.WaitGroup
)
setErr := func(err error) {
if err == nil {
return
}
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
setFailure := func(failure *ExecutionFailure) {
errMu.Lock()
defer errMu.Unlock()
if firstErr == nil {
firstErr = err
if firstFailure == nil {
firstFailure = failure
cancel()
}
}
getErr := func() error {
errMu.Lock()
defer errMu.Unlock()
return firstErr
proposalResults := collectSectionProposals(runCtx, input, setFailure)
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: nextProposalIndex + i,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: &sectionIndex,
},
})
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range sectionEnriched {
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
}
}
}
return sectionEnriched
}
for result := range proposalResults {
@@ -494,11 +513,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
unit.State = outcomes.WorkInterrupted
}
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.err != nil {
setErr(result.err)
continue
}
if getErr() != nil {
if result.failure != nil {
unit.Failure = result.failure
continue
}
if !result.started {
@@ -507,7 +523,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
pending[result.sectionPos] = result
for {
if getErr() != nil {
if runCtx.Err() != nil {
break
}
sectionResult, ok := pending[nextSectionToProcess]
@@ -516,30 +532,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := sectionMeta.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: nextProposalIndex + i,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: &sectionIndex,
},
})
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations {
operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range sectionEnriched {
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
}
}
}
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
vwg.Add(1)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
@@ -561,10 +554,18 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
setErr(err)
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
setFailure(failure)
}
validationResults <- sectionValidationResult{
repairs: repairs,
sectionPos: sectionPos,
@@ -574,7 +575,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
err: err,
failure: failure,
}
}(nextSectionToProcess, sectionEnriched, sectionMeta)
nextSectionToProcess++
@@ -584,11 +585,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
vwg.Wait()
close(validationResults)
// A failed or canceled module can still have usable proposals from later
// sections. Assign their indices in order after preserving the dispatched
// prefix, without submitting new validation work.
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
if result, ok := pending[pos]; ok {
collectCandidates(result, result.section.Index)
}
}
for v := range validationResults {
validationBySection[v.sectionPos] = v
if v.err != nil {
setErr(v.err)
}
}
for i := range input.Sections {
@@ -600,7 +607,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered)
if res.err == nil {
unit.Failure = res.failure
if res.failure == nil {
unit.State = outcomes.WorkCompleted
} else {
unit.State = outcomes.WorkInterrupted
@@ -625,11 +633,11 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
})
if ctx.Err() != nil {
setErr(ctx.Err())
if err := ctx.Err(); err != nil {
setFailure(locateFailure(err, "processing", failureReference(input.Spec, nil, ""), false, nil))
}
if err := getErr(); err != nil {
if firstFailure != nil {
for i := range out.WorkUnits {
for j := range out.WorkUnits[i].Repairs {
if out.WorkUnits[i].Repairs[j].Outcome == "approved" {
@@ -637,7 +645,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
}
}
return out, err
return out, firstFailure
}
return out, nil
@@ -713,6 +721,13 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
})
operations = append(operations, vResult.Operations...)
locate := func(cause error) *ExecutionFailure {
ref := failureReference(input.Spec, input.SectionIndex, outcomes.OperationValidation)
ref.ValidatorName, ref.ValidatorOrder = validator.Name(), &validatorOrder
ref.ProposalIndex, ref.RepairAttempt = input.RepairProposalIndex, input.RepairAttempt
return locateFailure(cause, "validation", ref, input.SectionIndex != nil, vResult.Operations)
}
var decisionErr error
if err != nil {
decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions)
@@ -722,7 +737,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if decisionErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
}, locate(fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr)))
}
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted))
@@ -737,7 +752,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if contractErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
}, locate(fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr)))
}
dispositions[i] = disposition
}
@@ -789,7 +804,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
// approval to candidates that have not finished the full chain.
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
}, locate(fmt.Errorf("validator %q failed: %w", validator.Name(), err))
}
eligible = nextEligible
}

View File

@@ -634,19 +634,22 @@ func TestRunnerSkippedRecorded(t *testing.T) {
}
func TestRunnerFailureReturnsPartialProgress(t *testing.T) {
origin := errors.New("boom")
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return nil, errors.New("boom")
return nil, origin
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
if err == nil {
t.Fatal("expected error")
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {Index: 1, ModuleKey: "m2", InstanceName: "m2"}}})
var failure *ExecutionFailure
if !errors.Is(err, origin) || !errors.As(err, &failure) || failure.Phase != "proposal" ||
!failure.HasSection || *failure.Origin.StageIndex != 1 || failure.Origin.ModuleInstance != "m2" || failure.Origin.SectionIndex != 0 {
t.Fatalf("lost initiating proposal failure: %+v", failure)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial transcript progress preserved, got %q", out.FinalTranscript.Segments[0].Text)