Commit module changes only after successful execution

This commit is contained in:
2026-11-16 14:42:24 +00:00
parent e2710d869e
commit 1c43264f6f
7 changed files with 388 additions and 191 deletions

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
@@ -128,6 +129,17 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
if len(outcome.result.WorkUnits) != 3 {
t.Fatalf("lost work inventory: %+v", outcome.result.WorkUnits)
}
var failure *ExecutionFailure
if !errors.As(outcome.err, &failure) || failure.Phase != "validation" || !failure.HasSection ||
failure.Origin.SectionIndex != 0 || failure.Origin.ValidatorName != validator.Name() ||
outcome.result.WorkUnits[0].Failure != failure {
t.Fatalf("lost primary validation owner: failure=%+v units=%+v", failure, outcome.result.WorkUnits)
}
for _, peer := range outcome.result.WorkUnits[1:] {
if peer.State != outcomes.WorkInterrupted || peer.Failure == nil || !errors.Is(peer.Failure, context.Canceled) || peer.Failure == failure {
t.Fatalf("lost peer interruption: %+v", peer)
}
}
if len(outcome.result.Approved) != 0 {
t.Fatalf("fatal validation returned unchecked candidates: %+v", outcome.result.Approved)
}
@@ -150,3 +162,98 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
t.Fatalf("attempt count: %+v", got)
}
}
func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
out, err := New(fakeFactory{}).Run(ctx, RunInput{
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original"}}},
})
if !errors.Is(err, context.Canceled) || out.FinalTranscript == nil || out.FinalTranscript.Segments[0].Text != "original" {
t.Fatalf("canceled entry: result=%+v error=%v", out, err)
}
}
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
origin := errors.New("earlier proposal failed")
laterCompleted := make(chan struct{})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var validations atomic.Int32
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) {
validations.Add(1)
return validators.Result{}, errors.New("validation should not be submitted")
}},
}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index == 0 {
select {
case <-laterCompleted:
return contracts.ProposalResult{}, origin
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
close(laterCompleted)
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
SegmentReferences: []int{1},
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
}, nil
},
}
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, CaptureCandidates: true,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
})
if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err)
}
var failure *ExecutionFailure
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
failure.Origin.SectionIndex != 0 || failure.Origin.Kind != outcomes.OperationProposal || out.WorkUnits[0].Failure != failure {
t.Fatalf("lost originating proposal failure: %+v", failure)
}
candidate := out.Candidates[0]
later := out.WorkUnits[1]
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted ||
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
out.FinalTranscript.Segments[1].Text != "other text" {
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)
}
}
func TestRunnerCancellationAfterApprovalKeepsCommittedModuleText(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
propose := func(original, replacement string) func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return func(contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: replacement}}, nil
}
}
last := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
cancel()
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"first": fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("teh", "the")},
"second": fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: propose("cat", "dog"), validators: []validators.Validator{last}},
}})
out, err := r.Run(ctx, RunInput{
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {Index: 1, ModuleKey: "second", InstanceName: "second"}},
})
var failure *ExecutionFailure
if !errors.Is(err, context.Canceled) || !errors.As(err, &failure) || failure.Origin == nil || *failure.Origin.StageIndex != 1 ||
len(out.ModuleResults) != 2 || out.ModuleResults[0].Status != ModuleStatusSuccess || out.ModuleResults[1].Status != ModuleStatusFailed ||
len(out.ModuleResults[1].AppliedChanges) != 0 || len(out.ModuleResults[1].ValidatorDecisions) != 1 ||
!out.ModuleResults[1].ValidatorDecisions[0].Approved || out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("commit boundary lost earlier text or approval evidence: result=%+v error=%+v", out, failure)
}
}

View File

@@ -0,0 +1,51 @@
package runner
import (
"errors"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
)
// ExecutionFailure associates the initiating cause with its processing owner.
// The section is optional, for example when module setup fails.
type ExecutionFailure struct {
Phase string
Origin *outcomes.Reference
HasSection bool
Cause error
}
func (f *ExecutionFailure) Error() string { return f.Cause.Error() }
func (f *ExecutionFailure) Unwrap() error { return f.Cause }
func locateFailure(cause error, phase string, origin outcomes.Reference, hasSection bool, operations []outcomes.Operation) *ExecutionFailure {
for _, operation := range operations {
if operation.Cause == nil || !errors.Is(cause, operation.Cause) {
continue
}
ref := operation.Reference
if ref.StageIndex == nil {
ref.StageIndex, ref.StageID = origin.StageIndex, origin.StageID
}
if ref.ProposalIndex == nil {
ref.ProposalIndex = origin.ProposalIndex
}
if ref.RepairAttempt == nil {
ref.RepairAttempt = origin.RepairAttempt
}
return &ExecutionFailure{Phase: phase, Origin: &ref, HasSection: hasSection, Cause: cause}
}
return &ExecutionFailure{Phase: phase, Origin: &origin, HasSection: hasSection, Cause: cause}
}
func failureReference(spec contracts.ModuleRunSpec, section *int, kind outcomes.OperationKind) outcomes.Reference {
ref := outcomes.Reference{
StageIndex: &spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, Kind: kind,
}
if section != nil {
ref.SectionIndex = *section
}
return ref
}

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
"errors"
"fmt"
"slices"
@@ -278,6 +279,12 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
if coordinationErr != nil {
branch.Outcome = "interrupted"
result.approved = nil
var failure *ExecutionFailure
if !errors.As(coordinationErr, &failure) {
ref := failureReference(input.Spec, input.SectionIndex, outcomes.OperationCorrectionRepair)
ref.ProposalIndex = &original.ProposalIndex
coordinationErr = locateFailure(coordinationErr, "correction_repair", ref, input.SectionIndex != nil, result.operations)
}
} else if exhausted {
branch.Outcome = "exhausted"
}

View File

@@ -63,6 +63,7 @@ type WorkUnitOutcome struct {
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
@@ -116,15 +117,18 @@ func New(factory ModuleFactory) *Runner {
}
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err := ctx.Err(); err != nil {
return RunOutput{FinalTranscript: input.Transcript.Clone()}, &ExecutionFailure{Phase: "processing", Cause: err}
}
if r == nil || r.factory == nil {
return RunOutput{}, fmt.Errorf("runner module factory is required")
return RunOutput{}, &ExecutionFailure{Phase: "setup", Cause: errors.New("runner module factory is required")}
}
working := input.Transcript.Clone()
results := make([]ModuleResult, 0, len(input.ModuleSpecs))
sections, err := chunkWorkingTranscript(input.Config, working)
if err != nil {
return RunOutput{FinalTranscript: working}, fmt.Errorf("chunk normalized transcript: %w", err)
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
}
inventory := make([]contracts.SectionMetadata, len(sections))
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
@@ -149,135 +153,124 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
}
for moduleIndex, spec := range input.ModuleSpecs {
module, err := r.factory.ModuleForSpec(spec)
if err != nil {
failed := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
Status: ModuleStatusFailed,
ErrorMessage: err.Error(),
}
results = append(results, failed)
return snapshot(), fmt.Errorf("module %q setup failed: %w", spec.InstanceName, err)
execution, err := r.executeModule(ctx, spec, sections, input.Interactions)
units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
copy(units, execution.pipeline.WorkUnits)
if err == nil && ctx.Err() != nil {
err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil)
}
if err == nil {
previous := sections
sections = execution.updated
attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units)
}
policy := module.ReplacementPolicy()
if err := spec.PrepareGlossary(); err != nil {
return snapshot(), fmt.Errorf("module %q glossary preparation: %w", spec.InstanceName, err)
var failure *ExecutionFailure
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
appliedIndices := make(map[int]struct{}, len(execution.applied))
for _, change := range execution.applied {
appliedIndices[change.ProposalIndex] = struct{}{}
}
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Spec: spec,
Policy: policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
Interactions: input.Interactions,
ValidationClient: spec.ValidationClient,
GlossaryVariables: spec.GlossaryVariables,
GlossaryView: spec.GlossaryView,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
}
if pipelineErr == nil && ctx.Err() != nil {
pipelineErr = ctx.Err()
for i := range pipelineResult.WorkUnits {
for j := range pipelineResult.WorkUnits[i].Repairs {
if pipelineResult.WorkUnits[i].Repairs[j].Outcome == "approved" {
pipelineResult.WorkUnits[i].Repairs[j].Outcome = "interrupted"
}
}
}
}
if pipelineErr != nil {
failed := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusFailed,
ProposalCount: pipelineResult.ProposalCount,
Operations: pipelineResult.Operations,
ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected,
ErrorMessage: pipelineErr.Error(),
}
results = append(results, failed)
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
for sectionPos := range sections {
workUnits[moduleIndex*len(sections)+sectionPos].ModuleIndex = moduleIndex
}
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
}
updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord)
if applyErr != nil {
for i := range pipelineResult.WorkUnits {
unit := &pipelineResult.WorkUnits[i]
unit.ModuleIndex = moduleIndex
unit.State = outcomes.WorkInterrupted
for j := range unit.Repairs {
if unit.Repairs[j].Outcome == "approved" {
unit.Repairs[j].Outcome = "interrupted"
}
}
}
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
results = append(results, ModuleResult{StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, ReplacementPolicy: policy, Status: ModuleStatusFailed, ProposalCount: pipelineResult.ProposalCount, Operations: pipelineResult.Operations, ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorRejected: pipelineResult.ValidatorRejected, ErrorMessage: applyErr.Error()})
return snapshot(), fmt.Errorf("module %q application failed: %w", spec.InstanceName, applyErr)
}
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
for sectionPos := range sections {
unit := &workUnits[moduleIndex*len(sections)+sectionPos]
var operations []outcomes.Operation
for i := range units {
unit := &units[i]
unit.ModuleIndex = moduleIndex
unit.State = outcomes.WorkCompleted
unit.CorrectionsApplied = appliedBySection[sectionPos]
}
var appliedIndices map[int]struct{}
for i := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] {
unit := &workUnits[moduleIndex*len(sections)+i]
if err == nil {
unit.State = outcomes.WorkCompleted
unit.CorrectionsApplied = execution.appliedBySection[i]
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
unit.State = outcomes.WorkInterrupted
}
for j := range unit.Repairs {
branch := &unit.Repairs[j]
if branch.Outcome != "approved" {
continue
}
if appliedIndices == nil {
appliedIndices = make(map[int]struct{}, len(applied))
for _, change := range applied {
appliedIndices[change.ProposalIndex] = struct{}{}
}
}
branch.Outcome = "approved_skipped"
if _, ok := appliedIndices[branch.ProposalIndex]; ok {
if err != nil {
branch.Outcome = "interrupted"
} else if _, ok := appliedIndices[branch.ProposalIndex]; ok {
branch.Outcome = "applied"
} else {
branch.Outcome = "approved_skipped"
}
}
}
attributeApplyFiltering(sections, pipelineResult.Approved, skipped, policy, workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)])
sections = updated
moduleOperations := make([]outcomes.Operation, 0)
for _, unit := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] {
moduleOperations = append(moduleOperations, unit.Operations...)
operations = append(operations, unit.Operations...)
}
results = append(results, ModuleResult{
StageIndex: spec.Index, StageID: spec.ID,
ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName,
ReplacementPolicy: policy,
Status: ModuleStatusSuccess,
ProposalCount: pipelineResult.ProposalCount,
Operations: moduleOperations,
ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected,
AppliedChanges: applied,
SkippedChanges: skipped,
})
if input.CaptureCandidates {
candidates = append(candidates, execution.pipeline.Candidates...)
}
result := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
ProposalCount: execution.pipeline.ProposalCount, Operations: operations,
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
}
if err != nil {
result.Status = ModuleStatusFailed
result.ErrorMessage = err.Error()
} else {
result.Status = ModuleStatusSuccess
result.AppliedChanges = execution.applied
result.SkippedChanges = execution.skipped
}
results = append(results, result)
if err != nil {
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, err)
}
}
output := snapshot()
return output, nil
if err := ctx.Err(); err != nil {
return snapshot(), &ExecutionFailure{Phase: "processing", Cause: err}
}
return snapshot(), nil
}
type moduleExecution struct {
policy proposals.ReplacementPolicy
pipeline modulePipelineResult
updated []chunking.Section
applied []proposals.AppliedChange
skipped []proposals.SkippedChange
appliedBySection []int
}
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) {
var execution moduleExecution
origin := failureReference(spec, nil, "")
if err := ctx.Err(); err != nil {
return execution, locateFailure(err, "processing", origin, false, nil)
}
module, err := r.factory.ModuleForSpec(spec)
if err != nil {
return execution, locateFailure(fmt.Errorf("setup failed: %w", err), "setup", origin, false, nil)
}
execution.policy = module.ReplacementPolicy()
if err := spec.PrepareGlossary(); err != nil {
return execution, locateFailure(fmt.Errorf("glossary preparation failed: %w", err), "setup", origin, false, nil)
}
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Spec: spec,
Policy: execution.policy,
Sections: sections,
ProposalClient: spec.ProposalClient,
Interactions: interactions,
ValidationClient: spec.ValidationClient,
GlossaryVariables: spec.GlossaryVariables,
GlossaryView: spec.GlossaryView,
})
if err != nil {
return execution, err
}
if err := ctx.Err(); err != nil {
return execution, locateFailure(err, "application", origin, false, nil)
}
execution.updated, execution.applied, execution.skipped, execution.appliedBySection, err = applyWithinSections(sections, execution.pipeline.Approved, execution.policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord)
if err != nil {
return execution, locateFailure(fmt.Errorf("application failed: %w", err), "application", origin, false, nil)
}
return execution, nil
}
func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
@@ -345,7 +338,7 @@ type sectionProposalResult struct {
section chunking.Section
proposal contracts.ProposalResult
started bool
err error
failure *ExecutionFailure
}
type sectionValidationResult struct {
@@ -357,7 +350,7 @@ type sectionValidationResult struct {
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
err error
failure *ExecutionFailure
}
type modulePipelineResult struct {
@@ -370,9 +363,8 @@ type modulePipelineResult struct {
ValidatorRejected []ValidatorRejectedChange
}
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) {
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
results := make(chan sectionProposalResult, len(input.Sections))
runCtx, cancel := context.WithCancel(ctx)
go func() {
defer close(results)
@@ -383,12 +375,12 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
wg.Add(1)
go func() {
defer wg.Done()
if runCtx.Err() != nil {
if ctx.Err() != nil {
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
return
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
@@ -407,19 +399,24 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
},
LLMClient: input.ProposalClient,
})
var failure *ExecutionFailure
if err != nil {
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
fail(failure)
}
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
proposal: proposalResult,
started: true,
err: err,
failure: failure,
}
}()
}
wg.Wait()
}()
return runCtx, results, cancel
return results
}
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
@@ -445,18 +442,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 {
provider, ok := input.Module.(interface{ RepairPromptID() string })
if !ok || input.Spec.RepairCoordinator == nil {
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"}
cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"}
return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil)
}
repairPromptID = provider.RepairPromptID()
if repairPromptID == "" {
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"}
cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"}
return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil)
}
input.Spec.CaptureProposalExchange = true
}
runCtx, proposalResults, cancel := collectSectionProposals(ctx, input)
defer cancel()
pending := make(map[int]sectionProposalResult, len(input.Sections))
validationResults := make(chan sectionValidationResult, len(input.Sections))
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
@@ -466,26 +462,49 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
nextProposalIndex := 0
var (
firstErr error
errMu sync.Mutex
vwg sync.WaitGroup
firstFailure *ExecutionFailure
errMu sync.Mutex
vwg sync.WaitGroup
)
setErr := func(err error) {
if err == nil {
return
}
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
setFailure := func(failure *ExecutionFailure) {
errMu.Lock()
defer errMu.Unlock()
if firstErr == nil {
firstErr = err
if firstFailure == nil {
firstFailure = failure
cancel()
}
}
getErr := func() error {
errMu.Lock()
defer errMu.Unlock()
return firstErr
proposalResults := collectSectionProposals(runCtx, input, setFailure)
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: nextProposalIndex + i,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: &sectionIndex,
},
})
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range sectionEnriched {
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
}
}
}
return sectionEnriched
}
for result := range proposalResults {
@@ -494,11 +513,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
unit.State = outcomes.WorkInterrupted
}
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.err != nil {
setErr(result.err)
continue
}
if getErr() != nil {
if result.failure != nil {
unit.Failure = result.failure
continue
}
if !result.started {
@@ -507,7 +523,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
pending[result.sectionPos] = result
for {
if getErr() != nil {
if runCtx.Err() != nil {
break
}
sectionResult, ok := pending[nextSectionToProcess]
@@ -516,30 +532,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := sectionMeta.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: nextProposalIndex + i,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: &sectionIndex,
},
})
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations {
operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range sectionEnriched {
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
}
}
}
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
vwg.Add(1)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
@@ -561,10 +554,18 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
setErr(err)
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
setFailure(failure)
}
validationResults <- sectionValidationResult{
repairs: repairs,
sectionPos: sectionPos,
@@ -574,7 +575,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
err: err,
failure: failure,
}
}(nextSectionToProcess, sectionEnriched, sectionMeta)
nextSectionToProcess++
@@ -584,11 +585,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
vwg.Wait()
close(validationResults)
// A failed or canceled module can still have usable proposals from later
// sections. Assign their indices in order after preserving the dispatched
// prefix, without submitting new validation work.
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
if result, ok := pending[pos]; ok {
collectCandidates(result, result.section.Index)
}
}
for v := range validationResults {
validationBySection[v.sectionPos] = v
if v.err != nil {
setErr(v.err)
}
}
for i := range input.Sections {
@@ -600,7 +607,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered)
if res.err == nil {
unit.Failure = res.failure
if res.failure == nil {
unit.State = outcomes.WorkCompleted
} else {
unit.State = outcomes.WorkInterrupted
@@ -625,11 +633,11 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
})
if ctx.Err() != nil {
setErr(ctx.Err())
if err := ctx.Err(); err != nil {
setFailure(locateFailure(err, "processing", failureReference(input.Spec, nil, ""), false, nil))
}
if err := getErr(); err != nil {
if firstFailure != nil {
for i := range out.WorkUnits {
for j := range out.WorkUnits[i].Repairs {
if out.WorkUnits[i].Repairs[j].Outcome == "approved" {
@@ -637,7 +645,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
}
}
return out, err
return out, firstFailure
}
return out, nil
@@ -713,6 +721,13 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
})
operations = append(operations, vResult.Operations...)
locate := func(cause error) *ExecutionFailure {
ref := failureReference(input.Spec, input.SectionIndex, outcomes.OperationValidation)
ref.ValidatorName, ref.ValidatorOrder = validator.Name(), &validatorOrder
ref.ProposalIndex, ref.RepairAttempt = input.RepairProposalIndex, input.RepairAttempt
return locateFailure(cause, "validation", ref, input.SectionIndex != nil, vResult.Operations)
}
var decisionErr error
if err != nil {
decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions)
@@ -722,7 +737,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if decisionErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
}, locate(fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr)))
}
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted))
@@ -737,7 +752,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if contractErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
}, locate(fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr)))
}
dispositions[i] = disposition
}
@@ -789,7 +804,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
// approval to candidates that have not finished the full chain.
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
}, locate(fmt.Errorf("validator %q failed: %w", validator.Name(), err))
}
eligible = nextEligible
}

View File

@@ -634,19 +634,22 @@ func TestRunnerSkippedRecorded(t *testing.T) {
}
func TestRunnerFailureReturnsPartialProgress(t *testing.T) {
origin := errors.New("boom")
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return nil, errors.New("boom")
return nil, origin
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
if err == nil {
t.Fatal("expected error")
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {Index: 1, ModuleKey: "m2", InstanceName: "m2"}}})
var failure *ExecutionFailure
if !errors.Is(err, origin) || !errors.As(err, &failure) || failure.Phase != "proposal" ||
!failure.HasSection || *failure.Origin.StageIndex != 1 || failure.Origin.ModuleInstance != "m2" || failure.Origin.SectionIndex != 0 {
t.Fatalf("lost initiating proposal failure: %+v", failure)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial transcript progress preserved, got %q", out.FinalTranscript.Segments[0].Text)