Preserve partial run results and terminal failure causes

This commit is contained in:
2026-11-20 05:47:41 +00:00
parent 85df36a770
commit a6c6f1e4ab
13 changed files with 451 additions and 108 deletions

View File

@@ -20,14 +20,16 @@ type executionInput struct {
}
type runReport struct {
Campaign string `json:"campaign"`
SessionID string `json:"session_id"`
RunID string `json:"run_id"`
Stages []string `json:"stages"`
Executed []string `json:"executed"`
Skipped []string `json:"skipped"`
ArtifactTargets []string `json:"artifact_targets"`
Complete bool `json:"complete"`
Campaign string `json:"campaign"`
SessionID string `json:"session_id"`
RunID string `json:"run_id"`
Stages []string `json:"stages"`
Executed []string `json:"executed"`
Skipped []string `json:"skipped"`
ArtifactTargets []string `json:"artifact_targets"`
StageResults []RunStageResult `json:"stage_results"`
AvailableArtifacts []RunArtifact `json:"available_artifacts"`
Complete bool `json:"complete"`
}
func checkExecutionInput(input *executionInput) error {
@@ -137,5 +139,5 @@ func executeSelectedPipeline(ctx context.Context, input *executionInput, deps co
}
return nil, err
}
return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), Complete: err == nil}, err
return &runReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: summary.SessionID, RunID: summary.RunID, Stages: input.plan.Names(), Executed: summary.Executed, Skipped: summary.Skipped, ArtifactTargets: effective.Keys(), StageResults: summary.StageResults, AvailableArtifacts: summary.AvailableArtifacts, Complete: err == nil}, err
}

View File

@@ -4,7 +4,12 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/config"
@@ -101,3 +106,102 @@ func TestExecutionCommandsRejectInvalidSelectionBeforeBootstrap(t *testing.T) {
}
}
}
type releaseObservedStore struct {
artifacts.Store
released bool
}
func (s *releaseObservedStore) ReleaseSessionLock(lock *artifacts.LockHandle) error {
err := s.Store.ReleaseSessionLock(lock)
s.released = true
return err
}
type failingRunReportWriter struct {
t *testing.T
store *releaseObservedStore
err error
}
func (w failingRunReportWriter) Write([]byte) (int, error) {
if !w.store.released {
w.t.Fatal("report written before lock release")
}
return 0, w.err
}
func TestExecutionCommandReportFailurePreservesCompletedWork(t *testing.T) {
pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir())
writeErr := errors.New("report destination unavailable")
var saved *RunSummary
var local *releaseObservedStore
app, err := newApplication(commandDependencies{}, func(ctx context.Context, cfg *config.Config, _ BoundedPlan, opts RunOptions) (*RunSummary, error) {
local = &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
opts.Env = &Env{ArtifactStore: local}
var err error
saved, err = executeStages(ctx, cfg, []stage.Stage{resultStage{name: "prepare"}}, opts)
return saved, err
})
if err != nil {
t.Fatal(err)
}
// Defer resolving the store until Report invokes Write.
writer := runReportWriteFunc(func(p []byte) (int, error) {
return (failingRunReportWriter{t: t, store: local, err: writeErr}).Write(p)
})
var diagnostic bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "2026-05-03", "--stage", "prepare", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"}, commandkit.Streams{Out: writer, Err: &diagnostic})
if outcome.Status != 1 || !errors.Is(outcome.Cause, writeErr) {
t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String())
}
if saved == nil || !saved.Complete {
t.Fatalf("completed domain result=%#v", saved)
}
durable, err := (&manifest.LocalStore{}).Load(context.Background(), saved.ManifestPath)
if err != nil {
t.Fatal(err)
}
if durable.Stages["prepare"].Status != manifest.StatusSucceeded || durable.LastError != nil {
t.Fatalf("report failure changed domain state: %#v", durable)
}
}
type runReportWriteFunc func([]byte) (int, error)
func (f runReportWriteFunc) Write(p []byte) (int, error) { return f(p) }
func TestExecutionCommandPresentsPartialFailureThroughNativeVisibility(t *testing.T) {
pipeline, campaign, session := writeValidConfigFiles(t, t.TempDir())
cause := errors.New("synthetic terminal operation failure")
app, err := newApplication(commandDependencies{}, func(_ context.Context, cfg *config.Config, _ BoundedPlan, _ RunOptions) (*RunSummary, error) {
return &RunSummary{SessionID: cfg.Session.SessionID, RunID: "synthetic-run", Executed: []string{"prepare", "transcribe"}, StageResults: []RunStageResult{{Name: "prepare", Attempted: true, Outcome: "completed"}, {Name: "transcribe", Attempted: true, Outcome: "failed"}}}, cause
})
if err != nil {
t.Fatal(err)
}
for _, quiet := range []bool{false, true} {
args := []string{"run", "2026-05-03", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json"}
if quiet {
args = append(args, "--quiet")
}
var out, diagnostic bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
if outcome.Status != 1 || !errors.Is(outcome.Cause, cause) || strings.Count(diagnostic.String(), cause.Error()) != 1 {
t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String())
}
if quiet {
if out.Len() != 0 {
t.Fatal("quiet emitted a report")
}
continue
}
var report runReport
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
t.Fatal(err)
}
if report.Complete || len(report.StageResults) != 2 || report.StageResults[1].Outcome != "failed" {
t.Fatalf("report=%#v", report)
}
}
}

View File

@@ -26,12 +26,15 @@ type RunOptions struct {
}
type RunSummary struct {
SessionID string
RunID string
ManifestPath string
StageNames []string
Executed []string
Skipped []string
SessionID string
RunID string
ManifestPath string
StageNames []string
Executed []string
Skipped []string
StageResults []RunStageResult
AvailableArtifacts []RunArtifact
Complete bool
}
type pipelineExecutor func(context.Context, *config.Config, BoundedPlan, RunOptions) (*RunSummary, error)
@@ -62,7 +65,20 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
resultErr = errors.Join(resultErr, fmt.Errorf("release session lock: %w", releaseErr))
}
}
if summary != nil {
summary.Complete = resultErr == nil
}
}()
defer func() {
if runtime.started {
summary, resultErr = finalizeInvocation(runtime, resultErr)
}
}()
for _, selected := range runtime.stages {
runtime.stageNames = append(runtime.stageNames, selected.Name())
runtime.stageResults = append(runtime.stageResults, RunStageResult{Name: selected.Name(), Outcome: "not_attempted"})
}
if err := initializeLockedInvocation(runtime); err != nil {
return nil, err
@@ -71,20 +87,32 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
return nil, err
}
for _, selected := range runtime.stages {
runtime.stageNames = append(runtime.stageNames, selected.Name())
for index, selected := range runtime.stages {
outcome, err := executeSelectedStage(runtime, selected)
if err != nil {
return nil, err
result := &runtime.stageResults[index]
result.Attempted = outcome.executed
result.Reason = outcome.reason
switch {
case err != nil:
result.Outcome = "failed"
case outcome.reused:
result.Outcome = "reused"
case outcome.skipped:
result.Outcome = "skipped"
default:
result.Outcome = "completed"
}
if outcome.executed {
runtime.executed = append(runtime.executed, selected.Name())
}
if outcome.skipped {
if outcome.skipped && err == nil {
runtime.skipped = append(runtime.skipped, selected.Name())
}
if err != nil {
return nil, err
}
}
return finalizeInvocation(runtime)
return nil, nil
}
func stagesContainAny(stages []stage.Stage, names ...string) bool {

View File

@@ -14,21 +14,21 @@ func composeInvocation(runtime *invocationRuntime) error {
if env.WhisperX == nil && stagesContainAny(stages, "transcribe") {
client, err := dependencies.newWhisperX(env.Config)
if err != nil {
return runtime.persistTerminalFailure(fmt.Errorf("initialize whisperx client: %w", err))
return fmt.Errorf("initialize whisperx client: %w", err)
}
env.WhisperX = client
}
if env.Seriatim == nil && stagesContainAny(stages, "merge", "normalize", "trim", "render") {
runner, err := dependencies.newSeriatim(env.Config)
if err != nil {
return runtime.persistTerminalFailure(fmt.Errorf("initialize seriatim runner: %w", err))
return fmt.Errorf("initialize seriatim runner: %w", err)
}
env.Seriatim = runner
}
if env.Audita == nil && stagesContainAny(stages, "polish") {
runner, err := dependencies.newAudita(env.Config)
if err != nil {
return runtime.persistTerminalFailure(fmt.Errorf("initialize audita runner: %w", err))
return fmt.Errorf("initialize audita runner: %w", err)
}
env.Audita = runner
}
@@ -41,7 +41,7 @@ func composeInvocation(runtime *invocationRuntime) error {
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages, runtime.effectiveArtifacts) {
objectStore, err := dependencies.newObjectStore(runtime.ctx, env.Config)
if err != nil {
return runtime.persistTerminalFailure(err)
return err
}
env.ObjectStore = objectStore
}

View File

@@ -1,12 +1,106 @@
package app
func finalizeInvocation(runtime *invocationRuntime) (*RunSummary, error) {
return &RunSummary{
SessionID: runtime.identity.SessionID,
RunID: runtime.identity.RunID,
ManifestPath: runtime.manifestPath,
StageNames: runtime.stageNames,
Executed: runtime.executed,
Skipped: runtime.skipped,
}, nil
import (
"errors"
"fmt"
"sort"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
// RunStageResult describes this invocation's work, independently of prior state.
type RunStageResult struct {
Name string `json:"name"`
Attempted bool `json:"attempted"`
Outcome string `json:"outcome"`
Reason string `json:"reason,omitempty"`
}
// RunArtifact identifies durable output evidence without exposing manifest internals.
type RunArtifact struct {
Stage string `json:"stage"`
Source string `json:"source,omitempty"`
Kind string `json:"kind"`
Path string `json:"path"`
ProducerRunID string `json:"producer_run_id,omitempty"`
Evidence string `json:"evidence"`
}
func finalizeInvocation(runtime *invocationRuntime, operationErr error) (*RunSummary, error) {
evidence := runtime.manifest
resultErr := operationErr
if operationErr != nil {
resultErr = persistSessionTerminalFailure(runtime.ctx, runtime.manifestStore, runtime.manifestPath, runtime.manifest, operationErr, runtime.dependencies.now())
// Read durable authority after a failure: checkpoint saves can succeed
// even when terminal persistence fails.
var err error
evidence, err = runtime.manifestStore.Load(runtime.ctx, runtime.manifestPath)
if err != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("read durable run evidence: %w", err))
evidence = nil
}
}
return &RunSummary{
SessionID: runtime.identity.SessionID, RunID: runtime.identity.RunID,
ManifestPath: runtime.manifestPath, StageNames: runtime.stageNames,
Executed: runtime.executed, Skipped: runtime.skipped,
StageResults: runtime.stageResults, AvailableArtifacts: durableRunArtifacts(evidence),
Complete: resultErr == nil,
}, resultErr
}
func durableRunArtifacts(m *manifest.Manifest) []RunArtifact {
out := []RunArtifact{}
if m == nil {
return out
}
names := make([]string, 0, len(m.Stages))
for name := range m.Stages {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
record := m.Stages[name]
if record == nil {
continue
}
seen := map[string]bool{}
add := func(value manifest.ArtifactRecord, evidence string) {
key := value.SourceID + "\x00" + value.LocalPath
if seen[key] {
return
}
seen[key] = true
out = append(out, RunArtifact{Stage: name, Source: value.SourceID, Kind: value.Kind, Path: value.LocalPath, ProducerRunID: value.ProducerRunID, Evidence: evidence})
}
if record.Status == manifest.StatusSucceeded {
for _, value := range record.Outputs {
add(value, "stage_result")
}
}
if record.HasVersionedTranscribeState() {
keys := make([]string, 0, len(record.TranscribeSpeakers))
for key := range record.TranscribeSpeakers {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
add(record.TranscribeSpeakers[key].Output, "speaker_checkpoint")
}
}
if record.HasVersionedAnalyzeState() {
keys := make([]string, 0, len(record.AnalyzeArtifacts))
for key := range record.AnalyzeArtifacts {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
value := record.AnalyzeArtifacts[key]
if value.Status == manifest.AnalyzeArtifactCurrent && value.Output != nil {
add(*value.Output, "artifact_checkpoint")
}
}
}
}
return out
}

View File

@@ -0,0 +1,94 @@
package app
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type terminalCountingStore struct {
manifest.Store
terminalSaves int
terminalErr error
}
func (s *terminalCountingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
if m.LastError != nil {
s.terminalSaves++
if s.terminalErr != nil {
return s.terminalErr
}
}
return s.Store.Save(ctx, path, m)
}
func TestPartialRunRetainsAllFailureCausesAndDurableSuccess(t *testing.T) {
for _, compositionFailure := range []bool{false, true} {
for _, saveFails := range []bool{false, true} {
t.Run(map[bool]string{false: "stage", true: "composition"}[compositionFailure]+map[bool]string{false: " saved", true: " save failed"}[saveFails], func(t *testing.T) {
cfg := testConfig(t)
cause := errors.New("operation unavailable")
saveErr := errors.New("terminal persistence unavailable")
releaseErr := errors.New("lock release failed")
store := &terminalCountingStore{Store: &manifest.LocalStore{}}
if saveFails {
store.terminalErr = saveErr
}
local := &releaseErrorArtifactStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root), err: releaseErr}
firstRuns := 0
stages := []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, failingStage{name: "transcribe", err: cause}, countingStage{name: "merge", runs: new(int)}}
deps := invocationDependencies{newWhisperX: func(*config.Config) (whisperx.Client, error) {
if compositionFailure {
return nil, cause
}
return &whisperx.NoopClient{}, nil
}}
summary, err := executePlanWithDependencies(context.Background(), cfg, BoundedPlan{stages: stages}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local}}, deps)
if !errors.Is(err, cause) || !errors.Is(err, releaseErr) || (saveFails && !errors.Is(err, saveErr)) {
t.Fatalf("lost causes: %v", err)
}
if store.terminalSaves != 1 {
t.Fatalf("terminal saves=%d", store.terminalSaves)
}
if summary == nil || summary.Complete || len(summary.StageResults) != 3 {
t.Fatalf("summary=%#v", summary)
}
if compositionFailure {
for _, result := range summary.StageResults {
if result.Attempted || result.Outcome != "not_attempted" {
t.Fatalf("composition scheduled work: %#v", summary)
}
}
return
}
if firstRuns != 1 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" {
t.Fatalf("partial results=%#v", summary)
}
durable, err := store.Load(context.Background(), summary.ManifestPath)
if err != nil {
t.Fatal(err)
}
if durable.Stages["prepare"].Status != manifest.StatusSucceeded {
t.Fatal("earlier success lost")
}
if saveFails {
if durable.Stages["transcribe"].Status != manifest.StatusRunning {
t.Fatal("failed terminal save advertised durable failure")
}
} else if durable.Stages["transcribe"].Status != manifest.StatusFailed {
t.Fatal("failed attempt not persisted")
}
retry, err := executeStages(context.Background(), cfg, []stage.Stage{countingStage{name: "prepare", runs: &firstRuns}, countingStage{name: "transcribe", runs: new(int)}}, RunOptions{})
if err != nil || firstRuns != 1 || retry.StageResults[0].Outcome != "reused" || !retry.Complete {
t.Fatalf("retry=%#v runs=%d err=%v", retry, firstRuns, err)
}
})
}
}
}

View File

@@ -28,6 +28,8 @@ type invocationRuntime struct {
artifactStore artifacts.Store
manifestPath string
manifest *manifest.Manifest
started bool
stageResults []RunStageResult
stageNames []string
executed []string
skipped []string
@@ -159,6 +161,7 @@ func initializeLockedInvocation(runtime *invocationRuntime) error {
if err := validateBoundedPrerequisites(runtime.plan, m); err != nil {
return fmt.Errorf("validate bounded run prerequisites under session lock: %w", err)
}
runtime.started = true
runtime.identity.applyToSessionManifest(m)
applyEffectiveConfigProvenance(m, runtime.cfg)
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
@@ -176,14 +179,3 @@ func (runtime *invocationRuntime) ensureRunWorkspaceFor(selected stage.Stage) er
}
return nil
}
func (runtime *invocationRuntime) persistTerminalFailure(err error) error {
return persistSessionTerminalFailure(
runtime.ctx,
runtime.manifestStore,
runtime.manifestPath,
runtime.manifest,
err,
runtime.dependencies.now(),
)
}

View File

@@ -10,40 +10,45 @@ import (
type stageExecutionOutcome struct {
executed bool
skipped bool
reused bool
reason string
}
func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (stageExecutionOutcome, error) {
func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (outcome stageExecutionOutcome, resultErr error) {
defer func() {
if resultErr == nil || !outcome.executed {
return
}
failedAt := runtime.dependencies.now()
runtime.manifest.MarkStageFailed(selected.Name(), failedAt, resultErr.Error())
_, invalidationErr := invalidateDependentSucceededStagesWithReason(runtime.manifest, selected.Name(), failedAt, staleReasonFailure)
if invalidationErr != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
}
}()
env := runtime.env
m := runtime.manifest
env.Force = runtime.opts.Force
semanticConfig, err := currentStageSemanticConfig(selected, env)
if err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("fingerprint semantic configuration for stage %q: %w", selected.Name(), err)
}
decision := decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, nil)
if decision.action == stageActionSkip {
validation, err := evaluateStageResume(runtime.ctx, selected, env, m, semanticConfig)
if err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("validate resume for stage %q: %w", selected.Name(), err)
}
decision = decideStageExecution(recordedStageStatus(m, selected.Name()), runtime.opts.Force, validation)
if decision.markSelectedStale {
staleAt := runtime.dependencies.now()
if _, err := applyPreExecutionDecision(m, selected.Name(), decision, staleAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents after resume validation for stage %q: %w", selected.Name(), err)
}
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("save manifest after resume validation for stage %q: %w", selected.Name(), err)
}
env.Logger.Info("stage result is not resumable", "stage", selected.Name(), "reason", decision.selectedStaleReason)
}
@@ -51,28 +56,22 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta
if decision.action == stageActionSkip {
env.Logger.Info("skipping stage", "stage", selected.Name(), "reason", "already_succeeded", "force", runtime.opts.Force)
return stageExecutionOutcome{skipped: true}, nil
return stageExecutionOutcome{skipped: true, reused: true, reason: "already_succeeded"}, nil
}
if err := runtime.ensureRunWorkspaceFor(selected); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(err)
return outcome, err
}
priorOutcome := capturePriorStageOutcome(m, selected.Name())
attempted := stageExecutionOutcome{executed: true}
outcome.executed = true
startedAt := runtime.dependencies.now()
if _, err := beginStageAttempt(m, selected.Name(), decision, startedAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents before forced stage %q: %w", selected.Name(), err)
}
env.Logger.Info("starting stage", "stage", selected.Name())
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
operationErr := fmt.Errorf("save manifest before stage %q: %w", selected.Name(), err)
m.MarkStageFailed(selected.Name(), runtime.dependencies.now(), operationErr.Error())
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), runtime.dependencies.now(), staleReasonFailure); invalidationErr != nil {
operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr))
}
return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr)
return outcome, operationErr
}
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "running", "path", runtime.manifestPath)
@@ -81,49 +80,33 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (sta
err = validateStageResult(result)
}
if err != nil {
failedAt := runtime.dependencies.now()
m.MarkStageFailed(selected.Name(), failedAt, err.Error())
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil {
err = errors.Join(err, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
}
env.Logger.Info("stage failed", "stage", selected.Name(), "error", err)
return stageExecutionOutcome{}, runtime.persistTerminalFailure(fmt.Errorf("stage %q failed: %w", selected.Name(), err))
return outcome, fmt.Errorf("stage %q failed: %w", selected.Name(), err)
}
if result != nil && result.Disposition == stage.StageDispositionSkipped {
skippedAt := runtime.dependencies.now()
if _, err := applyCompletedStageOutcome(m, selected.Name(), result, nil, semanticConfig, priorOutcome, skippedAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents after stage %q self-skip: %w", selected.Name(), err)
}
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("save manifest after self-skip %q: %w", selected.Name(), err)
}
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "skipped", "path", runtime.manifestPath)
env.Logger.Info("stage skipped", "stage", selected.Name(), "reason", result.SkipReason)
attempted.skipped = true
return attempted, nil
outcome.skipped = true
outcome.reason = result.SkipReason
return outcome, nil
}
outputs := mapResultOutputs(selected.Name(), result, runtime.identity.RunID)
succeededAt := runtime.dependencies.now()
if _, err := applyCompletedStageOutcome(m, selected.Name(), result, outputs, semanticConfig, priorOutcome, succeededAt); err != nil {
return stageExecutionOutcome{}, runtime.persistTerminalFailure(
fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err),
)
return outcome, fmt.Errorf("invalidate dependents after changed stage %q result: %w", selected.Name(), err)
}
if err := runtime.manifestStore.Save(runtime.ctx, runtime.manifestPath, m); err != nil {
operationErr := fmt.Errorf("save manifest after stage %q: %w", selected.Name(), err)
failedAt := runtime.dependencies.now()
m.MarkStageFailed(selected.Name(), failedAt, operationErr.Error())
if _, invalidationErr := invalidateDependentSucceededStagesWithReason(m, selected.Name(), failedAt, staleReasonFailure); invalidationErr != nil {
operationErr = errors.Join(operationErr, fmt.Errorf("invalidate dependents after stage %q persistence failure: %w", selected.Name(), invalidationErr))
}
return stageExecutionOutcome{}, runtime.persistTerminalFailure(operationErr)
return outcome, operationErr
}
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "succeeded", "path", runtime.manifestPath)
env.Logger.Info("stage succeeded", "stage", selected.Name())
return attempted, nil
return outcome, nil
}

View File

@@ -18,10 +18,11 @@ type terminalSessionStore struct {
}
type successfulStageSaveFailingStore struct {
delegate manifest.Store
stageName string
dependent string
failed bool
delegate manifest.Store
stageName string
failStatus manifest.StageStatus
dependent string
failed bool
}
func (s *successfulStageSaveFailingStore) Create(ctx context.Context, sessionID string) (*manifest.Manifest, error) {
@@ -34,7 +35,11 @@ func (s *successfulStageSaveFailingStore) Load(ctx context.Context, path string)
func (s *successfulStageSaveFailingStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
record := m.Stages[s.stageName]
if !s.failed && record != nil && record.Status == manifest.StatusSucceeded {
status := s.failStatus
if status == "" {
status = manifest.StatusSucceeded
}
if !s.failed && record != nil && record.Status == status {
s.failed = true
m.MarkStageSucceeded(s.dependent, time.Now().UTC(), nil)
return errors.New("injected successful-stage save failure")
@@ -166,3 +171,19 @@ func TestExecuteStagesSuccessfulSaveFailureMarksStageFailedAndInvalidatesDepende
t.Fatalf("normalize stage = %#v, want stale", got)
}
}
func TestExecuteStagesSelfSkipSaveFailureIsNotReportedComplete(t *testing.T) {
cfg := testConfig(t)
store := &successfulStageSaveFailingStore{delegate: &manifest.LocalStore{}, stageName: "polish", dependent: "normalize", failStatus: manifest.StatusSkipped}
summary, err := executeStages(context.Background(), cfg, []stage.Stage{resultStage{name: "polish", result: &stage.StageResult{Disposition: stage.StageDispositionSkipped, SkipReason: "disabled"}}}, RunOptions{Env: &Env{ManifestStore: store}})
if err == nil || summary == nil || summary.Complete || len(summary.Skipped) != 0 || summary.StageResults[0].Outcome != "failed" {
t.Fatalf("summary=%#v error=%v", summary, err)
}
saved, err := store.Load(context.Background(), summary.ManifestPath)
if err != nil {
t.Fatal(err)
}
if saved.Stages["polish"].Status != manifest.StatusFailed || saved.Stages["normalize"].Status != manifest.StatusStale {
t.Fatalf("unpersisted skip preserved as completion: %#v", saved.Stages)
}
}

View File

@@ -499,7 +499,7 @@ func TestExecuteStagesPublishSkipsRequiredUnselectedConfiguredOutput(t *testing.
},
},
}
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts}
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: cfg.Pipeline.Scriptorium.Artifacts, PublishOutputs: cfg.Pipeline.Publish.Outputs}
cfg.ArtifactCatalog.Artifacts["player_handout"] = config.ScriptoriumArtifactConfig{PromptID: "handout", OutputPath: "artifacts/handout.md"}
store := &manifest.LocalStore{}
@@ -860,8 +860,8 @@ func TestExecuteStagesFailureUpdatesManifest(t *testing.T) {
if err == nil {
t.Fatal("expected error, got nil")
}
if summary != nil {
t.Fatalf("summary = %#v, want nil on failure", summary)
if summary == nil || summary.Complete || len(summary.StageResults) != 3 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" || !summary.StageResults[1].Attempted || summary.StageResults[2].Outcome != "not_attempted" {
t.Fatalf("incorrect partial summary: %#v", summary)
}
if !strings.Contains(err.Error(), "stage \"transcribe\" failed") {
t.Fatalf("error = %q, want stage failure", err.Error())
@@ -1191,8 +1191,8 @@ func TestExecuteStagesRejectsSkippedResultWithOutputs(t *testing.T) {
if err == nil {
t.Fatal("executeStages() error = nil, want invalid skipped result failure")
}
if summary != nil {
t.Fatalf("summary = %#v, want nil", summary)
if summary == nil || summary.Complete || len(summary.StageResults) != 1 || summary.StageResults[0].Outcome != "failed" {
t.Fatalf("incorrect partial summary: %#v", summary)
}
if !strings.Contains(err.Error(), "skipped result contains 1 output") {
t.Fatalf("error = %q, want skipped-output validation", err)

View File

@@ -88,11 +88,24 @@ func TestExecuteStagesRetriesOnlyUncommittedTranscribeSpeakers(t *testing.T) {
}}
transcribe := mustSelectStage(t, "transcribe")
if _, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
summary, err := executeStages(context.Background(), cfg, []stage.Stage{transcribe}, RunOptions{
Env: &Env{ManifestStore: baseStore, WhisperX: failing},
}); !errors.Is(err, cause) {
})
if !errors.Is(err, cause) {
t.Fatalf("failed executeStages() error = %v, want %v", err, cause)
}
if summary == nil || summary.Complete || summary.StageResults[0].Outcome != "failed" {
t.Fatalf("partial report = %#v", summary)
}
checkpoints := 0
for _, artifact := range summary.AvailableArtifacts {
if artifact.Evidence == "speaker_checkpoint" {
checkpoints++
}
}
if checkpoints != 2 {
t.Fatalf("reported speaker checkpoints = %d, want 2", checkpoints)
}
failed := loadTranscribeRecoveryManifest(t, baseStore, cfg)
if failed.Stages["transcribe"].Status != manifest.StatusFailed {
t.Fatalf("transcribe status = %q, want failed", failed.Stages["transcribe"].Status)

View File

@@ -104,8 +104,8 @@ func TestExecuteStagesDefaultWiringWhisperXFailureMarksManifestFailed(t *testing
if err == nil {
t.Fatal("expected error, got nil")
}
if summary != nil {
t.Fatalf("summary = %#v, want nil on failure", summary)
if summary == nil || summary.Complete || len(summary.StageResults) != 2 || summary.StageResults[0].Outcome != "completed" || summary.StageResults[1].Outcome != "failed" {
t.Fatalf("incorrect partial summary: %#v", summary)
}
if calls.Load() != 2 {
t.Fatalf("whisperx calls = %d, want 2 (retry + initial)", calls.Load())