Propagate cancellation and preserve resumable state

This commit is contained in:
2026-11-20 05:53:49 +00:00
parent a6c6f1e4ab
commit c7a5ff316d
21 changed files with 356 additions and 27 deletions

View File

@@ -92,6 +92,9 @@ func cleanSelectedSession(ctx context.Context, input *cleanCommandInput, deps co
return report, err
}
appendTarget := func(root, target, policy string, directory bool) error {
if err := ctx.Err(); err != nil {
return err
}
action, err := cleanScopedTarget(root, target, policy, directory, input.DryRun)
report.Actions = append(report.Actions, action)
return err
@@ -100,7 +103,7 @@ func cleanSelectedSession(ctx context.Context, input *cleanCommandInput, deps co
if err := appendTarget(pipeline.Workspace.Root, filepath.Join(pipeline.Workspace.Root, config.PathWorkDirSegment), "clean.workspace.all", true); err != nil {
return report, err
}
actions, err := cleanRootChildren(pipeline.Spool.Root, "clean.spool.all", input.DryRun)
actions, err := cleanRootChildren(ctx, pipeline.Spool.Root, "clean.spool.all", input.DryRun)
report.Actions = append(report.Actions, actions...)
if err != nil {
return report, err
@@ -173,7 +176,7 @@ func cleanScopedTarget(root, target, policy string, directory, dryRun bool) (cle
return action, nil
}
func cleanRootChildren(root, policy string, dryRun bool) ([]cleanAction, error) {
func cleanRootChildren(ctx context.Context, root, policy string, dryRun bool) ([]cleanAction, error) {
rootAbs, entries, err := cleanableRootChildren(root, policy)
if err != nil {
return []cleanAction{{Path: root, Status: "failed", Error: err.Error()}}, err
@@ -183,6 +186,9 @@ func cleanRootChildren(root, policy string, dryRun bool) ([]cleanAction, error)
}
actions := make([]cleanAction, 0, len(entries))
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return actions, err
}
action := cleanAction{Path: entry, Status: "would_delete"}
if !dryRun {
if err := fileops.RemoveAllUnderRoot(rootAbs, entry); err != nil {
@@ -213,6 +219,9 @@ func cleanSessionCache(ctx context.Context, cfg *config.Config, dryRun bool, dep
}
actions := []cleanAction{}
for _, object := range objects {
if err := ctx.Err(); err != nil {
return actions, err
}
key := strings.TrimSpace(object.Key)
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
continue

View File

@@ -131,7 +131,11 @@ func loadRemoteSessionConfig(ctx context.Context, base *config.LoadedPipelineCam
return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey)
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}

View File

@@ -69,7 +69,7 @@ func (a *configurationAcquisition) bootstrap(ctx context.Context) error {
a.deps.secretsBootstrap.selection = selectedString(a.selection.SecretsDir)
a.deps.secretsBootstrap.selectionSet = true
}
return a.deps.secretsBootstrap.load(a.deps.loadSecretsDir, a.deps.diagnostics)
return errors.Join(a.deps.secretsBootstrap.load(a.deps.loadSecretsDir, a.deps.diagnostics), ctx.Err())
}
func (a *configurationAcquisition) Pipeline(ctx context.Context) (*config.PipelineConfig, error) {
@@ -82,12 +82,10 @@ func (a *configurationAcquisition) Pipeline(ctx context.Context) (*config.Pipeli
path, cfg, err := loadPipelineConfigWithDependencies(a.selection.Config.Value,
config.PipelineLoadOptions{Overlay: selectedString(a.selection.Overlay), Overrides: a.pipelineOverrides},
selectedString(a.selection.SecretsDir), a.deps)
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
a.pipeline, a.pipelinePath = cfg, path
return cfg, nil
}
@@ -115,12 +113,10 @@ func (a *configurationAcquisition) Campaign(ctx context.Context) (*config.Campai
return nil, err
}
cfg, err := config.LoadCampaign(path)
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
if a.selection.Campaign.Set && config.CampaignID(cfg) != a.selection.Campaign.Value {
return nil, &config.SessionIdentityError{Source: path, Field: "campaign_id", Expected: a.selection.Campaign.Value, Actual: config.CampaignID(cfg)}
}
@@ -144,12 +140,10 @@ func (a *configurationAcquisition) Party(ctx context.Context) (*config.ResolvedP
}
}
party, err := config.ResolveParty(a.campaignPath, campaign, selectedString(a.selection.PartyFile))
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
a.party = &party
return a.party, nil
}
@@ -175,16 +169,15 @@ func (a *configurationAcquisition) Session(ctx context.Context) (*config.Session
return nil, err
}
session, err := config.LoadSessionWithOptions(path, opts)
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
a.session = &acquiredSession{Config: session, Path: path, Source: config.SessionSource{Source: "session_config", LocalPath: path}}
return session, nil
}
session, path, err := config.DiscoverSession(config.DefaultSessionConfigSearchPaths, opts)
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}
@@ -204,6 +197,7 @@ func (a *configurationAcquisition) Session(ctx context.Context) (*config.Session
}
base.Pipeline, base.PipelinePath = pipeline, a.pipelinePath
loaded, err := loadRemoteSessionConfig(ctx, base, opts, a.deps)
err = errors.Join(err, ctx.Err())
if err != nil {
if loaded != nil && loaded.cleanup != nil {
err = errors.Join(err, loaded.cleanup())
@@ -246,6 +240,7 @@ func (a *configurationAcquisition) Domain(ctx context.Context) (*config.Config,
}
base := config.LoadedPipelineCampaign{PipelinePath: a.pipelinePath, Pipeline: pipeline, CampaignPath: a.campaignPath, Campaign: campaign, Party: *party, ArtifactCatalog: catalog}
cfg, err := config.ResolveLoadedPipelineCampaign(base, a.session.Path, session, a.session.Source)
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}
@@ -279,6 +274,7 @@ func (a *configurationAcquisition) Catalog(ctx context.Context) (*config.Artifac
party = *selected
}
catalog, err := config.DeriveArtifactCatalog(pipeline, party)
err = errors.Join(err, ctx.Err())
if err != nil {
return nil, err
}

View File

@@ -196,3 +196,20 @@ func TestSessionAcquisitionPreservesRemoteFailureCause(t *testing.T) {
t.Fatal(err)
}
}
func TestConfigurationAcquisitionStopsAfterSynchronousCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
loadErr := errors.New("native configuration load failed")
acquired := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection("synthetic.yaml")}, commandDependencies{loadPipelineConfig: func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error) {
cancel()
return nil, loadErr
}})
_, err := acquired.Pipeline(ctx)
if !errors.Is(err, context.Canceled) || !errors.Is(err, loadErr) || operationOutcome(err).Status != 1 {
t.Fatalf("lost canceled native load causes: %v", err)
}
if acquired.pipeline != nil {
t.Fatal("cached a canceled native load")
}
}

View File

@@ -132,6 +132,9 @@ func executeSelectedPipeline(ctx context.Context, input *executionInput, deps co
if err != nil {
return nil, &inputError{message: err.Error(), cause: err}
}
if err := ctx.Err(); err != nil {
return nil, err
}
summary, err := executor(ctx, cfg, input.plan, RunOptions{Force: input.Force.Value, SelectedArtifacts: selected, EffectiveArtifacts: effective})
if summary == nil {
if err == nil {

View File

@@ -93,6 +93,9 @@ func planSelectedPipeline(ctx context.Context, input *executionInput, deps comma
}
func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionInput, selectedArtifacts []string, effective artifacts.EffectiveArtifactSet) (*sessionPlanReport, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
m, err := loadManifestIfPresent(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("plan: %w", err)
@@ -114,6 +117,9 @@ func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionI
report := &sessionPlanReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: cfg.Session.SessionID, Stages: []plannedStage{}, Publication: []plannedPublication{}}
for _, selectedStage := range stages {
if err := ctx.Err(); err != nil {
return report, err
}
semanticConfig, err := currentStageSemanticConfig(selectedStage, stageEnv)
if err != nil {
return nil, fmt.Errorf("plan: fingerprint semantic configuration for stage %q: %w", selectedStage.Name(), err)
@@ -176,5 +182,5 @@ func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionI
report.Publication = append(report.Publication, plannedPublication{Source: action.Source, Destination: action.Destination, Required: action.Required, Selected: action.Selected})
}
}
return report, nil
return report, ctx.Err()
}

View File

@@ -129,6 +129,7 @@ func restoreSessionState(ctx context.Context, cfg *config.Config, opts RestorePl
}
result, err := deps.executeRestorePlan(ctx, cfg, current, plan, report, objectStore)
err = errors.Join(err, ctx.Err())
if err != nil {
report.setFailed(err)
if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil {

View File

@@ -33,6 +33,9 @@ func executeRestorePlan(
report *RestoreReport,
store storage.ObjectStore,
) (*RestoreExecutionResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return nil, fmt.Errorf("resolved config with pipeline/session is required")
}
@@ -100,6 +103,9 @@ func executeRestoreDownloadAction(
action RestoreAction,
store storage.ObjectStore,
) error {
if err := ctx.Err(); err != nil {
return err
}
safeLocalPath, err := joinWithinSessionRoot(sessionRoot, action.LocalRelativePath)
if err != nil {
return fmt.Errorf("resolve safe local path: %w", err)
@@ -165,6 +171,9 @@ func executeRestoreDownloadAction(
if err := validateRestoreDestinationLeaf(parent, leaf); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
if err := fileops.InstallSiblingTempFile(temporaryPath, safeLocalPath, fileops.WorkspaceFileMode); err != nil {
return fmt.Errorf("install file atomically: %w", err)
}

View File

@@ -72,6 +72,9 @@ type RestorePlanOptions struct {
}
func buildRestorePlan(ctx context.Context, cfg *config.Config, current *RemoteCurrentState, store storage.ObjectStore, opts RestorePlanOptions) (*RestorePlan, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
return nil, fmt.Errorf("resolved config with pipeline/session is required")
}

View File

@@ -88,6 +88,9 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
}
for index, selected := range runtime.stages {
if err := ctx.Err(); err != nil {
return nil, err
}
outcome, err := executeSelectedStage(runtime, selected)
result := &runtime.stageResults[index]
result.Attempted = outcome.executed
@@ -112,7 +115,7 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
return nil, err
}
}
return nil, nil
return nil, ctx.Err()
}
func stagesContainAny(stages []stage.Stage, names ...string) bool {
@@ -147,7 +150,9 @@ func persistSessionTerminalFailure(
sessionManifest.RecordFailure(at, operationErr.Error())
}
sessionErr := sessionStore.Save(ctx, sessionManifestPath, sessionManifest)
// Only terminal local persistence is detached; adapters and checkpoints keep
// the canceled operation context.
sessionErr := sessionStore.Save(context.WithoutCancel(ctx), sessionManifestPath, sessionManifest)
return errors.Join(
operationErr,
wrapTerminalPersistenceError("save terminal session manifest", sessionErr),

View File

@@ -0,0 +1,152 @@
package app
import (
"context"
"errors"
"slices"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type cancellationTerminalStore struct {
manifest.Store
terminalCalls int
terminalErr error
value any
}
func (s *cancellationTerminalStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
if m.LastError != nil {
s.terminalCalls++
if ctx.Err() != nil || ctx.Value(cancellationContextKey{}) != s.value {
return errors.New("terminal save lost its detached invocation context")
}
if s.terminalErr != nil {
return s.terminalErr
}
}
return s.Store.Save(ctx, path, m)
}
type cancellationContextKey struct{}
func TestCancellationFinalizesCheckpointsAndStopsLaterWork(t *testing.T) {
for _, saveFails := range []bool{false, true} {
t.Run(map[bool]string{false: "saved", true: "save failure"}[saveFails], func(t *testing.T) {
cfg, base := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"})
concurrency := 1
cfg.Pipeline.WhisperX.Concurrency = &concurrency
ctx, cancel := context.WithCancel(context.WithValue(context.Background(), cancellationContextKey{}, "invocation-value"))
defer cancel()
saveErr := errors.New("terminal save unavailable")
shutdownErr := errors.New("adapter shutdown failed")
store := &cancellationTerminalStore{Store: base, value: "invocation-value"}
if saveFails {
store.terminalErr = saveErr
}
local := &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
fake := &whisperx.FakeClient{TranscribeFn: func(adapterCtx context.Context, request whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
switch request.SpeakerID {
case "a":
return writeTranscribeRecoveryResult(request)
case "b":
cancel()
<-adapterCtx.Done()
return whisperx.TranscribeResult{}, errors.Join(adapterCtx.Err(), shutdownErr)
default:
t.Fatal("dispatched a speaker after cancellation")
return whisperx.TranscribeResult{}, nil
}
}}
later := 0
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "transcribe"), countingStage{name: "merge", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local, WhisperX: fake}})
if !errors.Is(err, context.Canceled) || !errors.Is(err, shutdownErr) || (saveFails && !errors.Is(err, saveErr)) {
t.Fatalf("lost cancellation/finalization cause: %v", err)
}
if operationOutcome(err).Status != 1 || summary == nil || summary.Complete || !local.released || store.terminalCalls != 1 || later != 0 {
t.Fatalf("summary=%#v released=%v saves=%d later=%d err=%v", summary, local.released, store.terminalCalls, later, err)
}
if summary.StageResults[0].Outcome != "failed" || summary.StageResults[1].Outcome != "not_attempted" || summary.StageResults[2].Outcome != "not_attempted" {
t.Fatalf("outcomes=%#v", summary.StageResults)
}
checkpointed := 0
for _, item := range summary.AvailableArtifacts {
if item.Evidence == "speaker_checkpoint" {
checkpointed++
}
}
if checkpointed != 1 {
t.Fatalf("durable checkpoint report=%#v", summary.AvailableArtifacts)
}
saved := loadTranscribeRecoveryManifest(t, base, cfg)
if !slices.Equal(transcribeSpeakerKeys(saved), []string{"a"}) {
t.Fatal("successful speaker checkpoint lost")
}
if !saveFails && (saved.Stages["transcribe"].Status != manifest.StatusFailed || saved.LastError == nil) {
t.Fatal("terminal manifest was not saved with cancelled context")
}
retry := &whisperx.FakeClient{}
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "transcribe")}, RunOptions{Env: &Env{ManifestStore: base, WhisperX: retry}})
if err != nil || !resumed.Complete || !slices.Equal(requestedTranscribeSpeakers(retry), []string{"b", "c"}) {
t.Fatalf("retry=%#v requests=%v err=%v", resumed, requestedTranscribeSpeakers(retry), err)
}
})
}
}
type cancelingAnalysisRunner struct {
analyzeCheckpointRunner
cancel context.CancelFunc
}
func (r *cancelingAnalysisRunner) RunArtifact(ctx context.Context, req scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) {
if len(r.Calls) == 1 {
r.cancel()
return scriptorium.ArtifactResult{}, ctx.Err()
}
return r.analyzeCheckpointRunner.RunArtifact(ctx, req)
}
func TestCancellationRetainsArtifactCheckpointForResume(t *testing.T) {
cfg := testConfig(t)
definitions := map[string]config.ScriptoriumArtifactConfig{
"alpha": {Enabled: true, PromptID: "alpha", OutputPath: "artifacts/alpha.md"},
"beta": {Enabled: true, PromptID: "beta", OutputPath: "artifacts/beta.md"},
"gamma": {Enabled: true, PromptID: "gamma", OutputPath: "artifacts/gamma.md"},
}
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: definitions}
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: definitions}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runner := &cancelingAnalysisRunner{cancel: cancel}
later := 0
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "analyze"), countingStage{name: "publish", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{Scriptorium: runner}})
if !errors.Is(err, context.Canceled) || summary == nil || summary.Complete || later != 0 {
t.Fatalf("summary=%#v err=%v later=%d", summary, err, later)
}
store := &manifest.LocalStore{}
durable := loadAnalyzeRecoveryManifest(t, store, cfg)
if durable.Stages["analyze"].Status != manifest.StatusFailed || durable.Stages["analyze"].AnalyzeArtifacts["alpha"].Status != manifest.AnalyzeArtifactCurrent {
t.Fatalf("checkpoint lost: %#v", durable.Stages["analyze"])
}
checkpointed := 0
for _, artifact := range summary.AvailableArtifacts {
if artifact.Evidence == "artifact_checkpoint" && artifact.Source == "narratio.artifact.alpha" {
checkpointed++
}
}
if checkpointed != 1 {
t.Fatalf("partial artifacts=%#v", summary.AvailableArtifacts)
}
retry := &analyzeCheckpointRunner{}
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{Env: &Env{Scriptorium: retry}})
if err != nil || !resumed.Complete || !slices.Equal(retry.Calls, []string{"beta", "gamma"}) {
t.Fatalf("resume=%#v calls=%v err=%v", resumed, retry.Calls, err)
}
}

View File

@@ -11,6 +11,9 @@ func composeInvocation(runtime *invocationRuntime) error {
stages := runtime.stages
dependencies := runtime.dependencies
if err := runtime.ctx.Err(); err != nil {
return err
}
if env.WhisperX == nil && stagesContainAny(stages, "transcribe") {
client, err := dependencies.newWhisperX(env.Config)
if err != nil {
@@ -18,6 +21,9 @@ func composeInvocation(runtime *invocationRuntime) error {
}
env.WhisperX = client
}
if err := runtime.ctx.Err(); err != nil {
return err
}
if env.Seriatim == nil && stagesContainAny(stages, "merge", "normalize", "trim", "render") {
runner, err := dependencies.newSeriatim(env.Config)
if err != nil {
@@ -25,6 +31,9 @@ func composeInvocation(runtime *invocationRuntime) error {
}
env.Seriatim = runner
}
if err := runtime.ctx.Err(); err != nil {
return err
}
if env.Audita == nil && stagesContainAny(stages, "polish") {
runner, err := dependencies.newAudita(env.Config)
if err != nil {
@@ -32,12 +41,21 @@ func composeInvocation(runtime *invocationRuntime) error {
}
env.Audita = runner
}
if err := runtime.ctx.Err(); err != nil {
return err
}
if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
env.Notarius = dependencies.newNotarius()
}
if err := runtime.ctx.Err(); err != nil {
return err
}
if env.Scriptorium == nil && stagesContainAny(stages, "trim", "analyze") {
env.Scriptorium = dependencies.newScriptorium()
}
if err := runtime.ctx.Err(); err != nil {
return err
}
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages, runtime.effectiveArtifacts) {
objectStore, err := dependencies.newObjectStore(runtime.ctx, env.Config)
if err != nil {
@@ -45,8 +63,11 @@ func composeInvocation(runtime *invocationRuntime) error {
}
env.ObjectStore = objectStore
}
if err := runtime.ctx.Err(); err != nil {
return err
}
if env.Notifier == nil && stagesContainAny(stages, "notify") {
env.Notifier = &notify.NoopSender{}
}
return nil
return runtime.ctx.Err()
}

View File

@@ -1,6 +1,7 @@
package app
import (
"context"
"errors"
"fmt"
"sort"
@@ -34,7 +35,7 @@ func finalizeInvocation(runtime *invocationRuntime, operationErr error) (*RunSum
// Read durable authority after a failure: checkpoint saves can succeed
// even when terminal persistence fails.
var err error
evidence, err = runtime.manifestStore.Load(runtime.ctx, runtime.manifestPath)
evidence, err = runtime.manifestStore.Load(context.WithoutCancel(runtime.ctx), runtime.manifestPath)
if err != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("read durable run evidence: %w", err))
evidence = nil

View File

@@ -37,6 +37,9 @@ type invocationRuntime struct {
}
func prepareInvocation(ctx context.Context, cfg *config.Config, plan BoundedPlan, opts RunOptions, dependencies invocationDependencies) (*invocationRuntime, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var prerequisiteStore manifest.Store
if opts.Env != nil {
prerequisiteStore = opts.Env.ManifestStore
@@ -125,6 +128,9 @@ func prepareInvocation(ctx context.Context, cfg *config.Config, plan BoundedPlan
}
func prepareInvocationLayout(runtime *invocationRuntime) error {
if err := runtime.ctx.Err(); err != nil {
return err
}
paths, err := runtime.artifactStore.EnsureLayoutFor(runtime.identity.Campaign, runtime.identity.SessionID)
if err != nil {
return fmt.Errorf("prepare workdir: %w", err)

View File

@@ -26,6 +26,9 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out
resultErr = errors.Join(resultErr, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
}
}()
if err := runtime.ctx.Err(); err != nil {
return outcome, err
}
env := runtime.env
m := runtime.manifest
env.Force = runtime.opts.Force
@@ -54,6 +57,9 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out
}
}
if err := runtime.ctx.Err(); err != nil {
return outcome, err
}
if decision.action == stageActionSkip {
env.Logger.Info("skipping stage", "stage", selected.Name(), "reason", "already_succeeded", "force", runtime.opts.Force)
return stageExecutionOutcome{skipped: true, reused: true, reason: "already_succeeded"}, nil
@@ -75,7 +81,11 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out
}
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "running", "path", runtime.manifestPath)
if err := runtime.ctx.Err(); err != nil {
return outcome, err
}
result, err := selected.Run(runtime.ctx, env, m)
err = errors.Join(err, runtime.ctx.Err())
if err == nil {
err = validateStageResult(result)
}

View File

@@ -149,6 +149,7 @@ func initializeSession(ctx context.Context, request *sessionInitializationInput,
SessionID: sessionID,
PreviousSessionID: previousSessionID,
})
err = errors.Join(err, ctx.Err())
if err != nil {
return report, fmt.Errorf("session init: %w", err)
}
@@ -164,6 +165,9 @@ func initializeSession(ctx context.Context, request *sessionInitializationInput,
return report, fmt.Errorf("session init: render effective session: %w", err)
}
if err := ctx.Err(); err != nil {
return report, err
}
if !remote {
if err := writeLocalFile(output, data, force); err != nil {
return report, fmt.Errorf("session init: %w", err)

View File

@@ -46,6 +46,9 @@ type analyzeExecutionContext struct {
}
func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if env == nil || env.Config == nil {
return nil, fmt.Errorf("analyze: stage environment config is required")
}
@@ -138,6 +141,9 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
}
for _, item := range workPlan.ExecutionOrder {
if err := ctx.Err(); err != nil {
return nil, err
}
artifactCfg := env.Config.ConcreteArtifacts()[item.Key]
fingerprint, _, _, err := computeAnalyzeArtifactFingerprint(
item.Key,

View File

@@ -3,6 +3,7 @@ package stage
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@@ -86,6 +87,9 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
group, groupCtx := errgroup.WithContext(ctx)
group.SetLimit(concurrency)
for _, job := range jobs {
if groupCtx.Err() != nil {
break
}
job := job
group.Go(func() error {
if err := groupCtx.Err(); err != nil {
@@ -116,13 +120,10 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
return nil
})
}
groupErr := group.Wait()
if err := state.checkpointFailure(); err != nil {
return nil, err
}
groupErr := errors.Join(group.Wait(), state.checkpointFailure())
completedCount := state.completedCount()
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err)
return nil, errors.Join(fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err), groupErr)
}
if groupErr != nil {
return nil, groupErr