Propagate cancellation and preserve resumable state
This commit is contained in:
@@ -92,6 +92,9 @@ func cleanSelectedSession(ctx context.Context, input *cleanCommandInput, deps co
|
||||
return report, err
|
||||
}
|
||||
appendTarget := func(root, target, policy string, directory bool) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
action, err := cleanScopedTarget(root, target, policy, directory, input.DryRun)
|
||||
report.Actions = append(report.Actions, action)
|
||||
return err
|
||||
@@ -100,7 +103,7 @@ func cleanSelectedSession(ctx context.Context, input *cleanCommandInput, deps co
|
||||
if err := appendTarget(pipeline.Workspace.Root, filepath.Join(pipeline.Workspace.Root, config.PathWorkDirSegment), "clean.workspace.all", true); err != nil {
|
||||
return report, err
|
||||
}
|
||||
actions, err := cleanRootChildren(pipeline.Spool.Root, "clean.spool.all", input.DryRun)
|
||||
actions, err := cleanRootChildren(ctx, pipeline.Spool.Root, "clean.spool.all", input.DryRun)
|
||||
report.Actions = append(report.Actions, actions...)
|
||||
if err != nil {
|
||||
return report, err
|
||||
@@ -173,7 +176,7 @@ func cleanScopedTarget(root, target, policy string, directory, dryRun bool) (cle
|
||||
return action, nil
|
||||
}
|
||||
|
||||
func cleanRootChildren(root, policy string, dryRun bool) ([]cleanAction, error) {
|
||||
func cleanRootChildren(ctx context.Context, root, policy string, dryRun bool) ([]cleanAction, error) {
|
||||
rootAbs, entries, err := cleanableRootChildren(root, policy)
|
||||
if err != nil {
|
||||
return []cleanAction{{Path: root, Status: "failed", Error: err.Error()}}, err
|
||||
@@ -183,6 +186,9 @@ func cleanRootChildren(root, policy string, dryRun bool) ([]cleanAction, error)
|
||||
}
|
||||
actions := make([]cleanAction, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return actions, err
|
||||
}
|
||||
action := cleanAction{Path: entry, Status: "would_delete"}
|
||||
if !dryRun {
|
||||
if err := fileops.RemoveAllUnderRoot(rootAbs, entry); err != nil {
|
||||
@@ -213,6 +219,9 @@ func cleanSessionCache(ctx context.Context, cfg *config.Config, dryRun bool, dep
|
||||
}
|
||||
actions := []cleanAction{}
|
||||
for _, object := range objects {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return actions, err
|
||||
}
|
||||
key := strings.TrimSpace(object.Key)
|
||||
if key == "" || strings.HasSuffix(key, "/") || !cleanIsFlac(key) {
|
||||
continue
|
||||
|
||||
@@ -131,7 +131,11 @@ func loadRemoteSessionConfig(ctx context.Context, base *config.LoadedPipelineCam
|
||||
return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ func (a *configurationAcquisition) bootstrap(ctx context.Context) error {
|
||||
a.deps.secretsBootstrap.selection = selectedString(a.selection.SecretsDir)
|
||||
a.deps.secretsBootstrap.selectionSet = true
|
||||
}
|
||||
return a.deps.secretsBootstrap.load(a.deps.loadSecretsDir, a.deps.diagnostics)
|
||||
return errors.Join(a.deps.secretsBootstrap.load(a.deps.loadSecretsDir, a.deps.diagnostics), ctx.Err())
|
||||
}
|
||||
|
||||
func (a *configurationAcquisition) Pipeline(ctx context.Context) (*config.PipelineConfig, error) {
|
||||
@@ -82,12 +82,10 @@ func (a *configurationAcquisition) Pipeline(ctx context.Context) (*config.Pipeli
|
||||
path, cfg, err := loadPipelineConfigWithDependencies(a.selection.Config.Value,
|
||||
config.PipelineLoadOptions{Overlay: selectedString(a.selection.Overlay), Overrides: a.pipelineOverrides},
|
||||
selectedString(a.selection.SecretsDir), a.deps)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.pipeline, a.pipelinePath = cfg, path
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -115,12 +113,10 @@ func (a *configurationAcquisition) Campaign(ctx context.Context) (*config.Campai
|
||||
return nil, err
|
||||
}
|
||||
cfg, err := config.LoadCampaign(path)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.selection.Campaign.Set && config.CampaignID(cfg) != a.selection.Campaign.Value {
|
||||
return nil, &config.SessionIdentityError{Source: path, Field: "campaign_id", Expected: a.selection.Campaign.Value, Actual: config.CampaignID(cfg)}
|
||||
}
|
||||
@@ -144,12 +140,10 @@ func (a *configurationAcquisition) Party(ctx context.Context) (*config.ResolvedP
|
||||
}
|
||||
}
|
||||
party, err := config.ResolveParty(a.campaignPath, campaign, selectedString(a.selection.PartyFile))
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.party = &party
|
||||
return a.party, nil
|
||||
}
|
||||
@@ -175,16 +169,15 @@ func (a *configurationAcquisition) Session(ctx context.Context) (*config.Session
|
||||
return nil, err
|
||||
}
|
||||
session, err := config.LoadSessionWithOptions(path, opts)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.session = &acquiredSession{Config: session, Path: path, Source: config.SessionSource{Source: "session_config", LocalPath: path}}
|
||||
return session, nil
|
||||
}
|
||||
session, path, err := config.DiscoverSession(config.DefaultSessionConfigSearchPaths, opts)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -204,6 +197,7 @@ func (a *configurationAcquisition) Session(ctx context.Context) (*config.Session
|
||||
}
|
||||
base.Pipeline, base.PipelinePath = pipeline, a.pipelinePath
|
||||
loaded, err := loadRemoteSessionConfig(ctx, base, opts, a.deps)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
if loaded != nil && loaded.cleanup != nil {
|
||||
err = errors.Join(err, loaded.cleanup())
|
||||
@@ -246,6 +240,7 @@ func (a *configurationAcquisition) Domain(ctx context.Context) (*config.Config,
|
||||
}
|
||||
base := config.LoadedPipelineCampaign{PipelinePath: a.pipelinePath, Pipeline: pipeline, CampaignPath: a.campaignPath, Campaign: campaign, Party: *party, ArtifactCatalog: catalog}
|
||||
cfg, err := config.ResolveLoadedPipelineCampaign(base, a.session.Path, session, a.session.Source)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -279,6 +274,7 @@ func (a *configurationAcquisition) Catalog(ctx context.Context) (*config.Artifac
|
||||
party = *selected
|
||||
}
|
||||
catalog, err := config.DeriveArtifactCatalog(pipeline, party)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -196,3 +196,20 @@ func TestSessionAcquisitionPreservesRemoteFailureCause(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationAcquisitionStopsAfterSynchronousCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
loadErr := errors.New("native configuration load failed")
|
||||
acquired := newConfigurationAcquisition(configurationSelection{Config: suppliedSelection("synthetic.yaml")}, commandDependencies{loadPipelineConfig: func(string, config.PipelineLoadOptions) (*config.PipelineConfig, error) {
|
||||
cancel()
|
||||
return nil, loadErr
|
||||
}})
|
||||
_, err := acquired.Pipeline(ctx)
|
||||
if !errors.Is(err, context.Canceled) || !errors.Is(err, loadErr) || operationOutcome(err).Status != 1 {
|
||||
t.Fatalf("lost canceled native load causes: %v", err)
|
||||
}
|
||||
if acquired.pipeline != nil {
|
||||
t.Fatal("cached a canceled native load")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,6 +132,9 @@ func executeSelectedPipeline(ctx context.Context, input *executionInput, deps co
|
||||
if err != nil {
|
||||
return nil, &inputError{message: err.Error(), cause: err}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
summary, err := executor(ctx, cfg, input.plan, RunOptions{Force: input.Force.Value, SelectedArtifacts: selected, EffectiveArtifacts: effective})
|
||||
if summary == nil {
|
||||
if err == nil {
|
||||
|
||||
@@ -93,6 +93,9 @@ func planSelectedPipeline(ctx context.Context, input *executionInput, deps comma
|
||||
}
|
||||
|
||||
func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionInput, selectedArtifacts []string, effective artifacts.EffectiveArtifactSet) (*sessionPlanReport, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := loadManifestIfPresent(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("plan: %w", err)
|
||||
@@ -114,6 +117,9 @@ func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionI
|
||||
|
||||
report := &sessionPlanReport{Campaign: config.CampaignID(cfg.Campaign), SessionID: cfg.Session.SessionID, Stages: []plannedStage{}, Publication: []plannedPublication{}}
|
||||
for _, selectedStage := range stages {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
semanticConfig, err := currentStageSemanticConfig(selectedStage, stageEnv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("plan: fingerprint semantic configuration for stage %q: %w", selectedStage.Name(), err)
|
||||
@@ -176,5 +182,5 @@ func buildSessionPlan(ctx context.Context, cfg *config.Config, input *executionI
|
||||
report.Publication = append(report.Publication, plannedPublication{Source: action.Source, Destination: action.Destination, Required: action.Required, Selected: action.Selected})
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
return report, ctx.Err()
|
||||
}
|
||||
|
||||
@@ -129,6 +129,7 @@ func restoreSessionState(ctx context.Context, cfg *config.Config, opts RestorePl
|
||||
}
|
||||
|
||||
result, err := deps.executeRestorePlan(ctx, cfg, current, plan, report, objectStore)
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
report.setFailed(err)
|
||||
if _, reportErr := persistRestoreReport(artifactStore, cfg, report); reportErr != nil {
|
||||
|
||||
@@ -33,6 +33,9 @@ func executeRestorePlan(
|
||||
report *RestoreReport,
|
||||
store storage.ObjectStore,
|
||||
) (*RestoreExecutionResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return nil, fmt.Errorf("resolved config with pipeline/session is required")
|
||||
}
|
||||
@@ -100,6 +103,9 @@ func executeRestoreDownloadAction(
|
||||
action RestoreAction,
|
||||
store storage.ObjectStore,
|
||||
) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
safeLocalPath, err := joinWithinSessionRoot(sessionRoot, action.LocalRelativePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve safe local path: %w", err)
|
||||
@@ -165,6 +171,9 @@ func executeRestoreDownloadAction(
|
||||
if err := validateRestoreDestinationLeaf(parent, leaf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fileops.InstallSiblingTempFile(temporaryPath, safeLocalPath, fileops.WorkspaceFileMode); err != nil {
|
||||
return fmt.Errorf("install file atomically: %w", err)
|
||||
}
|
||||
|
||||
@@ -72,6 +72,9 @@ type RestorePlanOptions struct {
|
||||
}
|
||||
|
||||
func buildRestorePlan(ctx context.Context, cfg *config.Config, current *RemoteCurrentState, store storage.ObjectStore, opts RestorePlanOptions) (*RestorePlan, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg == nil || cfg.Pipeline == nil || cfg.Session == nil {
|
||||
return nil, fmt.Errorf("resolved config with pipeline/session is required")
|
||||
}
|
||||
|
||||
@@ -88,6 +88,9 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
|
||||
}
|
||||
|
||||
for index, selected := range runtime.stages {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outcome, err := executeSelectedStage(runtime, selected)
|
||||
result := &runtime.stageResults[index]
|
||||
result.Attempted = outcome.executed
|
||||
@@ -112,7 +115,7 @@ func executePlanWithDependencies(ctx context.Context, cfg *config.Config, plan B
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
func stagesContainAny(stages []stage.Stage, names ...string) bool {
|
||||
@@ -147,7 +150,9 @@ func persistSessionTerminalFailure(
|
||||
sessionManifest.RecordFailure(at, operationErr.Error())
|
||||
}
|
||||
|
||||
sessionErr := sessionStore.Save(ctx, sessionManifestPath, sessionManifest)
|
||||
// Only terminal local persistence is detached; adapters and checkpoints keep
|
||||
// the canceled operation context.
|
||||
sessionErr := sessionStore.Save(context.WithoutCancel(ctx), sessionManifestPath, sessionManifest)
|
||||
return errors.Join(
|
||||
operationErr,
|
||||
wrapTerminalPersistenceError("save terminal session manifest", sessionErr),
|
||||
|
||||
152
internal/app/runner_cancellation_test.go
Normal file
152
internal/app/runner_cancellation_test.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type cancellationTerminalStore struct {
|
||||
manifest.Store
|
||||
terminalCalls int
|
||||
terminalErr error
|
||||
value any
|
||||
}
|
||||
|
||||
func (s *cancellationTerminalStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if m.LastError != nil {
|
||||
s.terminalCalls++
|
||||
if ctx.Err() != nil || ctx.Value(cancellationContextKey{}) != s.value {
|
||||
return errors.New("terminal save lost its detached invocation context")
|
||||
}
|
||||
if s.terminalErr != nil {
|
||||
return s.terminalErr
|
||||
}
|
||||
}
|
||||
return s.Store.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
type cancellationContextKey struct{}
|
||||
|
||||
func TestCancellationFinalizesCheckpointsAndStopsLaterWork(t *testing.T) {
|
||||
for _, saveFails := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "saved", true: "save failure"}[saveFails], func(t *testing.T) {
|
||||
cfg, base := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"})
|
||||
concurrency := 1
|
||||
cfg.Pipeline.WhisperX.Concurrency = &concurrency
|
||||
ctx, cancel := context.WithCancel(context.WithValue(context.Background(), cancellationContextKey{}, "invocation-value"))
|
||||
defer cancel()
|
||||
saveErr := errors.New("terminal save unavailable")
|
||||
shutdownErr := errors.New("adapter shutdown failed")
|
||||
store := &cancellationTerminalStore{Store: base, value: "invocation-value"}
|
||||
if saveFails {
|
||||
store.terminalErr = saveErr
|
||||
}
|
||||
local := &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
|
||||
fake := &whisperx.FakeClient{TranscribeFn: func(adapterCtx context.Context, request whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
|
||||
switch request.SpeakerID {
|
||||
case "a":
|
||||
return writeTranscribeRecoveryResult(request)
|
||||
case "b":
|
||||
cancel()
|
||||
<-adapterCtx.Done()
|
||||
return whisperx.TranscribeResult{}, errors.Join(adapterCtx.Err(), shutdownErr)
|
||||
default:
|
||||
t.Fatal("dispatched a speaker after cancellation")
|
||||
return whisperx.TranscribeResult{}, nil
|
||||
}
|
||||
}}
|
||||
later := 0
|
||||
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "transcribe"), countingStage{name: "merge", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local, WhisperX: fake}})
|
||||
if !errors.Is(err, context.Canceled) || !errors.Is(err, shutdownErr) || (saveFails && !errors.Is(err, saveErr)) {
|
||||
t.Fatalf("lost cancellation/finalization cause: %v", err)
|
||||
}
|
||||
if operationOutcome(err).Status != 1 || summary == nil || summary.Complete || !local.released || store.terminalCalls != 1 || later != 0 {
|
||||
t.Fatalf("summary=%#v released=%v saves=%d later=%d err=%v", summary, local.released, store.terminalCalls, later, err)
|
||||
}
|
||||
if summary.StageResults[0].Outcome != "failed" || summary.StageResults[1].Outcome != "not_attempted" || summary.StageResults[2].Outcome != "not_attempted" {
|
||||
t.Fatalf("outcomes=%#v", summary.StageResults)
|
||||
}
|
||||
checkpointed := 0
|
||||
for _, item := range summary.AvailableArtifacts {
|
||||
if item.Evidence == "speaker_checkpoint" {
|
||||
checkpointed++
|
||||
}
|
||||
}
|
||||
if checkpointed != 1 {
|
||||
t.Fatalf("durable checkpoint report=%#v", summary.AvailableArtifacts)
|
||||
}
|
||||
saved := loadTranscribeRecoveryManifest(t, base, cfg)
|
||||
if !slices.Equal(transcribeSpeakerKeys(saved), []string{"a"}) {
|
||||
t.Fatal("successful speaker checkpoint lost")
|
||||
}
|
||||
if !saveFails && (saved.Stages["transcribe"].Status != manifest.StatusFailed || saved.LastError == nil) {
|
||||
t.Fatal("terminal manifest was not saved with cancelled context")
|
||||
}
|
||||
retry := &whisperx.FakeClient{}
|
||||
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "transcribe")}, RunOptions{Env: &Env{ManifestStore: base, WhisperX: retry}})
|
||||
if err != nil || !resumed.Complete || !slices.Equal(requestedTranscribeSpeakers(retry), []string{"b", "c"}) {
|
||||
t.Fatalf("retry=%#v requests=%v err=%v", resumed, requestedTranscribeSpeakers(retry), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type cancelingAnalysisRunner struct {
|
||||
analyzeCheckpointRunner
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (r *cancelingAnalysisRunner) RunArtifact(ctx context.Context, req scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) {
|
||||
if len(r.Calls) == 1 {
|
||||
r.cancel()
|
||||
return scriptorium.ArtifactResult{}, ctx.Err()
|
||||
}
|
||||
return r.analyzeCheckpointRunner.RunArtifact(ctx, req)
|
||||
}
|
||||
|
||||
func TestCancellationRetainsArtifactCheckpointForResume(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
definitions := map[string]config.ScriptoriumArtifactConfig{
|
||||
"alpha": {Enabled: true, PromptID: "alpha", OutputPath: "artifacts/alpha.md"},
|
||||
"beta": {Enabled: true, PromptID: "beta", OutputPath: "artifacts/beta.md"},
|
||||
"gamma": {Enabled: true, PromptID: "gamma", OutputPath: "artifacts/gamma.md"},
|
||||
}
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: definitions}
|
||||
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: definitions}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
runner := &cancelingAnalysisRunner{cancel: cancel}
|
||||
later := 0
|
||||
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "analyze"), countingStage{name: "publish", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{Scriptorium: runner}})
|
||||
if !errors.Is(err, context.Canceled) || summary == nil || summary.Complete || later != 0 {
|
||||
t.Fatalf("summary=%#v err=%v later=%d", summary, err, later)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
durable := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||
if durable.Stages["analyze"].Status != manifest.StatusFailed || durable.Stages["analyze"].AnalyzeArtifacts["alpha"].Status != manifest.AnalyzeArtifactCurrent {
|
||||
t.Fatalf("checkpoint lost: %#v", durable.Stages["analyze"])
|
||||
}
|
||||
checkpointed := 0
|
||||
for _, artifact := range summary.AvailableArtifacts {
|
||||
if artifact.Evidence == "artifact_checkpoint" && artifact.Source == "narratio.artifact.alpha" {
|
||||
checkpointed++
|
||||
}
|
||||
}
|
||||
if checkpointed != 1 {
|
||||
t.Fatalf("partial artifacts=%#v", summary.AvailableArtifacts)
|
||||
}
|
||||
retry := &analyzeCheckpointRunner{}
|
||||
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{Env: &Env{Scriptorium: retry}})
|
||||
if err != nil || !resumed.Complete || !slices.Equal(retry.Calls, []string{"beta", "gamma"}) {
|
||||
t.Fatalf("resume=%#v calls=%v err=%v", resumed, retry.Calls, err)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,9 @@ func composeInvocation(runtime *invocationRuntime) error {
|
||||
stages := runtime.stages
|
||||
dependencies := runtime.dependencies
|
||||
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if env.WhisperX == nil && stagesContainAny(stages, "transcribe") {
|
||||
client, err := dependencies.newWhisperX(env.Config)
|
||||
if err != nil {
|
||||
@@ -18,6 +21,9 @@ func composeInvocation(runtime *invocationRuntime) error {
|
||||
}
|
||||
env.WhisperX = client
|
||||
}
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if env.Seriatim == nil && stagesContainAny(stages, "merge", "normalize", "trim", "render") {
|
||||
runner, err := dependencies.newSeriatim(env.Config)
|
||||
if err != nil {
|
||||
@@ -25,6 +31,9 @@ func composeInvocation(runtime *invocationRuntime) error {
|
||||
}
|
||||
env.Seriatim = runner
|
||||
}
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if env.Audita == nil && stagesContainAny(stages, "polish") {
|
||||
runner, err := dependencies.newAudita(env.Config)
|
||||
if err != nil {
|
||||
@@ -32,12 +41,21 @@ func composeInvocation(runtime *invocationRuntime) error {
|
||||
}
|
||||
env.Audita = runner
|
||||
}
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if env.Notarius == nil && needsNotariusForRun(env.Config, stages) {
|
||||
env.Notarius = dependencies.newNotarius()
|
||||
}
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if env.Scriptorium == nil && stagesContainAny(stages, "trim", "analyze") {
|
||||
env.Scriptorium = dependencies.newScriptorium()
|
||||
}
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if env.ObjectStore == nil && needsObjectStoreForRun(env.Config, stages, runtime.effectiveArtifacts) {
|
||||
objectStore, err := dependencies.newObjectStore(runtime.ctx, env.Config)
|
||||
if err != nil {
|
||||
@@ -45,8 +63,11 @@ func composeInvocation(runtime *invocationRuntime) error {
|
||||
}
|
||||
env.ObjectStore = objectStore
|
||||
}
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if env.Notifier == nil && stagesContainAny(stages, "notify") {
|
||||
env.Notifier = ¬ify.NoopSender{}
|
||||
}
|
||||
return nil
|
||||
return runtime.ctx.Err()
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -34,7 +35,7 @@ func finalizeInvocation(runtime *invocationRuntime, operationErr error) (*RunSum
|
||||
// Read durable authority after a failure: checkpoint saves can succeed
|
||||
// even when terminal persistence fails.
|
||||
var err error
|
||||
evidence, err = runtime.manifestStore.Load(runtime.ctx, runtime.manifestPath)
|
||||
evidence, err = runtime.manifestStore.Load(context.WithoutCancel(runtime.ctx), runtime.manifestPath)
|
||||
if err != nil {
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("read durable run evidence: %w", err))
|
||||
evidence = nil
|
||||
|
||||
@@ -37,6 +37,9 @@ type invocationRuntime struct {
|
||||
}
|
||||
|
||||
func prepareInvocation(ctx context.Context, cfg *config.Config, plan BoundedPlan, opts RunOptions, dependencies invocationDependencies) (*invocationRuntime, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var prerequisiteStore manifest.Store
|
||||
if opts.Env != nil {
|
||||
prerequisiteStore = opts.Env.ManifestStore
|
||||
@@ -125,6 +128,9 @@ func prepareInvocation(ctx context.Context, cfg *config.Config, plan BoundedPlan
|
||||
}
|
||||
|
||||
func prepareInvocationLayout(runtime *invocationRuntime) error {
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
paths, err := runtime.artifactStore.EnsureLayoutFor(runtime.identity.Campaign, runtime.identity.SessionID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare workdir: %w", err)
|
||||
|
||||
@@ -26,6 +26,9 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("invalidate dependents after stage %q failure: %w", selected.Name(), invalidationErr))
|
||||
}
|
||||
}()
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return outcome, err
|
||||
}
|
||||
env := runtime.env
|
||||
m := runtime.manifest
|
||||
env.Force = runtime.opts.Force
|
||||
@@ -54,6 +57,9 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out
|
||||
}
|
||||
}
|
||||
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return outcome, err
|
||||
}
|
||||
if decision.action == stageActionSkip {
|
||||
env.Logger.Info("skipping stage", "stage", selected.Name(), "reason", "already_succeeded", "force", runtime.opts.Force)
|
||||
return stageExecutionOutcome{skipped: true, reused: true, reason: "already_succeeded"}, nil
|
||||
@@ -75,7 +81,11 @@ func executeSelectedStage(runtime *invocationRuntime, selected stage.Stage) (out
|
||||
}
|
||||
env.Logger.Debug("manifest saved", "stage", selected.Name(), "transition", "running", "path", runtime.manifestPath)
|
||||
|
||||
if err := runtime.ctx.Err(); err != nil {
|
||||
return outcome, err
|
||||
}
|
||||
result, err := selected.Run(runtime.ctx, env, m)
|
||||
err = errors.Join(err, runtime.ctx.Err())
|
||||
if err == nil {
|
||||
err = validateStageResult(result)
|
||||
}
|
||||
|
||||
@@ -149,6 +149,7 @@ func initializeSession(ctx context.Context, request *sessionInitializationInput,
|
||||
SessionID: sessionID,
|
||||
PreviousSessionID: previousSessionID,
|
||||
})
|
||||
err = errors.Join(err, ctx.Err())
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("session init: %w", err)
|
||||
}
|
||||
@@ -164,6 +165,9 @@ func initializeSession(ctx context.Context, request *sessionInitializationInput,
|
||||
return report, fmt.Errorf("session init: render effective session: %w", err)
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
if !remote {
|
||||
if err := writeLocalFile(output, data, force); err != nil {
|
||||
return report, fmt.Errorf("session init: %w", err)
|
||||
|
||||
@@ -46,6 +46,9 @@ type analyzeExecutionContext struct {
|
||||
}
|
||||
|
||||
func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if env == nil || env.Config == nil {
|
||||
return nil, fmt.Errorf("analyze: stage environment config is required")
|
||||
}
|
||||
@@ -138,6 +141,9 @@ func (analyzeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
}
|
||||
|
||||
for _, item := range workPlan.ExecutionOrder {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifactCfg := env.Config.ConcreteArtifacts()[item.Key]
|
||||
fingerprint, _, _, err := computeAnalyzeArtifactFingerprint(
|
||||
item.Key,
|
||||
|
||||
@@ -3,6 +3,7 @@ package stage
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -86,6 +87,9 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(concurrency)
|
||||
for _, job := range jobs {
|
||||
if groupCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
job := job
|
||||
group.Go(func() error {
|
||||
if err := groupCtx.Err(); err != nil {
|
||||
@@ -116,13 +120,10 @@ func (transcribeStage) Run(ctx context.Context, env *Env, m *manifest.Manifest)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
groupErr := group.Wait()
|
||||
if err := state.checkpointFailure(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groupErr := errors.Join(group.Wait(), state.checkpointFailure())
|
||||
completedCount := state.completedCount()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err)
|
||||
return nil, errors.Join(fmt.Errorf("transcribe: canceled after completing %d of %d audio jobs: %w", completedCount, len(jobs), err), groupErr)
|
||||
}
|
||||
if groupErr != nil {
|
||||
return nil, groupErr
|
||||
|
||||
Reference in New Issue
Block a user