Propagate cancellation and preserve resumable state
This commit is contained in:
152
internal/app/runner_cancellation_test.go
Normal file
152
internal/app/runner_cancellation_test.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
type cancellationTerminalStore struct {
|
||||
manifest.Store
|
||||
terminalCalls int
|
||||
terminalErr error
|
||||
value any
|
||||
}
|
||||
|
||||
func (s *cancellationTerminalStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||||
if m.LastError != nil {
|
||||
s.terminalCalls++
|
||||
if ctx.Err() != nil || ctx.Value(cancellationContextKey{}) != s.value {
|
||||
return errors.New("terminal save lost its detached invocation context")
|
||||
}
|
||||
if s.terminalErr != nil {
|
||||
return s.terminalErr
|
||||
}
|
||||
}
|
||||
return s.Store.Save(ctx, path, m)
|
||||
}
|
||||
|
||||
type cancellationContextKey struct{}
|
||||
|
||||
func TestCancellationFinalizesCheckpointsAndStopsLaterWork(t *testing.T) {
|
||||
for _, saveFails := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "saved", true: "save failure"}[saveFails], func(t *testing.T) {
|
||||
cfg, base := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"})
|
||||
concurrency := 1
|
||||
cfg.Pipeline.WhisperX.Concurrency = &concurrency
|
||||
ctx, cancel := context.WithCancel(context.WithValue(context.Background(), cancellationContextKey{}, "invocation-value"))
|
||||
defer cancel()
|
||||
saveErr := errors.New("terminal save unavailable")
|
||||
shutdownErr := errors.New("adapter shutdown failed")
|
||||
store := &cancellationTerminalStore{Store: base, value: "invocation-value"}
|
||||
if saveFails {
|
||||
store.terminalErr = saveErr
|
||||
}
|
||||
local := &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
|
||||
fake := &whisperx.FakeClient{TranscribeFn: func(adapterCtx context.Context, request whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
|
||||
switch request.SpeakerID {
|
||||
case "a":
|
||||
return writeTranscribeRecoveryResult(request)
|
||||
case "b":
|
||||
cancel()
|
||||
<-adapterCtx.Done()
|
||||
return whisperx.TranscribeResult{}, errors.Join(adapterCtx.Err(), shutdownErr)
|
||||
default:
|
||||
t.Fatal("dispatched a speaker after cancellation")
|
||||
return whisperx.TranscribeResult{}, nil
|
||||
}
|
||||
}}
|
||||
later := 0
|
||||
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "transcribe"), countingStage{name: "merge", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local, WhisperX: fake}})
|
||||
if !errors.Is(err, context.Canceled) || !errors.Is(err, shutdownErr) || (saveFails && !errors.Is(err, saveErr)) {
|
||||
t.Fatalf("lost cancellation/finalization cause: %v", err)
|
||||
}
|
||||
if operationOutcome(err).Status != 1 || summary == nil || summary.Complete || !local.released || store.terminalCalls != 1 || later != 0 {
|
||||
t.Fatalf("summary=%#v released=%v saves=%d later=%d err=%v", summary, local.released, store.terminalCalls, later, err)
|
||||
}
|
||||
if summary.StageResults[0].Outcome != "failed" || summary.StageResults[1].Outcome != "not_attempted" || summary.StageResults[2].Outcome != "not_attempted" {
|
||||
t.Fatalf("outcomes=%#v", summary.StageResults)
|
||||
}
|
||||
checkpointed := 0
|
||||
for _, item := range summary.AvailableArtifacts {
|
||||
if item.Evidence == "speaker_checkpoint" {
|
||||
checkpointed++
|
||||
}
|
||||
}
|
||||
if checkpointed != 1 {
|
||||
t.Fatalf("durable checkpoint report=%#v", summary.AvailableArtifacts)
|
||||
}
|
||||
saved := loadTranscribeRecoveryManifest(t, base, cfg)
|
||||
if !slices.Equal(transcribeSpeakerKeys(saved), []string{"a"}) {
|
||||
t.Fatal("successful speaker checkpoint lost")
|
||||
}
|
||||
if !saveFails && (saved.Stages["transcribe"].Status != manifest.StatusFailed || saved.LastError == nil) {
|
||||
t.Fatal("terminal manifest was not saved with cancelled context")
|
||||
}
|
||||
retry := &whisperx.FakeClient{}
|
||||
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "transcribe")}, RunOptions{Env: &Env{ManifestStore: base, WhisperX: retry}})
|
||||
if err != nil || !resumed.Complete || !slices.Equal(requestedTranscribeSpeakers(retry), []string{"b", "c"}) {
|
||||
t.Fatalf("retry=%#v requests=%v err=%v", resumed, requestedTranscribeSpeakers(retry), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type cancelingAnalysisRunner struct {
|
||||
analyzeCheckpointRunner
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (r *cancelingAnalysisRunner) RunArtifact(ctx context.Context, req scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) {
|
||||
if len(r.Calls) == 1 {
|
||||
r.cancel()
|
||||
return scriptorium.ArtifactResult{}, ctx.Err()
|
||||
}
|
||||
return r.analyzeCheckpointRunner.RunArtifact(ctx, req)
|
||||
}
|
||||
|
||||
func TestCancellationRetainsArtifactCheckpointForResume(t *testing.T) {
|
||||
cfg := testConfig(t)
|
||||
definitions := map[string]config.ScriptoriumArtifactConfig{
|
||||
"alpha": {Enabled: true, PromptID: "alpha", OutputPath: "artifacts/alpha.md"},
|
||||
"beta": {Enabled: true, PromptID: "beta", OutputPath: "artifacts/beta.md"},
|
||||
"gamma": {Enabled: true, PromptID: "gamma", OutputPath: "artifacts/gamma.md"},
|
||||
}
|
||||
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: definitions}
|
||||
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: definitions}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
runner := &cancelingAnalysisRunner{cancel: cancel}
|
||||
later := 0
|
||||
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "analyze"), countingStage{name: "publish", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{Scriptorium: runner}})
|
||||
if !errors.Is(err, context.Canceled) || summary == nil || summary.Complete || later != 0 {
|
||||
t.Fatalf("summary=%#v err=%v later=%d", summary, err, later)
|
||||
}
|
||||
store := &manifest.LocalStore{}
|
||||
durable := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||||
if durable.Stages["analyze"].Status != manifest.StatusFailed || durable.Stages["analyze"].AnalyzeArtifacts["alpha"].Status != manifest.AnalyzeArtifactCurrent {
|
||||
t.Fatalf("checkpoint lost: %#v", durable.Stages["analyze"])
|
||||
}
|
||||
checkpointed := 0
|
||||
for _, artifact := range summary.AvailableArtifacts {
|
||||
if artifact.Evidence == "artifact_checkpoint" && artifact.Source == "narratio.artifact.alpha" {
|
||||
checkpointed++
|
||||
}
|
||||
}
|
||||
if checkpointed != 1 {
|
||||
t.Fatalf("partial artifacts=%#v", summary.AvailableArtifacts)
|
||||
}
|
||||
retry := &analyzeCheckpointRunner{}
|
||||
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{Env: &Env{Scriptorium: retry}})
|
||||
if err != nil || !resumed.Complete || !slices.Equal(retry.Calls, []string{"beta", "gamma"}) {
|
||||
t.Fatalf("resume=%#v calls=%v err=%v", resumed, retry.Calls, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user