Files
narratio/internal/app/runner_cancellation_test.go

153 lines
6.6 KiB
Go
Raw Normal View History

package app
import (
"context"
"errors"
"slices"
"testing"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
type cancellationTerminalStore struct {
manifest.Store
terminalCalls int
terminalErr error
value any
}
func (s *cancellationTerminalStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
if m.LastError != nil {
s.terminalCalls++
if ctx.Err() != nil || ctx.Value(cancellationContextKey{}) != s.value {
return errors.New("terminal save lost its detached invocation context")
}
if s.terminalErr != nil {
return s.terminalErr
}
}
return s.Store.Save(ctx, path, m)
}
type cancellationContextKey struct{}
func TestCancellationFinalizesCheckpointsAndStopsLaterWork(t *testing.T) {
for _, saveFails := range []bool{false, true} {
t.Run(map[bool]string{false: "saved", true: "save failure"}[saveFails], func(t *testing.T) {
cfg, base := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"})
concurrency := 1
cfg.Pipeline.WhisperX.Concurrency = &concurrency
ctx, cancel := context.WithCancel(context.WithValue(context.Background(), cancellationContextKey{}, "invocation-value"))
defer cancel()
saveErr := errors.New("terminal save unavailable")
shutdownErr := errors.New("adapter shutdown failed")
store := &cancellationTerminalStore{Store: base, value: "invocation-value"}
if saveFails {
store.terminalErr = saveErr
}
local := &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
fake := &whisperx.FakeClient{TranscribeFn: func(adapterCtx context.Context, request whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
switch request.SpeakerID {
case "a":
return writeTranscribeRecoveryResult(request)
case "b":
cancel()
<-adapterCtx.Done()
return whisperx.TranscribeResult{}, errors.Join(adapterCtx.Err(), shutdownErr)
default:
t.Fatal("dispatched a speaker after cancellation")
return whisperx.TranscribeResult{}, nil
}
}}
later := 0
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "transcribe"), countingStage{name: "merge", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local, WhisperX: fake}})
if !errors.Is(err, context.Canceled) || !errors.Is(err, shutdownErr) || (saveFails && !errors.Is(err, saveErr)) {
t.Fatalf("lost cancellation/finalization cause: %v", err)
}
if operationOutcome(err).Status != 1 || summary == nil || summary.Complete || !local.released || store.terminalCalls != 1 || later != 0 {
t.Fatalf("summary=%#v released=%v saves=%d later=%d err=%v", summary, local.released, store.terminalCalls, later, err)
}
if summary.StageResults[0].Outcome != "failed" || summary.StageResults[1].Outcome != "not_attempted" || summary.StageResults[2].Outcome != "not_attempted" {
t.Fatalf("outcomes=%#v", summary.StageResults)
}
checkpointed := 0
for _, item := range summary.AvailableArtifacts {
if item.Evidence == "speaker_checkpoint" {
checkpointed++
}
}
if checkpointed != 1 {
t.Fatalf("durable checkpoint report=%#v", summary.AvailableArtifacts)
}
saved := loadTranscribeRecoveryManifest(t, base, cfg)
if !slices.Equal(transcribeSpeakerKeys(saved), []string{"a"}) {
t.Fatal("successful speaker checkpoint lost")
}
if !saveFails && (saved.Stages["transcribe"].Status != manifest.StatusFailed || saved.LastError == nil) {
t.Fatal("terminal manifest was not saved with cancelled context")
}
retry := &whisperx.FakeClient{}
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "transcribe")}, RunOptions{Env: &Env{ManifestStore: base, WhisperX: retry}})
if err != nil || !resumed.Complete || !slices.Equal(requestedTranscribeSpeakers(retry), []string{"b", "c"}) {
t.Fatalf("retry=%#v requests=%v err=%v", resumed, requestedTranscribeSpeakers(retry), err)
}
})
}
}
type cancelingAnalysisRunner struct {
analyzeCheckpointRunner
cancel context.CancelFunc
}
func (r *cancelingAnalysisRunner) RunArtifact(ctx context.Context, req scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) {
if len(r.Calls) == 1 {
r.cancel()
return scriptorium.ArtifactResult{}, ctx.Err()
}
return r.analyzeCheckpointRunner.RunArtifact(ctx, req)
}
func TestCancellationRetainsArtifactCheckpointForResume(t *testing.T) {
cfg := testConfig(t)
definitions := map[string]config.ScriptoriumArtifactConfig{
"alpha": {Enabled: true, PromptID: "alpha", OutputPath: "artifacts/alpha.md"},
"beta": {Enabled: true, PromptID: "beta", OutputPath: "artifacts/beta.md"},
"gamma": {Enabled: true, PromptID: "gamma", OutputPath: "artifacts/gamma.md"},
}
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: definitions}
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: definitions}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
runner := &cancelingAnalysisRunner{cancel: cancel}
later := 0
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "analyze"), countingStage{name: "publish", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{Scriptorium: runner}})
if !errors.Is(err, context.Canceled) || summary == nil || summary.Complete || later != 0 {
t.Fatalf("summary=%#v err=%v later=%d", summary, err, later)
}
store := &manifest.LocalStore{}
durable := loadAnalyzeRecoveryManifest(t, store, cfg)
if durable.Stages["analyze"].Status != manifest.StatusFailed || durable.Stages["analyze"].AnalyzeArtifacts["alpha"].Status != manifest.AnalyzeArtifactCurrent {
t.Fatalf("checkpoint lost: %#v", durable.Stages["analyze"])
}
checkpointed := 0
for _, artifact := range summary.AvailableArtifacts {
if artifact.Evidence == "artifact_checkpoint" && artifact.Source == "narratio.artifact.alpha" {
checkpointed++
}
}
if checkpointed != 1 {
t.Fatalf("partial artifacts=%#v", summary.AvailableArtifacts)
}
retry := &analyzeCheckpointRunner{}
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{Env: &Env{Scriptorium: retry}})
if err != nil || !resumed.Complete || !slices.Equal(retry.Calls, []string{"beta", "gamma"}) {
t.Fatalf("resume=%#v calls=%v err=%v", resumed, retry.Calls, err)
}
}