package app import ( "context" "errors" "slices" "testing" "gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium" "gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx" "gitea.maximumdirect.net/eric/narratio/internal/artifacts" "gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/manifest" "gitea.maximumdirect.net/eric/narratio/internal/stage" ) type cancellationTerminalStore struct { manifest.Store terminalCalls int terminalErr error value any } func (s *cancellationTerminalStore) Save(ctx context.Context, path string, m *manifest.Manifest) error { if m.LastError != nil { s.terminalCalls++ if ctx.Err() != nil || ctx.Value(cancellationContextKey{}) != s.value { return errors.New("terminal save lost its detached invocation context") } if s.terminalErr != nil { return s.terminalErr } } return s.Store.Save(ctx, path, m) } type cancellationContextKey struct{} func TestCancellationFinalizesCheckpointsAndStopsLaterWork(t *testing.T) { for _, saveFails := range []bool{false, true} { t.Run(map[bool]string{false: "saved", true: "save failure"}[saveFails], func(t *testing.T) { cfg, base := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"}) concurrency := 1 cfg.Pipeline.WhisperX.Concurrency = &concurrency ctx, cancel := context.WithCancel(context.WithValue(context.Background(), cancellationContextKey{}, "invocation-value")) defer cancel() saveErr := errors.New("terminal save unavailable") shutdownErr := errors.New("adapter shutdown failed") store := &cancellationTerminalStore{Store: base, value: "invocation-value"} if saveFails { store.terminalErr = saveErr } local := &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)} fake := &whisperx.FakeClient{TranscribeFn: func(adapterCtx context.Context, request whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) { switch request.SpeakerID { case "a": return writeTranscribeRecoveryResult(request) case "b": cancel() <-adapterCtx.Done() return whisperx.TranscribeResult{}, errors.Join(adapterCtx.Err(), shutdownErr) default: t.Fatal("dispatched a speaker after cancellation") return whisperx.TranscribeResult{}, nil } }} later := 0 summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "transcribe"), countingStage{name: "merge", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local, WhisperX: fake}}) if !errors.Is(err, context.Canceled) || !errors.Is(err, shutdownErr) || (saveFails && !errors.Is(err, saveErr)) { t.Fatalf("lost cancellation/finalization cause: %v", err) } if operationOutcome(err).Status != 1 || summary == nil || summary.Complete || !local.released || store.terminalCalls != 1 || later != 0 { t.Fatalf("summary=%#v released=%v saves=%d later=%d err=%v", summary, local.released, store.terminalCalls, later, err) } if summary.StageResults[0].Outcome != "failed" || summary.StageResults[1].Outcome != "not_attempted" || summary.StageResults[2].Outcome != "not_attempted" { t.Fatalf("outcomes=%#v", summary.StageResults) } checkpointed := 0 for _, item := range summary.AvailableArtifacts { if item.Evidence == "speaker_checkpoint" { checkpointed++ } } if checkpointed != 1 { t.Fatalf("durable checkpoint report=%#v", summary.AvailableArtifacts) } saved := loadTranscribeRecoveryManifest(t, base, cfg) if !slices.Equal(transcribeSpeakerKeys(saved), []string{"a"}) { t.Fatal("successful speaker checkpoint lost") } if !saveFails && (saved.Stages["transcribe"].Status != manifest.StatusFailed || saved.LastError == nil) { t.Fatal("terminal manifest was not saved with cancelled context") } retry := &whisperx.FakeClient{} resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "transcribe")}, RunOptions{Env: &Env{ManifestStore: base, WhisperX: retry}}) if err != nil || !resumed.Complete || !slices.Equal(requestedTranscribeSpeakers(retry), []string{"b", "c"}) { t.Fatalf("retry=%#v requests=%v err=%v", resumed, requestedTranscribeSpeakers(retry), err) } }) } } type cancelingAnalysisRunner struct { analyzeCheckpointRunner cancel context.CancelFunc } func (r *cancelingAnalysisRunner) RunArtifact(ctx context.Context, req scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) { if len(r.Calls) == 1 { r.cancel() return scriptorium.ArtifactResult{}, ctx.Err() } return r.analyzeCheckpointRunner.RunArtifact(ctx, req) } func TestCancellationRetainsArtifactCheckpointForResume(t *testing.T) { cfg := testConfig(t) definitions := map[string]config.ScriptoriumArtifactConfig{ "alpha": {Enabled: true, PromptID: "alpha", OutputPath: "artifacts/alpha.md"}, "beta": {Enabled: true, PromptID: "beta", OutputPath: "artifacts/beta.md"}, "gamma": {Enabled: true, PromptID: "gamma", OutputPath: "artifacts/gamma.md"}, } cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: definitions} cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: definitions} ctx, cancel := context.WithCancel(context.Background()) defer cancel() runner := &cancelingAnalysisRunner{cancel: cancel} later := 0 summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "analyze"), countingStage{name: "publish", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{Scriptorium: runner}}) if !errors.Is(err, context.Canceled) || summary == nil || summary.Complete || later != 0 { t.Fatalf("summary=%#v err=%v later=%d", summary, err, later) } store := &manifest.LocalStore{} durable := loadAnalyzeRecoveryManifest(t, store, cfg) if durable.Stages["analyze"].Status != manifest.StatusFailed || durable.Stages["analyze"].AnalyzeArtifacts["alpha"].Status != manifest.AnalyzeArtifactCurrent { t.Fatalf("checkpoint lost: %#v", durable.Stages["analyze"]) } checkpointed := 0 for _, artifact := range summary.AvailableArtifacts { if artifact.Evidence == "artifact_checkpoint" && artifact.Source == "narratio.artifact.alpha" { checkpointed++ } } if checkpointed != 1 { t.Fatalf("partial artifacts=%#v", summary.AvailableArtifacts) } retry := &analyzeCheckpointRunner{} resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{Env: &Env{Scriptorium: retry}}) if err != nil || !resumed.Complete || !slices.Equal(retry.Calls, []string{"beta", "gamma"}) { t.Fatalf("resume=%#v calls=%v err=%v", resumed, retry.Calls, err) } }