153 lines
6.6 KiB
Go
153 lines
6.6 KiB
Go
|
|
package app
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"slices"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/scriptorium"
|
||
|
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
||
|
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||
|
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||
|
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||
|
|
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||
|
|
)
|
||
|
|
|
||
|
|
type cancellationTerminalStore struct {
|
||
|
|
manifest.Store
|
||
|
|
terminalCalls int
|
||
|
|
terminalErr error
|
||
|
|
value any
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *cancellationTerminalStore) Save(ctx context.Context, path string, m *manifest.Manifest) error {
|
||
|
|
if m.LastError != nil {
|
||
|
|
s.terminalCalls++
|
||
|
|
if ctx.Err() != nil || ctx.Value(cancellationContextKey{}) != s.value {
|
||
|
|
return errors.New("terminal save lost its detached invocation context")
|
||
|
|
}
|
||
|
|
if s.terminalErr != nil {
|
||
|
|
return s.terminalErr
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return s.Store.Save(ctx, path, m)
|
||
|
|
}
|
||
|
|
|
||
|
|
type cancellationContextKey struct{}
|
||
|
|
|
||
|
|
func TestCancellationFinalizesCheckpointsAndStopsLaterWork(t *testing.T) {
|
||
|
|
for _, saveFails := range []bool{false, true} {
|
||
|
|
t.Run(map[bool]string{false: "saved", true: "save failure"}[saveFails], func(t *testing.T) {
|
||
|
|
cfg, base := setupTranscribeRecoveryFixture(t, []string{"a", "b", "c"})
|
||
|
|
concurrency := 1
|
||
|
|
cfg.Pipeline.WhisperX.Concurrency = &concurrency
|
||
|
|
ctx, cancel := context.WithCancel(context.WithValue(context.Background(), cancellationContextKey{}, "invocation-value"))
|
||
|
|
defer cancel()
|
||
|
|
saveErr := errors.New("terminal save unavailable")
|
||
|
|
shutdownErr := errors.New("adapter shutdown failed")
|
||
|
|
store := &cancellationTerminalStore{Store: base, value: "invocation-value"}
|
||
|
|
if saveFails {
|
||
|
|
store.terminalErr = saveErr
|
||
|
|
}
|
||
|
|
local := &releaseObservedStore{Store: artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)}
|
||
|
|
fake := &whisperx.FakeClient{TranscribeFn: func(adapterCtx context.Context, request whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
|
||
|
|
switch request.SpeakerID {
|
||
|
|
case "a":
|
||
|
|
return writeTranscribeRecoveryResult(request)
|
||
|
|
case "b":
|
||
|
|
cancel()
|
||
|
|
<-adapterCtx.Done()
|
||
|
|
return whisperx.TranscribeResult{}, errors.Join(adapterCtx.Err(), shutdownErr)
|
||
|
|
default:
|
||
|
|
t.Fatal("dispatched a speaker after cancellation")
|
||
|
|
return whisperx.TranscribeResult{}, nil
|
||
|
|
}
|
||
|
|
}}
|
||
|
|
later := 0
|
||
|
|
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "transcribe"), countingStage{name: "merge", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{ManifestStore: store, ArtifactStore: local, WhisperX: fake}})
|
||
|
|
if !errors.Is(err, context.Canceled) || !errors.Is(err, shutdownErr) || (saveFails && !errors.Is(err, saveErr)) {
|
||
|
|
t.Fatalf("lost cancellation/finalization cause: %v", err)
|
||
|
|
}
|
||
|
|
if operationOutcome(err).Status != 1 || summary == nil || summary.Complete || !local.released || store.terminalCalls != 1 || later != 0 {
|
||
|
|
t.Fatalf("summary=%#v released=%v saves=%d later=%d err=%v", summary, local.released, store.terminalCalls, later, err)
|
||
|
|
}
|
||
|
|
if summary.StageResults[0].Outcome != "failed" || summary.StageResults[1].Outcome != "not_attempted" || summary.StageResults[2].Outcome != "not_attempted" {
|
||
|
|
t.Fatalf("outcomes=%#v", summary.StageResults)
|
||
|
|
}
|
||
|
|
checkpointed := 0
|
||
|
|
for _, item := range summary.AvailableArtifacts {
|
||
|
|
if item.Evidence == "speaker_checkpoint" {
|
||
|
|
checkpointed++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if checkpointed != 1 {
|
||
|
|
t.Fatalf("durable checkpoint report=%#v", summary.AvailableArtifacts)
|
||
|
|
}
|
||
|
|
saved := loadTranscribeRecoveryManifest(t, base, cfg)
|
||
|
|
if !slices.Equal(transcribeSpeakerKeys(saved), []string{"a"}) {
|
||
|
|
t.Fatal("successful speaker checkpoint lost")
|
||
|
|
}
|
||
|
|
if !saveFails && (saved.Stages["transcribe"].Status != manifest.StatusFailed || saved.LastError == nil) {
|
||
|
|
t.Fatal("terminal manifest was not saved with cancelled context")
|
||
|
|
}
|
||
|
|
retry := &whisperx.FakeClient{}
|
||
|
|
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "transcribe")}, RunOptions{Env: &Env{ManifestStore: base, WhisperX: retry}})
|
||
|
|
if err != nil || !resumed.Complete || !slices.Equal(requestedTranscribeSpeakers(retry), []string{"b", "c"}) {
|
||
|
|
t.Fatalf("retry=%#v requests=%v err=%v", resumed, requestedTranscribeSpeakers(retry), err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
type cancelingAnalysisRunner struct {
|
||
|
|
analyzeCheckpointRunner
|
||
|
|
cancel context.CancelFunc
|
||
|
|
}
|
||
|
|
|
||
|
|
func (r *cancelingAnalysisRunner) RunArtifact(ctx context.Context, req scriptorium.RunArtifactRequest) (scriptorium.ArtifactResult, error) {
|
||
|
|
if len(r.Calls) == 1 {
|
||
|
|
r.cancel()
|
||
|
|
return scriptorium.ArtifactResult{}, ctx.Err()
|
||
|
|
}
|
||
|
|
return r.analyzeCheckpointRunner.RunArtifact(ctx, req)
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestCancellationRetainsArtifactCheckpointForResume(t *testing.T) {
|
||
|
|
cfg := testConfig(t)
|
||
|
|
definitions := map[string]config.ScriptoriumArtifactConfig{
|
||
|
|
"alpha": {Enabled: true, PromptID: "alpha", OutputPath: "artifacts/alpha.md"},
|
||
|
|
"beta": {Enabled: true, PromptID: "beta", OutputPath: "artifacts/beta.md"},
|
||
|
|
"gamma": {Enabled: true, PromptID: "gamma", OutputPath: "artifacts/gamma.md"},
|
||
|
|
}
|
||
|
|
cfg.Pipeline.Scriptorium = &config.ScriptoriumConfig{Artifacts: definitions}
|
||
|
|
cfg.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: definitions}
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
runner := &cancelingAnalysisRunner{cancel: cancel}
|
||
|
|
later := 0
|
||
|
|
summary, err := executeStages(ctx, cfg, []stage.Stage{mustSelectStage(t, "analyze"), countingStage{name: "publish", runs: &later}, countingStage{name: "cleanup", runs: &later}}, RunOptions{Env: &Env{Scriptorium: runner}})
|
||
|
|
if !errors.Is(err, context.Canceled) || summary == nil || summary.Complete || later != 0 {
|
||
|
|
t.Fatalf("summary=%#v err=%v later=%d", summary, err, later)
|
||
|
|
}
|
||
|
|
store := &manifest.LocalStore{}
|
||
|
|
durable := loadAnalyzeRecoveryManifest(t, store, cfg)
|
||
|
|
if durable.Stages["analyze"].Status != manifest.StatusFailed || durable.Stages["analyze"].AnalyzeArtifacts["alpha"].Status != manifest.AnalyzeArtifactCurrent {
|
||
|
|
t.Fatalf("checkpoint lost: %#v", durable.Stages["analyze"])
|
||
|
|
}
|
||
|
|
checkpointed := 0
|
||
|
|
for _, artifact := range summary.AvailableArtifacts {
|
||
|
|
if artifact.Evidence == "artifact_checkpoint" && artifact.Source == "narratio.artifact.alpha" {
|
||
|
|
checkpointed++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if checkpointed != 1 {
|
||
|
|
t.Fatalf("partial artifacts=%#v", summary.AvailableArtifacts)
|
||
|
|
}
|
||
|
|
retry := &analyzeCheckpointRunner{}
|
||
|
|
resumed, err := executeStages(context.Background(), cfg, []stage.Stage{mustSelectStage(t, "analyze")}, RunOptions{Env: &Env{Scriptorium: retry}})
|
||
|
|
if err != nil || !resumed.Complete || !slices.Equal(retry.Calls, []string{"beta", "gamma"}) {
|
||
|
|
t.Fatalf("resume=%#v calls=%v err=%v", resumed, retry.Calls, err)
|
||
|
|
}
|
||
|
|
}
|