Publish and restore prepared session documents

This commit is contained in:
2026-11-17 22:27:24 +00:00
parent bcfca7a044
commit ed7e722077
6 changed files with 238 additions and 7 deletions

View File

@@ -2,6 +2,8 @@ package app
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
@@ -91,10 +93,11 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
}
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
sessionInfo, checksum, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey, config.CampaignID(base.Campaign), sessionID)
if err != nil {
return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, err.Error())
}
remoteKey = sessionInfo.Key
sessionTempPath, err := deps.downloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
if err != nil {
return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
@@ -109,6 +112,12 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
if err != nil {
return nil, fmt.Errorf("read downloaded remote session config: %w", err)
}
if checksum != "" {
digest := sha256.Sum256(sessionBytes)
if !strings.EqualFold(hex.EncodeToString(digest[:]), checksum) || int64(len(sessionBytes)) != sessionInfo.Size {
return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey)
}
}
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
if err != nil {
return nil, err
@@ -182,17 +191,41 @@ func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.Pipelin
return resolvedPipelinePath, pipelineCfg, nil
}
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey, campaign, sessionID string) (storage.ObjectInfo, string, error) {
current, err := artifacts.LoadCurrentState(ctx, store, sessionPrefix, artifacts.CurrentStateValidation{ExpectedCampaign: campaign, ExpectedSessionID: sessionID, ValidateRunID: true})
var selected *artifacts.RemoteArtifact
if err == nil {
for _, item := range current.Commit.Artifacts {
if item.Type == artifacts.RemoteArtifactTypeRunFile && item.Source == "run:prepare/inputs/session.yml" {
identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, item.DestinationKey)
if err != nil || "run:"+identity.RelativePath != item.Source {
return storage.ObjectInfo{}, "", fmt.Errorf("committed session source does not match its object key")
}
copy := item
selected = &copy
remoteKey = item.DestinationKey
break
}
}
} else if !errors.Is(err, artifacts.ErrCurrentCommitPointerMissing) {
return storage.ObjectInfo{}, "", fmt.Errorf("load remote session selection: %w", err)
}
objects, err := store.List(ctx, sessionPrefix)
if err != nil {
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
}
for _, obj := range objects {
if obj.Key == remoteKey {
return obj, nil
if selected != nil {
if obj.Size != selected.Size || strings.TrimSpace(obj.ETag) != selected.Generation {
return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q differs from committed object identity", remoteKey)
}
return obj, selected.SHA256, nil
}
return obj, "", nil
}
}
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q not found", remoteKey)
}
func s3BucketName(cfg *config.PipelineConfig) string {

View File

@@ -0,0 +1,130 @@
package app
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
"gitea.maximumdirect.net/eric/narratio/internal/stage"
)
func TestPreparedSessionPublishAndRemoteLoad(t *testing.T) {
ctx := context.Background()
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, t.TempDir())
fragment := filepath.Join(filepath.Dir(sessionPath), "fragment.yml")
mustWriteTestFile(t, sessionPath, "config:\n imports:\n - path: fragment.yml\nsession_id: 2026-05-03\n")
mustWriteTestFile(t, fragment, "title: Published title\ninputs: {audio_dir: audio}\n")
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
if err != nil {
t.Fatal(err)
}
enabled := true
cfg.Pipeline.Publish.Enabled = &enabled
cfg.Pipeline.Publish.UploadRun = &enabled
cfg.Pipeline.Publish.Outputs = nil
fake := &storage.FakeBackend{}
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
env := &stage.Env{Config: cfg, ArtifactStore: store, ObjectStore: fake}
m := manifest.New("2026-05-03", time.Now().UTC())
m.Campaign = "sample-campaign"
m.RunID = "20260519T010203Z-a1b2c3d4"
runRoot := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, m.Campaign, m.SessionID, m.RunID)
m.LocalWorkDir = runRoot
if err := os.MkdirAll(runRoot, 0o755); err != nil {
t.Fatal(err)
}
prepare, err := stage.Select("prepare")
if err != nil {
t.Fatal(err)
}
if _, err := prepare.Run(ctx, env, m); err != nil {
t.Fatal(err)
}
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
if err := os.Remove(sessionPath); err != nil {
t.Fatal(err)
}
if err := os.Remove(fragment); err != nil {
t.Fatal(err)
}
publish, err := stage.Select("publish")
if err != nil {
t.Fatal(err)
}
if _, err := publish.Run(ctx, env, m); err != nil {
t.Fatal(err)
}
prefix := artifacts.S3SessionPrefix("dnd", m.Campaign, m.SessionID)
key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(prefix, m.RunID), "prepare/inputs/session.yml")
if got := string(fake.Objects[key].Data); !strings.Contains(got, "Published title") || strings.Contains(got, "imports:") || strings.Contains(got, "speakers_file:") {
t.Fatalf("published session = %q", got)
}
for _, name := range []string{"campaign.yml", "party.yml"} {
key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(prefix, m.RunID), "prepare/inputs/"+name)
if data := fake.Objects[key].Data; len(data) == 0 || strings.Contains(string(data), "imports:") {
t.Fatalf("published %s = %q", name, data)
}
}
setSessionConfigSearchPaths(t, []string{sessionPath})
deps, _ := remoteSessionLoaderDependencies(fake)
loaded, err := loadCommandConfigWithDependencies(ctx, pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: m.SessionID}, nil, deps)
if err != nil {
t.Fatal(err)
}
defer func() { _ = loaded.Close() }()
if loaded.Config.Session.Title != "Published title" || loaded.Config.SessionSource.S3Key != key {
t.Fatalf("remote session = %#v, source=%#v", loaded.Config.Session, loaded.Config.SessionSource)
}
if got := loaded.Config.StableInputs.SpeakersFile; got.Source != "campaign_config" || got.Path != filepath.Join(filepath.Dir(campaignPath), "speakers.yml") {
t.Fatalf("receiving campaign speakers = %#v", got)
}
current, err := discoverRemoteCurrentState(ctx, loaded.Config, fake)
if err != nil {
t.Fatal(err)
}
plan, err := buildRestorePlan(ctx, loaded.Config, current, fake, RestorePlanOptions{Force: true})
if err != nil {
t.Fatal(err)
}
want := map[string]bool{"inputs/session.yml": false, "inputs/campaign.yml": false, "inputs/party.yml": false}
for _, action := range plan.Actions {
if _, ok := want[action.LocalRelativePath]; ok {
want[action.LocalRelativePath] = true
}
}
for path, found := range want {
if !found {
t.Fatalf("restore omitted %q", path)
}
}
paths := store.SessionPathsFor(m.Campaign, m.SessionID)
for _, name := range []string{"session.yml", "campaign.yml", "party.yml"} {
if err := os.Remove(filepath.Join(paths.InputsDir, name)); err != nil {
t.Fatal(err)
}
}
var output strings.Builder
if err := restoreWithDependencies(ctx, []string{m.SessionID, "--config", pipelinePath, "--campaign-file", campaignPath, "--force"}, &output, deps); err != nil {
t.Fatalf("restore: %v", err)
}
restored, err := os.ReadFile(filepath.Join(paths.InputsDir, "session.yml"))
if err != nil || string(restored) != string(fake.Objects[key].Data) {
t.Fatalf("restored session = %q, %v", restored, err)
}
object := fake.Objects[key]
modified := append([]byte(nil), object.Data...)
modified[0] ^= 1
fake.SeedObject(storage.FakeObject{Key: key, Data: modified, ETag: object.ETag})
if _, err := loadCommandConfigWithDependencies(ctx, pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: m.SessionID}, nil, deps); err == nil || !strings.Contains(err.Error(), "committed checksum") {
t.Fatalf("changed remote session error = %v", err)
}
}

View File

@@ -179,6 +179,16 @@ func restoreLocalRelativePathForCommittedArtifact(sessionPrefix string, artifact
if artifact.Type == artifacts.RemoteArtifactTypeSessionManifest {
return config.PathManifestFile, true, nil
}
if artifact.Type == artifacts.RemoteArtifactTypeRunFile {
switch artifact.Source {
case "run:prepare/inputs/session.yml", "run:prepare/inputs/campaign.yml", "run:prepare/inputs/party.yml":
identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, artifact.DestinationKey)
if err != nil || "run:"+identity.RelativePath != artifact.Source {
return "", false, fmt.Errorf("prepared input does not match its run object key")
}
return strings.TrimPrefix(identity.RelativePath, "prepare/"), true, nil
}
}
if artifact.Type != artifacts.RemoteArtifactTypePublishedOutput {
return "", false, nil
}