Publish and restore prepared session documents
This commit is contained in:
@@ -2,6 +2,8 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -91,10 +93,11 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
|
||||
return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, fmt.Sprintf("remote session %q unavailable: %v", remoteKey, err))
|
||||
}
|
||||
|
||||
sessionInfo, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey)
|
||||
sessionInfo, checksum, err := findRemoteSessionConfig(ctx, store, sessionPrefix, remoteKey, config.CampaignID(base.Campaign), sessionID)
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, err.Error())
|
||||
}
|
||||
remoteKey = sessionInfo.Key
|
||||
sessionTempPath, err := deps.downloadObjectToTemp(ctx, store, remoteKey, "narratio-session-*.yml")
|
||||
if err != nil {
|
||||
return nil, missingSessionConfigError(config.DefaultSessionConfigSearchPaths, fmt.Sprintf("remote session %q download failed: %v", remoteKey, err))
|
||||
@@ -109,6 +112,12 @@ func loadCommandConfigWithDependencies(ctx context.Context, pipelineFlag, campai
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read downloaded remote session config: %w", err)
|
||||
}
|
||||
if checksum != "" {
|
||||
digest := sha256.Sum256(sessionBytes)
|
||||
if !strings.EqualFold(hex.EncodeToString(digest[:]), checksum) || int64(len(sessionBytes)) != sessionInfo.Size {
|
||||
return nil, fmt.Errorf("remote session %q does not match its committed checksum or size", remoteKey)
|
||||
}
|
||||
}
|
||||
sessionCfg, err := config.LoadSessionBytesWithOptions("s3://"+s3BucketName(base.Pipeline)+"/"+remoteKey, sessionBytes, sessionOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -182,17 +191,41 @@ func loadPipelineConfigWithDependencies(pipelineFlag string, opts config.Pipelin
|
||||
return resolvedPipelinePath, pipelineCfg, nil
|
||||
}
|
||||
|
||||
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey string) (storage.ObjectInfo, error) {
|
||||
func findRemoteSessionConfig(ctx context.Context, store storage.ObjectStore, sessionPrefix, remoteKey, campaign, sessionID string) (storage.ObjectInfo, string, error) {
|
||||
current, err := artifacts.LoadCurrentState(ctx, store, sessionPrefix, artifacts.CurrentStateValidation{ExpectedCampaign: campaign, ExpectedSessionID: sessionID, ValidateRunID: true})
|
||||
var selected *artifacts.RemoteArtifact
|
||||
if err == nil {
|
||||
for _, item := range current.Commit.Artifacts {
|
||||
if item.Type == artifacts.RemoteArtifactTypeRunFile && item.Source == "run:prepare/inputs/session.yml" {
|
||||
identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, item.DestinationKey)
|
||||
if err != nil || "run:"+identity.RelativePath != item.Source {
|
||||
return storage.ObjectInfo{}, "", fmt.Errorf("committed session source does not match its object key")
|
||||
}
|
||||
copy := item
|
||||
selected = ©
|
||||
remoteKey = item.DestinationKey
|
||||
break
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(err, artifacts.ErrCurrentCommitPointerMissing) {
|
||||
return storage.ObjectInfo{}, "", fmt.Errorf("load remote session selection: %w", err)
|
||||
}
|
||||
objects, err := store.List(ctx, sessionPrefix)
|
||||
if err != nil {
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||
return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q list failed: %w", remoteKey, err)
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.Key == remoteKey {
|
||||
return obj, nil
|
||||
if selected != nil {
|
||||
if obj.Size != selected.Size || strings.TrimSpace(obj.ETag) != selected.Generation {
|
||||
return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q differs from committed object identity", remoteKey)
|
||||
}
|
||||
return obj, selected.SHA256, nil
|
||||
}
|
||||
return obj, "", nil
|
||||
}
|
||||
}
|
||||
return storage.ObjectInfo{}, fmt.Errorf("remote session %q not found", remoteKey)
|
||||
return storage.ObjectInfo{}, "", fmt.Errorf("remote session %q not found", remoteKey)
|
||||
}
|
||||
|
||||
func s3BucketName(cfg *config.PipelineConfig) string {
|
||||
|
||||
130
internal/app/prepared_session_roundtrip_test.go
Normal file
130
internal/app/prepared_session_roundtrip_test.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/stage"
|
||||
)
|
||||
|
||||
func TestPreparedSessionPublishAndRemoteLoad(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, t.TempDir())
|
||||
fragment := filepath.Join(filepath.Dir(sessionPath), "fragment.yml")
|
||||
mustWriteTestFile(t, sessionPath, "config:\n imports:\n - path: fragment.yml\nsession_id: 2026-05-03\n")
|
||||
mustWriteTestFile(t, fragment, "title: Published title\ninputs: {audio_dir: audio}\n")
|
||||
cfg, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enabled := true
|
||||
cfg.Pipeline.Publish.Enabled = &enabled
|
||||
cfg.Pipeline.Publish.UploadRun = &enabled
|
||||
cfg.Pipeline.Publish.Outputs = nil
|
||||
fake := &storage.FakeBackend{}
|
||||
store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root)
|
||||
env := &stage.Env{Config: cfg, ArtifactStore: store, ObjectStore: fake}
|
||||
m := manifest.New("2026-05-03", time.Now().UTC())
|
||||
m.Campaign = "sample-campaign"
|
||||
m.RunID = "20260519T010203Z-a1b2c3d4"
|
||||
runRoot := artifacts.SessionRunRootForCampaign(cfg.Pipeline.Workspace.Root, m.Campaign, m.SessionID, m.RunID)
|
||||
m.LocalWorkDir = runRoot
|
||||
if err := os.MkdirAll(runRoot, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepare, err := stage.Select("prepare")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := prepare.Run(ctx, env, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
|
||||
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
||||
}
|
||||
if err := os.Remove(sessionPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(fragment); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publish, err := stage.Select("publish")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := publish.Run(ctx, env, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prefix := artifacts.S3SessionPrefix("dnd", m.Campaign, m.SessionID)
|
||||
key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(prefix, m.RunID), "prepare/inputs/session.yml")
|
||||
if got := string(fake.Objects[key].Data); !strings.Contains(got, "Published title") || strings.Contains(got, "imports:") || strings.Contains(got, "speakers_file:") {
|
||||
t.Fatalf("published session = %q", got)
|
||||
}
|
||||
for _, name := range []string{"campaign.yml", "party.yml"} {
|
||||
key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(prefix, m.RunID), "prepare/inputs/"+name)
|
||||
if data := fake.Objects[key].Data; len(data) == 0 || strings.Contains(string(data), "imports:") {
|
||||
t.Fatalf("published %s = %q", name, data)
|
||||
}
|
||||
}
|
||||
setSessionConfigSearchPaths(t, []string{sessionPath})
|
||||
deps, _ := remoteSessionLoaderDependencies(fake)
|
||||
loaded, err := loadCommandConfigWithDependencies(ctx, pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: m.SessionID}, nil, deps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = loaded.Close() }()
|
||||
if loaded.Config.Session.Title != "Published title" || loaded.Config.SessionSource.S3Key != key {
|
||||
t.Fatalf("remote session = %#v, source=%#v", loaded.Config.Session, loaded.Config.SessionSource)
|
||||
}
|
||||
if got := loaded.Config.StableInputs.SpeakersFile; got.Source != "campaign_config" || got.Path != filepath.Join(filepath.Dir(campaignPath), "speakers.yml") {
|
||||
t.Fatalf("receiving campaign speakers = %#v", got)
|
||||
}
|
||||
current, err := discoverRemoteCurrentState(ctx, loaded.Config, fake)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan, err := buildRestorePlan(ctx, loaded.Config, current, fake, RestorePlanOptions{Force: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]bool{"inputs/session.yml": false, "inputs/campaign.yml": false, "inputs/party.yml": false}
|
||||
for _, action := range plan.Actions {
|
||||
if _, ok := want[action.LocalRelativePath]; ok {
|
||||
want[action.LocalRelativePath] = true
|
||||
}
|
||||
}
|
||||
for path, found := range want {
|
||||
if !found {
|
||||
t.Fatalf("restore omitted %q", path)
|
||||
}
|
||||
}
|
||||
paths := store.SessionPathsFor(m.Campaign, m.SessionID)
|
||||
for _, name := range []string{"session.yml", "campaign.yml", "party.yml"} {
|
||||
if err := os.Remove(filepath.Join(paths.InputsDir, name)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var output strings.Builder
|
||||
if err := restoreWithDependencies(ctx, []string{m.SessionID, "--config", pipelinePath, "--campaign-file", campaignPath, "--force"}, &output, deps); err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
restored, err := os.ReadFile(filepath.Join(paths.InputsDir, "session.yml"))
|
||||
if err != nil || string(restored) != string(fake.Objects[key].Data) {
|
||||
t.Fatalf("restored session = %q, %v", restored, err)
|
||||
}
|
||||
object := fake.Objects[key]
|
||||
modified := append([]byte(nil), object.Data...)
|
||||
modified[0] ^= 1
|
||||
fake.SeedObject(storage.FakeObject{Key: key, Data: modified, ETag: object.ETag})
|
||||
if _, err := loadCommandConfigWithDependencies(ctx, pipelinePath, "", campaignPath, "", config.SessionLoadOptions{SessionID: m.SessionID}, nil, deps); err == nil || !strings.Contains(err.Error(), "committed checksum") {
|
||||
t.Fatalf("changed remote session error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -179,6 +179,16 @@ func restoreLocalRelativePathForCommittedArtifact(sessionPrefix string, artifact
|
||||
if artifact.Type == artifacts.RemoteArtifactTypeSessionManifest {
|
||||
return config.PathManifestFile, true, nil
|
||||
}
|
||||
if artifact.Type == artifacts.RemoteArtifactTypeRunFile {
|
||||
switch artifact.Source {
|
||||
case "run:prepare/inputs/session.yml", "run:prepare/inputs/campaign.yml", "run:prepare/inputs/party.yml":
|
||||
identity, err := artifacts.ParseImmutableRunObjectKey(sessionPrefix, artifact.DestinationKey)
|
||||
if err != nil || "run:"+identity.RelativePath != artifact.Source {
|
||||
return "", false, fmt.Errorf("prepared input does not match its run object key")
|
||||
}
|
||||
return strings.TrimPrefix(identity.RelativePath, "prepare/"), true, nil
|
||||
}
|
||||
}
|
||||
if artifact.Type != artifacts.RemoteArtifactTypePublishedOutput {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
||||
@@ -82,10 +83,27 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
return nil, fmt.Errorf("publish: run id is required")
|
||||
}
|
||||
|
||||
sessionPaths := publishSessionPaths(env, m)
|
||||
runFiles, err := collectPublishRunFiles(runRoot, m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("publish: collect run files: %w", err)
|
||||
}
|
||||
preparedFiles, err := collectPublishPreparedInputs(env.ArtifactStore, sessionPaths, runRoot, m.Inputs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("publish: collect prepared inputs: %w", err)
|
||||
}
|
||||
byPath := make(map[string]publishUploadFile, len(runFiles)+len(preparedFiles))
|
||||
for _, file := range runFiles {
|
||||
byPath[file.RelativePath] = file
|
||||
}
|
||||
for _, file := range preparedFiles {
|
||||
byPath[file.RelativePath] = file
|
||||
}
|
||||
runFiles = runFiles[:0]
|
||||
for _, file := range byPath {
|
||||
runFiles = append(runFiles, file)
|
||||
}
|
||||
sort.Slice(runFiles, func(i, j int) bool { return runFiles[i].RelativePath < runFiles[j].RelativePath })
|
||||
runSources, err := openPublishRunFiles(runRoot, runFiles)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("publish: verify run files: %w", err)
|
||||
@@ -95,7 +113,6 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("publish: object store does not support verified source uploads")
|
||||
}
|
||||
sessionPaths := publishSessionPaths(env, m)
|
||||
previousFiles, err := collectPublishPreviousFiles(ctx, sessionPaths.PreviousDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("publish: collect previous files: %w", err)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
@@ -23,6 +24,46 @@ type publishUploadFile struct {
|
||||
Checksum string
|
||||
}
|
||||
|
||||
// collectPublishPreparedInputs copies the prepared documents into this run's
|
||||
// archive, retaining their manifest checksums for the ordinary upload path.
|
||||
func collectPublishPreparedInputs(store artifacts.Store, paths artifacts.SessionPaths, runRoot string, records []manifest.InputRecord) ([]publishUploadFile, error) {
|
||||
filenames := map[string]string{"session_config": "session.yml", "campaign_config": "campaign.yml", "party": "party.yml"}
|
||||
var result []publishUploadFile
|
||||
for _, record := range records {
|
||||
name, ok := filenames[record.Kind]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rel, err := pathsafe.SlashRelativeFromRoot(paths.Root, record.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
source, err := openPublishSourceUnderRoot(paths.Root, rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := verifyPublishSourceChecksum(source, rel, record.Checksum); err != nil {
|
||||
_ = source.Close()
|
||||
return nil, err
|
||||
}
|
||||
data, err := io.ReadAll(source)
|
||||
closeErr := source.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
archive := filepath.ToSlash(filepath.Join("prepare", "inputs", name))
|
||||
destination := filepath.Join(runRoot, filepath.FromSlash(archive))
|
||||
if _, err := writeBytesIfChanged(store, destination, data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, publishUploadFile{RelativePath: archive, SourceRelativePath: archive, Checksum: record.Checksum})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func collectPublishRunFiles(runRoot string, sessionManifest *manifest.Manifest) ([]publishUploadFile, error) {
|
||||
if sessionManifest == nil {
|
||||
return nil, fmt.Errorf("session manifest is required")
|
||||
|
||||
Reference in New Issue
Block a user