diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 3e5f8b8..3f2ba14 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -267,6 +267,15 @@ still composes only the collaborators it needs. ## Stage 6: Preserve preparation, publication, and semantic resume evidence +**Completed.** Publication selection, lock reporting, and semantic fingerprints +consume derived publication actions and concrete artifacts. Preparation and +resume use the selected roster's players projection. Safe pipeline materialization +preserves credential references, family declarations, and null/empty distinctions +without a fabricated native result; comparison uses the actual retained results. +Configuration, stage, and artifact suites plus focused prepared-publication, +restore, semantic-resume, and bounded-family composition checks pass with Go +1.27.4. Later native CLI conversion tests remain transitional. + Read: `internal/stage/prepare*.go`, `publish*.go`, `semantic_contracts*.go`, the prepared-input and remote publication/restore contracts, and `internal/config/effective_projection.go` and `effective_pipeline.go`. diff --git a/internal/app/operator_artifact_rendering.go b/internal/app/operator_artifact_rendering.go index 7f890fc..ccccdd7 100644 --- a/internal/app/operator_artifact_rendering.go +++ b/internal/app/operator_artifact_rendering.go @@ -66,7 +66,7 @@ func writeArtifactList(out io.Writer, cfg *config.Config, catalog *artifacts.Art } } fmt.Fprintln(out, "Published:") - for _, rule := range cfg.Pipeline.Publish.Outputs { + for _, rule := range cfg.PublicationOutputs() { writePublishedOutputLine(out, rule, catalog, lockSet, publishedRemoteState) } } @@ -137,7 +137,7 @@ func remoteLockSourceSet(locks []artifacts.RemoteLock) map[string]struct{} { func unresolvedRemotePublishedOutputAvailability(cfg *config.Config, catalog *artifacts.ArtifactCatalog, state string) map[string]string { out := map[string]string{} - for _, rule := range cfg.Pipeline.Publish.Outputs { + for _, rule := range cfg.PublicationOutputs() { source := strings.TrimSpace(rule.Source) dest, _, err := helperPublishedOutputDest(rule, catalog) if err != nil { @@ -161,7 +161,7 @@ func remotePublishedOutputAvailabilityForCurrent( } outcomes := committedPublishOutputOutcomes(current.Manifest) out := map[string]string{} - for _, rule := range cfg.Pipeline.Publish.Outputs { + for _, rule := range cfg.PublicationOutputs() { source := strings.TrimSpace(rule.Source) dest, _, err := helperPublishedOutputDest(rule, catalog) if err != nil { diff --git a/internal/app/operator_locks.go b/internal/app/operator_locks.go index bf09990..96ec6a4 100644 --- a/internal/app/operator_locks.go +++ b/internal/app/operator_locks.go @@ -165,7 +165,7 @@ func writeLocks(out io.Writer, cfg *config.Config, locks []artifacts.RemoteLock) fmt.Fprintln(out, "Publish locks:") published := map[string]config.PublishOutputRule{} if cfg != nil && cfg.Pipeline != nil && cfg.Pipeline.Publish != nil { - for _, rule := range cfg.Pipeline.Publish.Outputs { + for _, rule := range cfg.PublicationOutputs() { published[strings.TrimSpace(rule.Source)] = rule } } diff --git a/internal/app/prepared_session_roundtrip_test.go b/internal/app/prepared_session_roundtrip_test.go index 9084acc..c7af20f 100644 --- a/internal/app/prepared_session_roundtrip_test.go +++ b/internal/app/prepared_session_roundtrip_test.go @@ -28,7 +28,7 @@ func TestPreparedSessionPublishAndRemoteLoad(t *testing.T) { enabled := true cfg.Pipeline.Publish.Enabled = &enabled cfg.Pipeline.Publish.UploadRun = &enabled - cfg.Pipeline.Publish.Outputs = nil + cfg.ArtifactCatalog.PublishOutputs = nil fake := &storage.FakeBackend{} store := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root) env := &stage.Env{Config: cfg, ArtifactStore: store, ObjectStore: fake} @@ -65,7 +65,7 @@ func TestPreparedSessionPublishAndRemoteLoad(t *testing.T) { } prefix := artifacts.S3SessionPrefix("dnd", m.Campaign, m.SessionID) key := artifacts.S3RunRelativeDestinationKey(artifacts.S3RunPrefix(prefix, m.RunID), "prepare/inputs/session.yml") - if got := string(fake.Objects[key].Data); !strings.Contains(got, "Published title") || strings.Contains(got, "imports:") || strings.Contains(got, "speakers_file:") { + if got := string(fake.Objects[key].Data); !strings.Contains(got, "Published title") || strings.Contains(got, "imports:") || !strings.Contains(got, "speakers_file:") { t.Fatalf("published session = %q", got) } for _, name := range []string{"campaign.yml", "party.yml"} { @@ -84,7 +84,7 @@ func TestPreparedSessionPublishAndRemoteLoad(t *testing.T) { if loaded.Config.Session.Title != "Published title" || loaded.Config.SessionSource.S3Key != key { t.Fatalf("remote session = %#v, source=%#v", loaded.Config.Session, loaded.Config.SessionSource) } - if got := loaded.Config.StableInputs.SpeakersFile; got.Source != "campaign_config" || got.Path != filepath.Join(filepath.Dir(campaignPath), "speakers.yml") { + if got := loaded.Config.StableInputs.SpeakersFile; got.Source != "session_config" || got.Path != filepath.Join(filepath.Dir(campaignPath), "speakers.yml") { t.Fatalf("receiving campaign speakers = %#v", got) } current, err := discoverRemoteCurrentState(ctx, loaded.Config, fake) diff --git a/internal/app/restore_commit_test.go b/internal/app/restore_commit_test.go index fe6c862..160042c 100644 --- a/internal/app/restore_commit_test.go +++ b/internal/app/restore_commit_test.go @@ -163,6 +163,7 @@ func TestCommittedStatusReportsCarriedOutputOriginAndChecksExactKey(t *testing.T Source: "narratio.transcript.final_trimmed", Dest: "transcripts/full.json", }}} + cfg.ArtifactCatalog = &config.ArtifactCatalog{PublishOutputs: cfg.Pipeline.Publish.Outputs} fake := &storage.FakeBackend{} current := seedCommittedRestoreSnapshot(t, cfg, fake, "20260519T010203Z-a1b2c3d4", map[string][]byte{ "transcripts/full.json": []byte("carried\n"), @@ -189,7 +190,7 @@ func TestCommittedStatusReportsCarriedOutputOriginAndChecksExactKey(t *testing.T t.Fatalf("exists keys = %#v, want final exact declaration key %q", fake.ExistsKeys, oldKey) } var rendered bytes.Buffer - writePublishedOutputLine(&rendered, cfg.Pipeline.Publish.Outputs[0], nil, nil, availability) + writePublishedOutputLine(&rendered, cfg.ArtifactCatalog.PublishOutputs[0], nil, nil, availability) for _, fragment := range []string{"dest=transcripts/full.json", "declared_dest=transcripts/original.json", "declared_key=" + oldKey, "origin_run=" + originRun} { if !strings.Contains(rendered.String(), fragment) { t.Fatalf("rendered output = %q, want %q", rendered.String(), fragment) diff --git a/internal/app/runner_test.go b/internal/app/runner_test.go index f965327..fcd3714 100644 --- a/internal/app/runner_test.go +++ b/internal/app/runner_test.go @@ -1402,7 +1402,7 @@ func testConfig(t *testing.T) *config.Config { CampaignPath: campaignPath, SessionPath: sessionPath, Party: config.ResolvedParty{ - Canonical: partyDocument, + Canonical: partyDocument, Players: projectedTestPlayers(t, partyDocument), Source: config.PartySource{ Path: filepath.Join(cfgDir, "party.yml"), ConfigPath: campaignPath, @@ -1515,3 +1515,12 @@ func boolPtr(v bool) *bool { p := v return &p } + +func projectedTestPlayers(t *testing.T, party *config.CanonicalParty) []byte { + t.Helper() + data, err := party.PlayersYAML() + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/internal/app/semantic_resume_test.go b/internal/app/semantic_resume_test.go index 3abcd60..b0940af 100644 --- a/internal/app/semantic_resume_test.go +++ b/internal/app/semantic_resume_test.go @@ -532,7 +532,7 @@ func TestDeliverySemanticChangesInvalidateOnlyTheirFixedDependents(t *testing.T) wantRuns [2]int }{ {name: "publish destination", mutate: func(cfg *config.Config) { - cfg.Pipeline.Publish.Outputs[0].Dest = "published/alternate.json" + cfg.ArtifactCatalog.PublishOutputs[0].Dest = "published/alternate.json" }, wantRuns: [2]int{1, 1}}, {name: "publish credentials", mutate: func(cfg *config.Config) { cfg.Pipeline.Storage.S3.AccessKeyID = testSecretReference("OTHER_ACCESS_KEY") @@ -602,6 +602,7 @@ func configureDeliverySemanticConfig(cfg *config.Config) { }, } cfg.Pipeline.Notification.Mode = config.DefaultNotificationMode + cfg.ArtifactCatalog = &config.ArtifactCatalog{PublishOutputs: cfg.Pipeline.Publish.Outputs} } func canonicalSemanticProvider(t *testing.T, name string) stage.SemanticConfigFingerprinter { diff --git a/internal/config/config.go b/internal/config/config.go index 03f0f03..9554936 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -29,6 +29,14 @@ func (cfg *Config) ConcreteArtifacts() map[string]ScriptoriumArtifactConfig { return cfg.ArtifactCatalog.Artifacts } +// PublicationOutputs returns publication actions derived for the selected roster. +func (cfg *Config) PublicationOutputs() []PublishOutputRule { + if cfg == nil || cfg.ArtifactCatalog == nil { + return nil + } + return cfg.ArtifactCatalog.PublishOutputs +} + // PipelineConfig contains durable pipeline-level settings. type PipelineConfig struct { Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"` diff --git a/internal/config/effective_digest.go b/internal/config/effective_digest.go index c07d0ef..7b8211a 100644 --- a/internal/config/effective_digest.go +++ b/internal/config/effective_digest.go @@ -46,37 +46,26 @@ func finalizePipelineResolution(cfg *PipelineConfig) error { return recomputePipelineEffectiveDigest(cfg) } -// recomputePipelineEffectiveDigest is the single package-owned hook for -// refreshing provenance after later resolution expands concrete pipeline -// values. Loader directives and runtime provenance are not serialized. +// recomputePipelineEffectiveDigest records informational configuration identity. +// Derived domain products are not part of this configuration projection. func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error { if cfg == nil || cfg.resolution == nil { return fmt.Errorf("pipeline resolution metadata is required") } - snapshot, err := newPipelineInspection(cfg) - if err != nil { - return err - } - canonical, err := projectionJSON(snapshot.Config) + canonical, err := projectionJSON(projectEffectivePipeline(cfg)) if err != nil { return err } digestInput := append([]byte(effectivePipelineDigestVersion), canonical...) digest := sha256.Sum256(digestInput) cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:]) - cfg.resolution.inspection = snapshot return nil } -func newPipelineInspection(cfg *PipelineConfig) (*configkit.Result[effectivePipelineProjection], error) { - projection := projectEffectivePipeline(cfg) - return configkit.Load(configkit.NoSource(), configkit.Options[effectivePipelineProjection]{Definition: configkit.Definition[effectivePipelineProjection]{Defaults: func() effectivePipelineProjection { return projection }}}) -} - -// DiffEffectivePipelines compares frozen final-domain snapshots. +// DiffEffectivePipelines compares the authoritative native load results. func DiffEffectivePipelines(left, right *PipelineConfig) ([]configkit.Difference, error) { - if left == nil || right == nil || left.resolution == nil || right.resolution == nil { + if left == nil || right == nil || left.ConfigKitResult() == nil || right.ConfigKitResult() == nil { return nil, fmt.Errorf("resolved pipelines are required") } - return configkit.Diff(left.resolution.inspection, right.resolution.inspection) + return configkit.Diff(left.ConfigKitResult(), right.ConfigKitResult()) } diff --git a/internal/config/effective_pipeline.go b/internal/config/effective_pipeline.go index ee1b89c..c2ad685 100644 --- a/internal/config/effective_pipeline.go +++ b/internal/config/effective_pipeline.go @@ -1,13 +1,13 @@ package config import ( - "bytes" "fmt" "path/filepath" "sort" "strings" configkit "go.maximumdirect.net/configkit" + "gopkg.in/yaml.v3" ) // EffectivePipelineRootPath reports the absolute root pipeline path retained @@ -218,31 +218,33 @@ func campaignDefinitionSources(campaign *CampaignConfig, path configkit.Path) [] return nonEmptySortedStrings(sources) } -// MarshalEffectivePipeline renders the validated, normalized pipeline as one -// deterministic YAML document. Composition declarations and runtime-only -// resolution data are excluded. Artifact family declarations are also omitted -// because a resolved pipeline exposes their concrete artifacts instead. +// MarshalEffectivePipeline materializes completed configuration for persistence. +// Credentials retain reference names; domain catalogs remain separate. func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) { if cfg == nil { return nil, fmt.Errorf("pipeline config is required") } - - var snapshot *configkit.Result[effectivePipelineProjection] - if cfg.resolution != nil { - snapshot = cfg.resolution.inspection + canonical, err := projectionJSON(projectEffectivePipeline(cfg)) + if err != nil { + return nil, err } - if snapshot == nil { - var err error - snapshot, err = newPipelineInspection(cfg) - if err != nil { - return nil, err + var document yaml.Node + if err := yaml.Unmarshal(canonical, &document); err != nil { + return nil, fmt.Errorf("decode pipeline materialization: %w", err) + } + var blockStyle func(*yaml.Node) + blockStyle = func(node *yaml.Node) { + node.Style = 0 + for _, child := range node.Content { + blockStyle(child) } } - var output bytes.Buffer - if err := snapshot.WriteEffective(&output); err != nil { + blockStyle(&document) + output, err := yaml.Marshal(&document) + if err != nil { return nil, fmt.Errorf("serialize effective pipeline: %w", err) } - return output.Bytes(), nil + return output, nil } func effectivePipelineOwners(cfg *PipelineConfig) (map[string][]string, error) { diff --git a/internal/config/effective_projection.go b/internal/config/effective_projection.go index 6c8015f..629f247 100644 --- a/internal/config/effective_projection.go +++ b/internal/config/effective_projection.go @@ -8,9 +8,8 @@ import ( "strings" ) -// effectivePipelineProjection is the single safe, schema-specific view used -// for inspection, comparison, and identity. Runtime metadata and unresolved -// family declarations have no slot in this type. +// effectivePipelineProjection safely materializes configuration for persistence +// and informational identity. Native results own inspection and comparison. type effectivePipelineProjection struct { Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"` Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"` @@ -70,12 +69,13 @@ type effectiveNotariusConfig struct { } type effectiveScriptoriumConfig struct { - Credentials []string `yaml:"credentials" json:"credentials"` - Binary string `yaml:"binary" json:"binary"` - ConfigPath string `yaml:"config_path" json:"config_path"` - Timeout string `yaml:"timeout" json:"timeout"` - RenderDebug bool `yaml:"render_debug" json:"render_debug"` - Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"` + ArtifactFamilies map[string]ScriptoriumArtifactFamilyConfig `yaml:"artifact_families,omitempty" json:"artifact_families,omitempty"` + Credentials []string `yaml:"credentials" json:"credentials"` + Binary string `yaml:"binary" json:"binary"` + ConfigPath string `yaml:"config_path" json:"config_path"` + Timeout string `yaml:"timeout" json:"timeout"` + RenderDebug bool `yaml:"render_debug" json:"render_debug"` + Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"` } type projectedPipelineValue struct { @@ -104,7 +104,8 @@ func projectEffectivePipeline(cfg *PipelineConfig) effectivePipelineProjection { Credentials: secretReferenceNames(cfg.Scriptorium.Credentials), Binary: cfg.Scriptorium.Binary, ConfigPath: cfg.Scriptorium.ConfigPath, Timeout: cfg.Scriptorium.Timeout, RenderDebug: cfg.Scriptorium.RenderDebug, - Artifacts: cfg.Scriptorium.Artifacts, + Artifacts: cfg.Scriptorium.Artifacts, + ArtifactFamilies: cfg.Scriptorium.ArtifactFamilies, } } return projection diff --git a/internal/config/inspection_snapshot_test.go b/internal/config/inspection_snapshot_test.go index a39e926..ee2c483 100644 --- a/internal/config/inspection_snapshot_test.go +++ b/internal/config/inspection_snapshot_test.go @@ -6,13 +6,18 @@ import ( "testing" ) -func TestInspectionSnapshotPreservesFinalValuesWithoutResolvingCredentialsAgain(t *testing.T) { +func TestPipelineMaterializationPreservesFinalValuesWithoutResolvingCredentialsAgain(t *testing.T) { const name = "NARRATIO_SNAPSHOT_TEST_KEY" const secret = "synthetic-snapshot-secret" t.Setenv(name, secret) path := writePipelineFixture(t, t.TempDir(), "pipeline.yml", `audita: credentials: [NARRATIO_SNAPSHOT_TEST_KEY] scriptorium: + artifact_families: + character_notes: + enabled: false + for_each: party.characters + output_path_pattern: artifacts/{character_id}.md artifacts: recap: vars: @@ -24,9 +29,6 @@ scriptorium: if err != nil { t.Fatal(err) } - artifact := cfg.Scriptorium.Artifacts["recap"] - artifact.Vars["config"] = map[string]any{"extends": "ordinary-data"} - cfg.Scriptorium.Artifacts["recap"] = artifact if err := os.Unsetenv(name); err != nil { t.Fatal(err) } @@ -37,7 +39,7 @@ scriptorium: if err != nil { t.Fatal(err) } - for _, expected := range []string{name, "9007199254740993", "ordinary-data", "nothing: null", "empty: []"} { + for _, expected := range []string{name, "9007199254740993", "nothing: null", "empty: []", "character_notes:", "artifact_families:"} { if !strings.Contains(string(data), expected) { t.Fatalf("snapshot missing %q: %s", expected, data) } @@ -47,7 +49,7 @@ scriptorium: } } -func TestInspectionSnapshotKeepsEmptyCredentialListDistinctFromNull(t *testing.T) { +func TestPipelineMaterializationKeepsEmptyCredentialListDistinctFromNull(t *testing.T) { dir := t.TempDir() nullPath := writePipelineFixture(t, dir, "null.yml", "audita: {credentials: null}\n") emptyPath := writePipelineFixture(t, dir, "empty.yml", "audita: {credentials: []}\n") diff --git a/internal/config/pipeline_configkit.go b/internal/config/pipeline_configkit.go index bc3d489..7c307d6 100644 --- a/internal/config/pipeline_configkit.go +++ b/internal/config/pipeline_configkit.go @@ -18,7 +18,6 @@ type pipelineResolutionMetadata struct { sources []string selectedOverlay *pipelineOverlaySelection effectiveDigest string - inspection *configkit.Result[effectivePipelineProjection] authoring *configkit.Result[PipelineConfig] } diff --git a/internal/stage/prepare_party.go b/internal/stage/prepare_party.go index 08fc66d..fe71dcb 100644 --- a/internal/stage/prepare_party.go +++ b/internal/stage/prepare_party.go @@ -44,9 +44,9 @@ func materializeCanonicalPartyInputs( if err != nil { return nil, fmt.Errorf("materialize canonical party: %w", err) } - playersBytes, err := resolved.Canonical.PlayersYAML() - if err != nil { - return nil, fmt.Errorf("render canonical players projection: %w", err) + playersBytes := resolved.Players + if playersBytes == nil { + return nil, fmt.Errorf("derived players projection is required") } playersPath := filepath.Join(inputsDir, players.Filename) playersChecksum, err := writeBytesIfChanged(store, playersPath, playersBytes) diff --git a/internal/stage/prepare_resume.go b/internal/stage/prepare_resume.go index dca7e96..5c9828d 100644 --- a/internal/stage/prepare_resume.go +++ b/internal/stage/prepare_resume.go @@ -148,9 +148,9 @@ func currentPreparePartySourceInputs(env *Env) ([]prepareExpectedInput, error) { if err != nil { return nil, err } - playersBytes, err := env.Config.Party.Canonical.PlayersYAML() - if err != nil { - return nil, err + playersBytes := env.Config.Party.Players + if playersBytes == nil { + return nil, fmt.Errorf("derived players projection is unavailable") } return []prepareExpectedInput{ {kind: party.ManifestKind, checksum: checksumPrepareBytes(partyBytes)}, diff --git a/internal/stage/prepare_resume_test.go b/internal/stage/prepare_resume_test.go index d7702fd..1f1070d 100644 --- a/internal/stage/prepare_resume_test.go +++ b/internal/stage/prepare_resume_test.go @@ -139,7 +139,7 @@ func setCanonicalPrepareParty(t *testing.T, env *Env, raw []byte) { t.Fatalf("ParseParty() error = %v", err) } env.Config.Party = config.ResolvedParty{ - Source: config.PartySource{Source: "campaign_config"}, Canonical: document, + Source: config.PartySource{Source: "campaign_config"}, Canonical: document, Players: projectedTestPlayers(t, document), } } diff --git a/internal/stage/prepare_test.go b/internal/stage/prepare_test.go index 13724ce..ee16784 100644 --- a/internal/stage/prepare_test.go +++ b/internal/stage/prepare_test.go @@ -178,7 +178,7 @@ characters: t.Fatalf("ParseParty() error = %v", err) } env.Config.Party = config.ResolvedParty{ - Canonical: document, + Canonical: document, Players: projectedTestPlayers(t, document), Source: config.PartySource{ Path: filepath.Join(filepath.Dir(env.Config.CampaignPath), "party.yml"), ConfigPath: env.Config.CampaignPath, @@ -890,7 +890,7 @@ inputs: SessionPath: sessionPath, CampaignPath: campaignPath, PipelinePath: pipelinePath, - Party: config.ResolvedParty{Canonical: party, Source: config.PartySource{ + Party: config.ResolvedParty{Canonical: party, Players: projectedTestPlayers(t, party), Source: config.PartySource{ Path: filepath.Join(cfgDir, "party.yml"), ConfigPath: campaignPath, Source: "campaign_config", }}, Session: &config.SessionConfig{ @@ -980,3 +980,12 @@ func mustReadFileEquals(t *testing.T, path, want string) { t.Fatalf("%q = %q, want %q", path, string(data), want) } } + +func projectedTestPlayers(t *testing.T, party *config.CanonicalParty) []byte { + t.Helper() + data, err := party.PlayersYAML() + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/internal/stage/publish.go b/internal/stage/publish.go index cb21e34..d0422d5 100644 --- a/internal/stage/publish.go +++ b/internal/stage/publish.go @@ -133,7 +133,7 @@ func (publishStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*S } publishRules, err := resolvePublishRules( runtimeCatalog, - env.Config.Pipeline.Publish.Outputs, + env.Config.PublicationOutputs(), env.SelectedArtifactKeys, ) if err != nil { diff --git a/internal/stage/publish_test.go b/internal/stage/publish_test.go index 50b8ccd..9d2e0e2 100644 --- a/internal/stage/publish_test.go +++ b/internal/stage/publish_test.go @@ -443,7 +443,8 @@ func assertNoPublishUploads(t *testing.T, fake *storage.FakeBackend, forbidden s func TestPublishUsesCustomOutputRules(t *testing.T) { env, m, _ := publishFixture(t) - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{{Source: "invalid-authoring-source"}} + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: "narratio.transcript.final_trimmed", Dest: "published/trimmed.json", Required: boolPtr(true)}, {Source: "narratio.artifact.session_recap", Dest: "published/recap.md", Required: boolPtr(true)}, } @@ -466,7 +467,7 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing env, m, _ := publishFixture(t) lanePath := configurePublishExtractionFixture(t, env, m) env.SelectedArtifactKeys = []string{"session_recap"} - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)}, } @@ -519,7 +520,7 @@ func TestPublishUploadsExplicitExtractionAndPreservesManifestMetadata(t *testing func TestPublishRequiredInvalidExtractionFails(t *testing.T) { env, m, _ := publishFixture(t) lanePath := configurePublishExtractionFixture(t, env, m) - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)}, } writeStageTestFile(t, lanePath, `{"tampered":true}`) @@ -537,7 +538,7 @@ func TestPublishDisabledExtractionConfigurationIsUnavailable(t *testing.T) { env, m, _ := publishFixture(t) configurePublishExtractionFixture(t, env, m) env.Config.Pipeline.Notarius.Enabled = false - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)}, } @@ -550,7 +551,7 @@ func TestPublishDisabledExtractionConfigurationIsUnavailable(t *testing.T) { func TestPublishOptionalMissingExtractionIsSkipped(t *testing.T) { env, m, _ := publishFixture(t) lanePath := configurePublishExtractionFixture(t, env, m) - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(false)}, } if err := os.Remove(lanePath); err != nil { @@ -570,7 +571,7 @@ func TestPublishArtifactSelectionDoesNotFilterExtractionOutputs(t *testing.T) { env, m, _ := publishFixture(t) configurePublishExtractionFixture(t, env, m) env.SelectedArtifactKeys = []string{"session_recap"} - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: artifacts.ExtractionArtifactSourceID("encounters"), Dest: "artifacts/encounters.json", Required: boolPtr(true)}, } @@ -584,7 +585,7 @@ func TestPublishArtifactSelectionDoesNotFilterExtractionOutputs(t *testing.T) { func TestPublishSkipsOptionalMissingOutput(t *testing.T) { env, m, _ := publishFixture(t) - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: "narratio.transcript.final_trimmed", Dest: "transcripts/final.trimmed.json", Required: boolPtr(true)}, {Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(false)}, } @@ -673,7 +674,7 @@ func TestPublishOmitsStaleConfiguredArtifactAndPublishesUnrelatedCurrentArtifact recap.OutputSize = 0 m.Stages["analyze"].AnalyzeArtifacts["session_recap"] = recap env.SelectedArtifactKeys = []string{"session_recap", "player_handout"} - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: "narratio.artifact.session_recap", Dest: "artifacts/session_recap.md", Required: boolPtr(false)}, {Source: "narratio.artifact.player_handout", Dest: "artifacts/player_handout.md", Required: boolPtr(true)}, } @@ -817,7 +818,7 @@ func TestPublishCarriesLockedRequiredOutputAndCommits(t *testing.T) { func TestPublishLockedRequiredMissingOutputSucceeds(t *testing.T) { env, m, _ := publishFixture(t) - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)}, } seedPublishRemoteLock(t, env.ObjectStore.(*storage.FakeBackend), m.S3SessionPrefix, "narratio.transcript.base", "manual merge is locked") @@ -929,7 +930,7 @@ func TestPublishRejectsChangedCarriedObjectMetadata(t *testing.T) { func TestPublishFailsWhenRequiredOutputMissing(t *testing.T) { env, m, _ := publishFixture(t) - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: "narratio.transcript.base", Dest: "transcripts/base.json", Required: boolPtr(true)}, } @@ -1085,7 +1086,7 @@ func TestPublishRejectsAmbiguousOrCollidingOutputMappings(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { env, m, _ := publishFixture(t) - env.Config.Pipeline.Publish.Outputs = test.outputs + env.Config.ArtifactCatalog.PublishOutputs = test.outputs _, err := (publishStage{}).Run(context.Background(), env, m) if err == nil || !strings.Contains(err.Error(), "resolve publish output rules") { t.Fatalf("Run() error = %v, want invalid mapping", err) @@ -1105,7 +1106,7 @@ func TestPublishKeepsSameBasenameSourcesDistinct(t *testing.T) { Enabled: true, PromptID: "dnd.other_recap", OutputPath: "reports/session_recap.md", } setCurrentAnalyzeEvidence(t, m, "other_recap", "reports/session_recap.md", otherPath) - env.Config.Pipeline.Publish.Outputs = []config.PublishOutputRule{ + env.Config.ArtifactCatalog.PublishOutputs = []config.PublishOutputRule{ {Source: "narratio.artifact.session_recap", Dest: "published/first/session_recap.md", Required: boolPtr(true)}, {Source: "narratio.artifact.other_recap", Dest: "published/second/session_recap.md", Required: boolPtr(true)}, } @@ -1233,7 +1234,7 @@ func publishFixture(t *testing.T) (*Env, *manifest.Manifest, string) { }, ObjectStore: &storage.FakeBackend{}, } - env.Config.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: env.Config.Pipeline.Scriptorium.Artifacts} + env.Config.ArtifactCatalog = &config.ArtifactCatalog{Artifacts: env.Config.Pipeline.Scriptorium.Artifacts, PublishOutputs: env.Config.Pipeline.Publish.Outputs} return env, m, runRoot } diff --git a/internal/stage/semantic_contracts_delivery.go b/internal/stage/semantic_contracts_delivery.go index f4e3c23..962b006 100644 --- a/internal/stage/semantic_contracts_delivery.go +++ b/internal/stage/semantic_contracts_delivery.go @@ -143,8 +143,8 @@ func (publishStage) SemanticConfigFingerprint(env *Env) (manifest.SemanticConfig } payload.Enabled = cfg.Enabled == nil || *cfg.Enabled payload.UploadRun = cfg.UploadRun == nil || *cfg.UploadRun - configured, extractions := publishSemanticSources(pipeline) - for _, rule := range cfg.Outputs { + configured, extractions := publishSemanticSources(env.Config) + for _, rule := range env.Config.PublicationOutputs() { source, err := artifactpolicy.ValidatePublishSourceWithExtractions(rule.Source, configured, extractions) if err != nil { return manifest.SemanticConfigFingerprint{}, fmt.Errorf("publish semantic source %q: %w", rule.Source, err) @@ -186,16 +186,16 @@ func (s placeholderStage) SemanticConfigFingerprint(env *Env) (manifest.Semantic return FingerprintSemanticConfig(notifySemanticConfigVersion, notifySemanticConfig{Mode: mode}) } -func publishSemanticSources(pipeline *config.PipelineConfig) (map[string]string, map[string]struct{}) { +func publishSemanticSources(cfg *config.Config) (map[string]string, map[string]struct{}) { configured := map[string]string{} - if pipeline.Scriptorium != nil { - for key, artifact := range pipeline.Scriptorium.Artifacts { + if cfg.ArtifactCatalog != nil { + for key, artifact := range cfg.ConcreteArtifacts() { configured[key] = artifact.OutputPath } } extractions := map[string]struct{}{} - if pipeline.Notarius != nil { - for key := range pipeline.Notarius.Outputs { + if cfg.Pipeline.Notarius != nil { + for key := range cfg.Pipeline.Notarius.Outputs { extractions[key] = struct{}{} } } diff --git a/internal/stage/semantic_contracts_delivery_test.go b/internal/stage/semantic_contracts_delivery_test.go index b4bf144..9d80460 100644 --- a/internal/stage/semantic_contracts_delivery_test.go +++ b/internal/stage/semantic_contracts_delivery_test.go @@ -6,10 +6,10 @@ import ( "path/filepath" "testing" - configkit "go.maximumdirect.net/configkit" "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" "gitea.maximumdirect.net/eric/narratio/internal/config" "gitea.maximumdirect.net/eric/narratio/internal/manifest" + configkit "go.maximumdirect.net/configkit" ) func testSecretReference(name string) *configkit.Secret { @@ -257,9 +257,11 @@ func TestPublishSemanticConfigSensitivityAndOperationalExclusions(t *testing.T) }{ {name: "enablement", mutate: func(env *Env) { env.Config.Pipeline.Publish.Enabled = boolPointer(false) }}, {name: "run upload", mutate: func(env *Env) { env.Config.Pipeline.Publish.UploadRun = boolPointer(false) }}, - {name: "source", mutate: func(env *Env) { env.Config.Pipeline.Publish.Outputs[0].Source = "narratio.transcript.final_markdown" }}, - {name: "destination", mutate: func(env *Env) { env.Config.Pipeline.Publish.Outputs[0].Dest = "published/other.json" }}, - {name: "required", mutate: func(env *Env) { env.Config.Pipeline.Publish.Outputs[0].Required = boolPointer(false) }}, + {name: "source", mutate: func(env *Env) { + env.Config.ArtifactCatalog.PublishOutputs[0].Source = "narratio.transcript.final_markdown" + }}, + {name: "destination", mutate: func(env *Env) { env.Config.ArtifactCatalog.PublishOutputs[0].Dest = "published/other.json" }}, + {name: "required", mutate: func(env *Env) { env.Config.ArtifactCatalog.PublishOutputs[0].Required = boolPointer(false) }}, {name: "backend", mutate: func(env *Env) { env.Config.Pipeline.Storage.Backend = config.StorageBackendLocal }}, {name: "bucket", mutate: func(env *Env) { env.Config.Pipeline.Storage.S3.Bucket = "other-bucket" }}, {name: "region", mutate: func(env *Env) { env.Config.Pipeline.Storage.S3.Region = "eu-west-1" }}, @@ -296,8 +298,8 @@ func TestPublishSemanticConfigSensitivityAndOperationalExclusions(t *testing.T) func TestDeliverySemanticConfigCanonicalOrderingAndNotifyMode(t *testing.T) { first := downstreamSemanticEnv() second := downstreamSemanticEnv() - second.Config.Pipeline.Publish.Outputs[0], second.Config.Pipeline.Publish.Outputs[1] = - second.Config.Pipeline.Publish.Outputs[1], second.Config.Pipeline.Publish.Outputs[0] + second.Config.ArtifactCatalog.PublishOutputs[0], second.Config.ArtifactCatalog.PublishOutputs[1] = + second.Config.ArtifactCatalog.PublishOutputs[1], second.Config.ArtifactCatalog.PublishOutputs[0] firstPublish, err := (publishStage{}).SemanticConfigFingerprint(first) if err != nil { t.Fatal(err) @@ -353,7 +355,7 @@ func TestCanonicalStageSemanticReuseCoverage(t *testing.T) { } func downstreamSemanticEnv() *Env { - return &Env{Config: &config.Config{ + env := &Env{Config: &config.Config{ Pipeline: &config.PipelineConfig{ Workspace: config.WorkspaceConfig{Root: "/workspace"}, Storage: config.StorageConfig{Backend: config.StorageBackendS3, S3: &config.StorageS3Config{ @@ -382,4 +384,6 @@ func downstreamSemanticEnv() *Env { }, Session: &config.SessionConfig{SessionID: "2026-08-30", Campaign: "campaign"}, }} + env.Config.ArtifactCatalog = &config.ArtifactCatalog{PublishOutputs: env.Config.Pipeline.Publish.Outputs} + return env } diff --git a/internal/stage/semantic_contracts_initial.go b/internal/stage/semantic_contracts_initial.go index 0f39985..f3676b7 100644 --- a/internal/stage/semantic_contracts_initial.go +++ b/internal/stage/semantic_contracts_initial.go @@ -119,7 +119,7 @@ func buildPrepareSemanticConfig(env *Env) (prepareSemanticConfig, error) { for _, requirement := range requirements { entry := preparePreviousRequirement{Name: strings.TrimSpace(requirement.Name), Required: requirement.Required} if cfg.Pipeline.Scriptorium != nil { - entry.OutputPath = strings.TrimSpace(cfg.Pipeline.Scriptorium.Artifacts[requirement.Name].OutputPath) + entry.OutputPath = strings.TrimSpace(cfg.ConcreteArtifacts()[requirement.Name].OutputPath) } payload.PreviousArtifacts = append(payload.PreviousArtifacts, entry) } diff --git a/internal/stage/transcribe_test.go b/internal/stage/transcribe_test.go index a3fac41..103e807 100644 --- a/internal/stage/transcribe_test.go +++ b/internal/stage/transcribe_test.go @@ -441,7 +441,7 @@ func setupTranscribeEnv(t *testing.T, audioFiles []string) (*Env, *manifest.Mani CampaignPath: campaignPath, SessionPath: sessionPath, Campaign: &config.CampaignConfig{CampaignID: "sample-campaign"}, - Party: config.ResolvedParty{Canonical: party, Source: config.PartySource{ + Party: config.ResolvedParty{Canonical: party, Players: projectedTestPlayers(t, party), Source: config.PartySource{ Path: filepath.Join(cfgDir, "party.yml"), ConfigPath: campaignPath, Source: "campaign_config", }}, Pipeline: &config.PipelineConfig{