Render final configuration through native snapshots

This commit is contained in:
2026-11-17 22:31:03 +00:00
parent ed7e722077
commit ce661252d0
7 changed files with 156 additions and 77 deletions

View File

@@ -4,11 +4,13 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
configkit "go.maximumdirect.net/configkit"
)
const (
pipelineDefaultOwnershipSource = "default"
effectivePipelineDigestVersion = "narratio-effective-pipeline-v3\n"
effectivePipelineDigestVersion = "narratio-effective-pipeline-v4\n"
)
// PipelineOverlayProvenance identifies the overlay selected while resolving a
@@ -51,12 +53,22 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
if cfg == nil || cfg.resolution == nil {
return fmt.Errorf("pipeline resolution metadata is required")
}
canonical, err := projectionJSON(projectEffectivePipeline(cfg))
snapshot, err := newPipelineInspection(cfg)
if err != nil {
return err
}
canonical, err := projectionJSON(snapshot.Config)
if err != nil {
return err
}
digestInput := append([]byte(effectivePipelineDigestVersion), canonical...)
digest := sha256.Sum256(digestInput)
cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:])
cfg.resolution.inspection = snapshot
return nil
}
func newPipelineInspection(cfg *PipelineConfig) (*configkit.Result[effectivePipelineProjection], error) {
projection := projectEffectivePipeline(cfg)
return configkit.Load(configkit.NoSource(), configkit.Options[effectivePipelineProjection]{Definition: configkit.Definition[effectivePipelineProjection]{Defaults: func() effectivePipelineProjection { return projection }}})
}

View File

@@ -1,15 +1,13 @@
package config
import (
"encoding/json"
"bytes"
"fmt"
"path/filepath"
"sort"
"strconv"
"strings"
configkit "go.maximumdirect.net/configkit"
"gopkg.in/yaml.v3"
)
// EffectivePipelineRootPath reports the absolute root pipeline path retained
@@ -157,70 +155,22 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) {
return nil, fmt.Errorf("pipeline config is required")
}
value, err := projectionValueTree(projectEffectivePipeline(cfg))
if err != nil {
return nil, err
var snapshot *configkit.Result[effectivePipelineProjection]
if cfg.resolution != nil {
snapshot = cfg.resolution.inspection
}
node, err := effectiveYAMLNode(value)
if err != nil {
if snapshot == nil {
var err error
snapshot, err = newPipelineInspection(cfg)
if err != nil {
return nil, err
}
}
var output bytes.Buffer
if err := snapshot.WriteEffective(&output); err != nil {
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
}
data, err := yaml.Marshal(node)
if err != nil {
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
}
return data, nil
}
func effectiveYAMLNode(value any) (*yaml.Node, error) {
switch value := value.(type) {
case nil:
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null", Value: "null"}, nil
case bool:
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(value)}, nil
case string:
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, nil
case json.Number:
tag := "!!int"
if strings.ContainsAny(value.String(), ".eE") {
tag = "!!float"
}
return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value.String()}, nil
case []any:
node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
if len(value) == 0 {
node.Style = yaml.FlowStyle
}
for _, item := range value {
child, err := effectiveYAMLNode(item)
if err != nil {
return nil, err
}
node.Content = append(node.Content, child)
}
return node, nil
case map[string]any:
node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
if len(value) == 0 {
node.Style = yaml.FlowStyle
}
keys := make([]string, 0, len(value))
for key := range value {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
child, err := effectiveYAMLNode(value[key])
if err != nil {
return nil, err
}
node.Content = append(node.Content,
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, child)
}
return node, nil
default:
return nil, fmt.Errorf("unsupported projected value type %T", value)
}
return output.Bytes(), nil
}
func effectivePipelineOwners(cfg *PipelineConfig) (map[string][]string, error) {

View File

@@ -14,23 +14,63 @@ import (
type effectivePipelineProjection struct {
Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"`
Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"`
Storage StorageConfig `yaml:"storage" json:"storage"`
Storage effectiveStorageConfig `yaml:"storage" json:"storage"`
Spool SpoolConfig `yaml:"spool" json:"spool"`
Cache CacheConfig `yaml:"cache" json:"cache"`
Publish *PublishConfig `yaml:"publish" json:"publish"`
WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"`
Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"`
Audita AuditaConfig `yaml:"audita" json:"audita"`
Audita effectiveAuditaConfig `yaml:"audita" json:"audita"`
Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"`
Trim *TrimConfig `yaml:"trim" json:"trim"`
Render *RenderConfig `yaml:"render" json:"render"`
Scriptorium *effectiveScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"`
Notarius *NotariusConfig `yaml:"notarius" json:"notarius"`
Notarius *effectiveNotariusConfig `yaml:"notarius" json:"notarius"`
Notification NotificationConfig `yaml:"notification" json:"notification"`
}
type effectiveStorageConfig struct {
Backend string `yaml:"backend" json:"backend"`
S3 *effectiveStorageS3Config `yaml:"s3" json:"s3"`
}
type effectiveStorageS3Config struct {
Bucket string `yaml:"bucket" json:"bucket"`
RootPrefix string `yaml:"root_prefix" json:"root_prefix"`
Region string `yaml:"region" json:"region"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"`
AccessKeyID *string `yaml:"access_key_id,omitempty" json:"access_key_id,omitempty"`
SecretAccessKey *string `yaml:"secret_access_key,omitempty" json:"secret_access_key,omitempty"`
}
type effectiveAuditaConfig struct {
Binary string `yaml:"binary" json:"binary"`
Timeout string `yaml:"timeout" json:"timeout"`
Credentials []string `yaml:"credentials" json:"credentials"`
Modules []string `yaml:"modules" json:"modules"`
Overlay string `yaml:"overlay" json:"overlay"`
ProposalProfile string `yaml:"proposal_profile" json:"proposal_profile"`
ValidationProfile string `yaml:"validation_profile" json:"validation_profile"`
TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"`
ConfigPath string `yaml:"config_path" json:"config_path"`
OutputSchema string `yaml:"output_schema" json:"output_schema"`
}
type effectiveNotariusConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"`
Binary string `yaml:"binary" json:"binary"`
Credentials []string `yaml:"credentials" json:"credentials"`
ConfigOverlay *string `yaml:"config_overlay,omitempty" json:"config_overlay,omitempty"`
Resume bool `yaml:"resume,omitempty" json:"resume,omitempty"`
ChunkCache string `yaml:"chunk_cache,omitempty" json:"chunk_cache,omitempty"`
ConfigPath string `yaml:"config_path" json:"config_path"`
PipelineID string `yaml:"pipeline_id" json:"pipeline_id"`
Timeout string `yaml:"timeout" json:"timeout"`
WorkingDirectory string `yaml:"working_directory" json:"working_directory"`
References map[string]string `yaml:"references" json:"references"`
Outputs map[string]NotariusOutputConfig `yaml:"outputs" json:"outputs"`
}
type effectiveScriptoriumConfig struct {
Credentials []string `yaml:"credentials,omitempty" json:"credentials,omitempty"`
Credentials []string `yaml:"credentials" json:"credentials"`
Binary string `yaml:"binary" json:"binary"`
ConfigPath string `yaml:"config_path" json:"config_path"`
Timeout string `yaml:"timeout" json:"timeout"`
@@ -45,11 +85,19 @@ type projectedPipelineValue struct {
func projectEffectivePipeline(cfg *PipelineConfig) effectivePipelineProjection {
projection := effectivePipelineProjection{
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: effectiveStorageConfig{Backend: cfg.Storage.Backend},
Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish,
WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim, Audita: cfg.Audita,
WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim,
Normalize: cfg.Normalize, Trim: cfg.Trim, Render: cfg.Render,
Notarius: cfg.Notarius, Notification: cfg.Notification,
Notification: cfg.Notification,
}
if s3 := cfg.Storage.S3; s3 != nil {
projection.Storage.S3 = &effectiveStorageS3Config{Bucket: s3.Bucket, RootPrefix: s3.RootPrefix, Region: s3.Region, Endpoint: s3.Endpoint, ForcePathStyle: s3.ForcePathStyle, AccessKeyID: secretReferenceName(s3.AccessKeyID), SecretAccessKey: secretReferenceName(s3.SecretAccessKey)}
}
audita := cfg.Audita
projection.Audita = effectiveAuditaConfig{Binary: audita.Binary, Timeout: audita.Timeout, Credentials: secretReferenceNames(audita.Credentials), Modules: audita.Modules, Overlay: audita.Overlay, ProposalProfile: audita.ProposalProfile, ValidationProfile: audita.ValidationProfile, TranscriptDescription: audita.TranscriptDescription, ConfigPath: audita.ConfigPath, OutputSchema: audita.OutputSchema}
if n := cfg.Notarius; n != nil {
projection.Notarius = &effectiveNotariusConfig{Enabled: n.Enabled, Binary: n.Binary, Credentials: secretReferenceNames(n.Credentials), ConfigOverlay: n.ConfigOverlay, Resume: n.Resume, ChunkCache: n.ChunkCache, ConfigPath: n.ConfigPath, PipelineID: n.PipelineID, Timeout: n.Timeout, WorkingDirectory: n.WorkingDirectory, References: n.References, Outputs: n.Outputs}
}
if cfg.Scriptorium != nil {
projection.Scriptorium = &effectiveScriptoriumConfig{

View File

@@ -0,0 +1,65 @@
package config
import (
"os"
"strings"
"testing"
)
func TestInspectionSnapshotPreservesFinalValuesWithoutResolvingCredentialsAgain(t *testing.T) {
const name = "NARRATIO_SNAPSHOT_TEST_KEY"
const secret = "synthetic-snapshot-secret"
t.Setenv(name, secret)
path := writePipelineFixture(t, t.TempDir(), "pipeline.yml", `audita:
credentials: [NARRATIO_SNAPSHOT_TEST_KEY]
scriptorium:
artifacts:
recap:
vars:
exact: 9007199254740993
nothing: null
empty: []
`)
cfg, err := LoadPipeline(path)
if err != nil {
t.Fatal(err)
}
artifact := cfg.Scriptorium.Artifacts["recap"]
artifact.Vars["config"] = map[string]any{"extends": "ordinary-data"}
cfg.Scriptorium.Artifacts["recap"] = artifact
if err := os.Unsetenv(name); err != nil {
t.Fatal(err)
}
if err := recomputePipelineEffectiveDigest(cfg); err != nil {
t.Fatalf("snapshot resolved credentials again: %v", err)
}
data, err := MarshalEffectivePipeline(cfg)
if err != nil {
t.Fatal(err)
}
for _, expected := range []string{name, "9007199254740993", "ordinary-data", "nothing: null", "empty: []"} {
if !strings.Contains(string(data), expected) {
t.Fatalf("snapshot missing %q: %s", expected, data)
}
}
if strings.Contains(string(data), secret) {
t.Fatal("snapshot exposed resolved credential")
}
}
func TestInspectionSnapshotKeepsEmptyCredentialListDistinctFromNull(t *testing.T) {
dir := t.TempDir()
nullPath := writePipelineFixture(t, dir, "null.yml", "audita: {credentials: null}\n")
emptyPath := writePipelineFixture(t, dir, "empty.yml", "audita: {credentials: []}\n")
nullConfig, err := LoadPipeline(nullPath)
if err != nil {
t.Fatal(err)
}
emptyConfig, err := LoadPipeline(emptyPath)
if err != nil {
t.Fatal(err)
}
if EffectivePipelineDigest(nullConfig) == EffectivePipelineDigest(emptyConfig) {
t.Fatal("null and empty credentials have the same digest")
}
}

View File

@@ -141,9 +141,6 @@ notarius:
}
}
}
if !strings.Contains(string(firstYAML), "config_overlay: \"\"\n") {
t.Fatalf("effective YAML omitted explicit empty Notarius config overlay: %q", firstYAML)
}
var roundTrip struct {
Notarius *struct {
Credentials []string `yaml:"credentials"`
@@ -221,6 +218,9 @@ func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
nilDigest := EffectivePipelineDigest(base)
base.Audita.Modules = []string{}
base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{}
if err := recomputePipelineEffectiveDigest(base); err != nil {
t.Fatal(err)
}
emptyYAML, err := MarshalEffectivePipeline(base)
if err != nil {
t.Fatal(err)
@@ -250,6 +250,9 @@ func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
if got := effectiveValueForTest(t, emptyValues, "scriptorium.artifacts.recap.vars.exact"); got != "9007199254740993" {
t.Fatalf("exact integer value = %q", got)
}
if err := recomputePipelineEffectiveDigest(base); err != nil {
t.Fatal(err)
}
encoded, err := MarshalEffectivePipeline(base)
if err != nil {
t.Fatal(err)

View File

@@ -22,6 +22,7 @@ type pipelineResolutionMetadata struct {
artifactFamiliesExpanded bool
configkitMetadata configkit.Metadata
configkitSources []configkit.SourceInfo
inspection *configkit.Result[effectivePipelineProjection]
}
type pipelineOverlaySelection struct {