Render final configuration through native snapshots
This commit is contained in:
@@ -4,11 +4,13 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
const (
|
||||
pipelineDefaultOwnershipSource = "default"
|
||||
effectivePipelineDigestVersion = "narratio-effective-pipeline-v3\n"
|
||||
effectivePipelineDigestVersion = "narratio-effective-pipeline-v4\n"
|
||||
)
|
||||
|
||||
// PipelineOverlayProvenance identifies the overlay selected while resolving a
|
||||
@@ -51,12 +53,22 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return fmt.Errorf("pipeline resolution metadata is required")
|
||||
}
|
||||
canonical, err := projectionJSON(projectEffectivePipeline(cfg))
|
||||
snapshot, err := newPipelineInspection(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
canonical, err := projectionJSON(snapshot.Config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digestInput := append([]byte(effectivePipelineDigestVersion), canonical...)
|
||||
digest := sha256.Sum256(digestInput)
|
||||
cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:])
|
||||
cfg.resolution.inspection = snapshot
|
||||
return nil
|
||||
}
|
||||
|
||||
func newPipelineInspection(cfg *PipelineConfig) (*configkit.Result[effectivePipelineProjection], error) {
|
||||
projection := projectEffectivePipeline(cfg)
|
||||
return configkit.Load(configkit.NoSource(), configkit.Options[effectivePipelineProjection]{Definition: configkit.Definition[effectivePipelineProjection]{Defaults: func() effectivePipelineProjection { return projection }}})
|
||||
}
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// EffectivePipelineRootPath reports the absolute root pipeline path retained
|
||||
@@ -157,70 +155,22 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) {
|
||||
return nil, fmt.Errorf("pipeline config is required")
|
||||
}
|
||||
|
||||
value, err := projectionValueTree(projectEffectivePipeline(cfg))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var snapshot *configkit.Result[effectivePipelineProjection]
|
||||
if cfg.resolution != nil {
|
||||
snapshot = cfg.resolution.inspection
|
||||
}
|
||||
node, err := effectiveYAMLNode(value)
|
||||
if err != nil {
|
||||
if snapshot == nil {
|
||||
var err error
|
||||
snapshot, err = newPipelineInspection(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var output bytes.Buffer
|
||||
if err := snapshot.WriteEffective(&output); err != nil {
|
||||
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
|
||||
}
|
||||
data, err := yaml.Marshal(node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize effective pipeline: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func effectiveYAMLNode(value any) (*yaml.Node, error) {
|
||||
switch value := value.(type) {
|
||||
case nil:
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null", Value: "null"}, nil
|
||||
case bool:
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(value)}, nil
|
||||
case string:
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, nil
|
||||
case json.Number:
|
||||
tag := "!!int"
|
||||
if strings.ContainsAny(value.String(), ".eE") {
|
||||
tag = "!!float"
|
||||
}
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value.String()}, nil
|
||||
case []any:
|
||||
node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
|
||||
if len(value) == 0 {
|
||||
node.Style = yaml.FlowStyle
|
||||
}
|
||||
for _, item := range value {
|
||||
child, err := effectiveYAMLNode(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node.Content = append(node.Content, child)
|
||||
}
|
||||
return node, nil
|
||||
case map[string]any:
|
||||
node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
|
||||
if len(value) == 0 {
|
||||
node.Style = yaml.FlowStyle
|
||||
}
|
||||
keys := make([]string, 0, len(value))
|
||||
for key := range value {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
child, err := effectiveYAMLNode(value[key])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node.Content = append(node.Content,
|
||||
&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, child)
|
||||
}
|
||||
return node, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported projected value type %T", value)
|
||||
}
|
||||
return output.Bytes(), nil
|
||||
}
|
||||
|
||||
func effectivePipelineOwners(cfg *PipelineConfig) (map[string][]string, error) {
|
||||
|
||||
@@ -14,23 +14,63 @@ import (
|
||||
type effectivePipelineProjection struct {
|
||||
Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"`
|
||||
Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"`
|
||||
Storage StorageConfig `yaml:"storage" json:"storage"`
|
||||
Storage effectiveStorageConfig `yaml:"storage" json:"storage"`
|
||||
Spool SpoolConfig `yaml:"spool" json:"spool"`
|
||||
Cache CacheConfig `yaml:"cache" json:"cache"`
|
||||
Publish *PublishConfig `yaml:"publish" json:"publish"`
|
||||
WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"`
|
||||
Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"`
|
||||
Audita AuditaConfig `yaml:"audita" json:"audita"`
|
||||
Audita effectiveAuditaConfig `yaml:"audita" json:"audita"`
|
||||
Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"`
|
||||
Trim *TrimConfig `yaml:"trim" json:"trim"`
|
||||
Render *RenderConfig `yaml:"render" json:"render"`
|
||||
Scriptorium *effectiveScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"`
|
||||
Notarius *NotariusConfig `yaml:"notarius" json:"notarius"`
|
||||
Notarius *effectiveNotariusConfig `yaml:"notarius" json:"notarius"`
|
||||
Notification NotificationConfig `yaml:"notification" json:"notification"`
|
||||
}
|
||||
|
||||
type effectiveStorageConfig struct {
|
||||
Backend string `yaml:"backend" json:"backend"`
|
||||
S3 *effectiveStorageS3Config `yaml:"s3" json:"s3"`
|
||||
}
|
||||
type effectiveStorageS3Config struct {
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
RootPrefix string `yaml:"root_prefix" json:"root_prefix"`
|
||||
Region string `yaml:"region" json:"region"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"`
|
||||
AccessKeyID *string `yaml:"access_key_id,omitempty" json:"access_key_id,omitempty"`
|
||||
SecretAccessKey *string `yaml:"secret_access_key,omitempty" json:"secret_access_key,omitempty"`
|
||||
}
|
||||
type effectiveAuditaConfig struct {
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
Credentials []string `yaml:"credentials" json:"credentials"`
|
||||
Modules []string `yaml:"modules" json:"modules"`
|
||||
Overlay string `yaml:"overlay" json:"overlay"`
|
||||
ProposalProfile string `yaml:"proposal_profile" json:"proposal_profile"`
|
||||
ValidationProfile string `yaml:"validation_profile" json:"validation_profile"`
|
||||
TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"`
|
||||
ConfigPath string `yaml:"config_path" json:"config_path"`
|
||||
OutputSchema string `yaml:"output_schema" json:"output_schema"`
|
||||
}
|
||||
type effectiveNotariusConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
Credentials []string `yaml:"credentials" json:"credentials"`
|
||||
ConfigOverlay *string `yaml:"config_overlay,omitempty" json:"config_overlay,omitempty"`
|
||||
Resume bool `yaml:"resume,omitempty" json:"resume,omitempty"`
|
||||
ChunkCache string `yaml:"chunk_cache,omitempty" json:"chunk_cache,omitempty"`
|
||||
ConfigPath string `yaml:"config_path" json:"config_path"`
|
||||
PipelineID string `yaml:"pipeline_id" json:"pipeline_id"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
WorkingDirectory string `yaml:"working_directory" json:"working_directory"`
|
||||
References map[string]string `yaml:"references" json:"references"`
|
||||
Outputs map[string]NotariusOutputConfig `yaml:"outputs" json:"outputs"`
|
||||
}
|
||||
|
||||
type effectiveScriptoriumConfig struct {
|
||||
Credentials []string `yaml:"credentials,omitempty" json:"credentials,omitempty"`
|
||||
Credentials []string `yaml:"credentials" json:"credentials"`
|
||||
Binary string `yaml:"binary" json:"binary"`
|
||||
ConfigPath string `yaml:"config_path" json:"config_path"`
|
||||
Timeout string `yaml:"timeout" json:"timeout"`
|
||||
@@ -45,11 +85,19 @@ type projectedPipelineValue struct {
|
||||
|
||||
func projectEffectivePipeline(cfg *PipelineConfig) effectivePipelineProjection {
|
||||
projection := effectivePipelineProjection{
|
||||
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
|
||||
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: effectiveStorageConfig{Backend: cfg.Storage.Backend},
|
||||
Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish,
|
||||
WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim, Audita: cfg.Audita,
|
||||
WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim,
|
||||
Normalize: cfg.Normalize, Trim: cfg.Trim, Render: cfg.Render,
|
||||
Notarius: cfg.Notarius, Notification: cfg.Notification,
|
||||
Notification: cfg.Notification,
|
||||
}
|
||||
if s3 := cfg.Storage.S3; s3 != nil {
|
||||
projection.Storage.S3 = &effectiveStorageS3Config{Bucket: s3.Bucket, RootPrefix: s3.RootPrefix, Region: s3.Region, Endpoint: s3.Endpoint, ForcePathStyle: s3.ForcePathStyle, AccessKeyID: secretReferenceName(s3.AccessKeyID), SecretAccessKey: secretReferenceName(s3.SecretAccessKey)}
|
||||
}
|
||||
audita := cfg.Audita
|
||||
projection.Audita = effectiveAuditaConfig{Binary: audita.Binary, Timeout: audita.Timeout, Credentials: secretReferenceNames(audita.Credentials), Modules: audita.Modules, Overlay: audita.Overlay, ProposalProfile: audita.ProposalProfile, ValidationProfile: audita.ValidationProfile, TranscriptDescription: audita.TranscriptDescription, ConfigPath: audita.ConfigPath, OutputSchema: audita.OutputSchema}
|
||||
if n := cfg.Notarius; n != nil {
|
||||
projection.Notarius = &effectiveNotariusConfig{Enabled: n.Enabled, Binary: n.Binary, Credentials: secretReferenceNames(n.Credentials), ConfigOverlay: n.ConfigOverlay, Resume: n.Resume, ChunkCache: n.ChunkCache, ConfigPath: n.ConfigPath, PipelineID: n.PipelineID, Timeout: n.Timeout, WorkingDirectory: n.WorkingDirectory, References: n.References, Outputs: n.Outputs}
|
||||
}
|
||||
if cfg.Scriptorium != nil {
|
||||
projection.Scriptorium = &effectiveScriptoriumConfig{
|
||||
|
||||
65
internal/config/inspection_snapshot_test.go
Normal file
65
internal/config/inspection_snapshot_test.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInspectionSnapshotPreservesFinalValuesWithoutResolvingCredentialsAgain(t *testing.T) {
|
||||
const name = "NARRATIO_SNAPSHOT_TEST_KEY"
|
||||
const secret = "synthetic-snapshot-secret"
|
||||
t.Setenv(name, secret)
|
||||
path := writePipelineFixture(t, t.TempDir(), "pipeline.yml", `audita:
|
||||
credentials: [NARRATIO_SNAPSHOT_TEST_KEY]
|
||||
scriptorium:
|
||||
artifacts:
|
||||
recap:
|
||||
vars:
|
||||
exact: 9007199254740993
|
||||
nothing: null
|
||||
empty: []
|
||||
`)
|
||||
cfg, err := LoadPipeline(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
artifact := cfg.Scriptorium.Artifacts["recap"]
|
||||
artifact.Vars["config"] = map[string]any{"extends": "ordinary-data"}
|
||||
cfg.Scriptorium.Artifacts["recap"] = artifact
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recomputePipelineEffectiveDigest(cfg); err != nil {
|
||||
t.Fatalf("snapshot resolved credentials again: %v", err)
|
||||
}
|
||||
data, err := MarshalEffectivePipeline(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, expected := range []string{name, "9007199254740993", "ordinary-data", "nothing: null", "empty: []"} {
|
||||
if !strings.Contains(string(data), expected) {
|
||||
t.Fatalf("snapshot missing %q: %s", expected, data)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(data), secret) {
|
||||
t.Fatal("snapshot exposed resolved credential")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectionSnapshotKeepsEmptyCredentialListDistinctFromNull(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
nullPath := writePipelineFixture(t, dir, "null.yml", "audita: {credentials: null}\n")
|
||||
emptyPath := writePipelineFixture(t, dir, "empty.yml", "audita: {credentials: []}\n")
|
||||
nullConfig, err := LoadPipeline(nullPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
emptyConfig, err := LoadPipeline(emptyPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if EffectivePipelineDigest(nullConfig) == EffectivePipelineDigest(emptyConfig) {
|
||||
t.Fatal("null and empty credentials have the same digest")
|
||||
}
|
||||
}
|
||||
@@ -141,9 +141,6 @@ notarius:
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(firstYAML), "config_overlay: \"\"\n") {
|
||||
t.Fatalf("effective YAML omitted explicit empty Notarius config overlay: %q", firstYAML)
|
||||
}
|
||||
var roundTrip struct {
|
||||
Notarius *struct {
|
||||
Credentials []string `yaml:"credentials"`
|
||||
@@ -221,6 +218,9 @@ func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
|
||||
nilDigest := EffectivePipelineDigest(base)
|
||||
base.Audita.Modules = []string{}
|
||||
base.Scriptorium.Artifacts = map[string]ScriptoriumArtifactConfig{}
|
||||
if err := recomputePipelineEffectiveDigest(base); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
emptyYAML, err := MarshalEffectivePipeline(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -250,6 +250,9 @@ func TestEffectivePipelineValuesPreserveNilEmptyAndExactIntegers(t *testing.T) {
|
||||
if got := effectiveValueForTest(t, emptyValues, "scriptorium.artifacts.recap.vars.exact"); got != "9007199254740993" {
|
||||
t.Fatalf("exact integer value = %q", got)
|
||||
}
|
||||
if err := recomputePipelineEffectiveDigest(base); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encoded, err := MarshalEffectivePipeline(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -22,6 +22,7 @@ type pipelineResolutionMetadata struct {
|
||||
artifactFamiliesExpanded bool
|
||||
configkitMetadata configkit.Metadata
|
||||
configkitSources []configkit.SourceInfo
|
||||
inspection *configkit.Result[effectivePipelineProjection]
|
||||
}
|
||||
|
||||
type pipelineOverlaySelection struct {
|
||||
|
||||
Reference in New Issue
Block a user