2026-09-07 03:35:46 +00:00
|
|
|
package config
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"fmt"
|
|
|
|
|
"sort"
|
2026-09-07 03:54:41 +00:00
|
|
|
"strings"
|
2026-09-07 03:35:46 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// effectivePipelineProjection is the single safe, schema-specific view used
|
|
|
|
|
// for inspection, comparison, and identity. Runtime metadata and unresolved
|
|
|
|
|
// family declarations have no slot in this type.
|
|
|
|
|
type effectivePipelineProjection struct {
|
|
|
|
|
Workspace WorkspaceConfig `yaml:"workspace" json:"workspace"`
|
|
|
|
|
Campaigns CampaignsConfig `yaml:"campaigns" json:"campaigns"`
|
2026-11-17 22:31:03 +00:00
|
|
|
Storage effectiveStorageConfig `yaml:"storage" json:"storage"`
|
2026-09-07 03:35:46 +00:00
|
|
|
Spool SpoolConfig `yaml:"spool" json:"spool"`
|
|
|
|
|
Cache CacheConfig `yaml:"cache" json:"cache"`
|
|
|
|
|
Publish *PublishConfig `yaml:"publish" json:"publish"`
|
|
|
|
|
WhisperX WhisperXConfig `yaml:"whisperx" json:"whisperx"`
|
|
|
|
|
Seriatim SeriatimConfig `yaml:"seriatim" json:"seriatim"`
|
2026-11-17 22:31:03 +00:00
|
|
|
Audita effectiveAuditaConfig `yaml:"audita" json:"audita"`
|
2026-09-07 03:35:46 +00:00
|
|
|
Normalize *NormalizeConfig `yaml:"normalize" json:"normalize"`
|
|
|
|
|
Trim *TrimConfig `yaml:"trim" json:"trim"`
|
|
|
|
|
Render *RenderConfig `yaml:"render" json:"render"`
|
|
|
|
|
Scriptorium *effectiveScriptoriumConfig `yaml:"scriptorium" json:"scriptorium"`
|
2026-11-17 22:31:03 +00:00
|
|
|
Notarius *effectiveNotariusConfig `yaml:"notarius" json:"notarius"`
|
2026-09-07 03:35:46 +00:00
|
|
|
Notification NotificationConfig `yaml:"notification" json:"notification"`
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 22:31:03 +00:00
|
|
|
type effectiveStorageConfig struct {
|
|
|
|
|
Backend string `yaml:"backend" json:"backend"`
|
|
|
|
|
S3 *effectiveStorageS3Config `yaml:"s3" json:"s3"`
|
|
|
|
|
}
|
|
|
|
|
type effectiveStorageS3Config struct {
|
|
|
|
|
Bucket string `yaml:"bucket" json:"bucket"`
|
|
|
|
|
RootPrefix string `yaml:"root_prefix" json:"root_prefix"`
|
|
|
|
|
Region string `yaml:"region" json:"region"`
|
|
|
|
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
|
|
|
|
ForcePathStyle bool `yaml:"force_path_style" json:"force_path_style"`
|
|
|
|
|
AccessKeyID *string `yaml:"access_key_id,omitempty" json:"access_key_id,omitempty"`
|
|
|
|
|
SecretAccessKey *string `yaml:"secret_access_key,omitempty" json:"secret_access_key,omitempty"`
|
|
|
|
|
}
|
|
|
|
|
type effectiveAuditaConfig struct {
|
|
|
|
|
Binary string `yaml:"binary" json:"binary"`
|
|
|
|
|
Timeout string `yaml:"timeout" json:"timeout"`
|
|
|
|
|
Credentials []string `yaml:"credentials" json:"credentials"`
|
|
|
|
|
Modules []string `yaml:"modules" json:"modules"`
|
|
|
|
|
Overlay string `yaml:"overlay" json:"overlay"`
|
|
|
|
|
ProposalProfile string `yaml:"proposal_profile" json:"proposal_profile"`
|
|
|
|
|
ValidationProfile string `yaml:"validation_profile" json:"validation_profile"`
|
|
|
|
|
TranscriptDescription string `yaml:"transcript_description" json:"transcript_description"`
|
|
|
|
|
ConfigPath string `yaml:"config_path" json:"config_path"`
|
|
|
|
|
OutputSchema string `yaml:"output_schema" json:"output_schema"`
|
|
|
|
|
}
|
|
|
|
|
type effectiveNotariusConfig struct {
|
|
|
|
|
Enabled bool `yaml:"enabled" json:"enabled"`
|
|
|
|
|
Binary string `yaml:"binary" json:"binary"`
|
|
|
|
|
Credentials []string `yaml:"credentials" json:"credentials"`
|
|
|
|
|
ConfigOverlay *string `yaml:"config_overlay,omitempty" json:"config_overlay,omitempty"`
|
|
|
|
|
Resume bool `yaml:"resume,omitempty" json:"resume,omitempty"`
|
|
|
|
|
ChunkCache string `yaml:"chunk_cache,omitempty" json:"chunk_cache,omitempty"`
|
|
|
|
|
ConfigPath string `yaml:"config_path" json:"config_path"`
|
|
|
|
|
PipelineID string `yaml:"pipeline_id" json:"pipeline_id"`
|
|
|
|
|
Timeout string `yaml:"timeout" json:"timeout"`
|
|
|
|
|
WorkingDirectory string `yaml:"working_directory" json:"working_directory"`
|
|
|
|
|
References map[string]string `yaml:"references" json:"references"`
|
|
|
|
|
Outputs map[string]NotariusOutputConfig `yaml:"outputs" json:"outputs"`
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-07 03:35:46 +00:00
|
|
|
type effectiveScriptoriumConfig struct {
|
2026-11-17 22:31:03 +00:00
|
|
|
Credentials []string `yaml:"credentials" json:"credentials"`
|
2026-09-07 03:35:46 +00:00
|
|
|
Binary string `yaml:"binary" json:"binary"`
|
|
|
|
|
ConfigPath string `yaml:"config_path" json:"config_path"`
|
|
|
|
|
Timeout string `yaml:"timeout" json:"timeout"`
|
|
|
|
|
RenderDebug bool `yaml:"render_debug" json:"render_debug"`
|
|
|
|
|
Artifacts map[string]ScriptoriumArtifactConfig `yaml:"artifacts" json:"artifacts"`
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-07 03:54:41 +00:00
|
|
|
type projectedPipelineValue struct {
|
|
|
|
|
record EffectivePipelineValueRecord
|
|
|
|
|
segments []string
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 22:09:58 +00:00
|
|
|
func projectEffectivePipeline(cfg *PipelineConfig) effectivePipelineProjection {
|
2026-09-07 03:35:46 +00:00
|
|
|
projection := effectivePipelineProjection{
|
2026-11-17 22:31:03 +00:00
|
|
|
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: effectiveStorageConfig{Backend: cfg.Storage.Backend},
|
2026-09-07 04:13:58 +00:00
|
|
|
Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish,
|
2026-11-17 22:31:03 +00:00
|
|
|
WhisperX: cfg.WhisperX, Seriatim: cfg.Seriatim,
|
2026-09-07 03:35:46 +00:00
|
|
|
Normalize: cfg.Normalize, Trim: cfg.Trim, Render: cfg.Render,
|
2026-11-17 22:31:03 +00:00
|
|
|
Notification: cfg.Notification,
|
|
|
|
|
}
|
|
|
|
|
if s3 := cfg.Storage.S3; s3 != nil {
|
|
|
|
|
projection.Storage.S3 = &effectiveStorageS3Config{Bucket: s3.Bucket, RootPrefix: s3.RootPrefix, Region: s3.Region, Endpoint: s3.Endpoint, ForcePathStyle: s3.ForcePathStyle, AccessKeyID: secretReferenceName(s3.AccessKeyID), SecretAccessKey: secretReferenceName(s3.SecretAccessKey)}
|
|
|
|
|
}
|
|
|
|
|
audita := cfg.Audita
|
|
|
|
|
projection.Audita = effectiveAuditaConfig{Binary: audita.Binary, Timeout: audita.Timeout, Credentials: secretReferenceNames(audita.Credentials), Modules: audita.Modules, Overlay: audita.Overlay, ProposalProfile: audita.ProposalProfile, ValidationProfile: audita.ValidationProfile, TranscriptDescription: audita.TranscriptDescription, ConfigPath: audita.ConfigPath, OutputSchema: audita.OutputSchema}
|
|
|
|
|
if n := cfg.Notarius; n != nil {
|
|
|
|
|
projection.Notarius = &effectiveNotariusConfig{Enabled: n.Enabled, Binary: n.Binary, Credentials: secretReferenceNames(n.Credentials), ConfigOverlay: n.ConfigOverlay, Resume: n.Resume, ChunkCache: n.ChunkCache, ConfigPath: n.ConfigPath, PipelineID: n.PipelineID, Timeout: n.Timeout, WorkingDirectory: n.WorkingDirectory, References: n.References, Outputs: n.Outputs}
|
2026-09-07 03:35:46 +00:00
|
|
|
}
|
|
|
|
|
if cfg.Scriptorium != nil {
|
|
|
|
|
projection.Scriptorium = &effectiveScriptoriumConfig{
|
2026-09-29 17:50:30 +00:00
|
|
|
Credentials: secretReferenceNames(cfg.Scriptorium.Credentials),
|
|
|
|
|
Binary: cfg.Scriptorium.Binary, ConfigPath: cfg.Scriptorium.ConfigPath,
|
2026-09-07 03:35:46 +00:00
|
|
|
Timeout: cfg.Scriptorium.Timeout, RenderDebug: cfg.Scriptorium.RenderDebug,
|
|
|
|
|
Artifacts: cfg.Scriptorium.Artifacts,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return projection
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func projectionJSON(projection effectivePipelineProjection) ([]byte, error) {
|
|
|
|
|
data, err := json.Marshal(projection)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("encode effective pipeline projection: %w", err)
|
|
|
|
|
}
|
|
|
|
|
return data, nil
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-07 12:03:45 +00:00
|
|
|
func projectionValueTree(value any) (any, error) {
|
|
|
|
|
data, err := json.Marshal(value)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return nil, fmt.Errorf("encode typed pipeline projection: %w", err)
|
|
|
|
|
}
|
|
|
|
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
|
|
|
decoder.UseNumber()
|
|
|
|
|
var root any
|
|
|
|
|
if err := decoder.Decode(&root); err != nil {
|
|
|
|
|
return nil, fmt.Errorf("decode effective pipeline projection: %w", err)
|
|
|
|
|
}
|
|
|
|
|
return root, nil
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-07 03:35:46 +00:00
|
|
|
func projectionValueRecords(projection effectivePipelineProjection) ([]EffectivePipelineValueRecord, error) {
|
2026-09-07 03:54:41 +00:00
|
|
|
detailed, err := projectionDetailedRecords(projection)
|
2026-09-07 03:35:46 +00:00
|
|
|
if err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
2026-09-07 03:54:41 +00:00
|
|
|
records := make([]EffectivePipelineValueRecord, 0, len(detailed))
|
|
|
|
|
for _, value := range detailed {
|
|
|
|
|
records = append(records, value.record)
|
|
|
|
|
}
|
|
|
|
|
return records, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func projectionDetailedRecords(value any) ([]projectedPipelineValue, error) {
|
2026-09-07 12:03:45 +00:00
|
|
|
decoded, err := projectionValueTree(value)
|
2026-09-07 03:54:41 +00:00
|
|
|
if err != nil {
|
2026-09-07 12:03:45 +00:00
|
|
|
return nil, err
|
2026-09-07 03:54:41 +00:00
|
|
|
}
|
2026-09-07 12:03:45 +00:00
|
|
|
root, ok := decoded.(map[string]any)
|
|
|
|
|
if !ok {
|
|
|
|
|
return nil, fmt.Errorf("effective pipeline projection must be a mapping")
|
2026-09-07 03:35:46 +00:00
|
|
|
}
|
2026-09-07 03:54:41 +00:00
|
|
|
var records []projectedPipelineValue
|
|
|
|
|
appendProjectionRecords(&records, nil, root)
|
|
|
|
|
sort.Slice(records, func(i, j int) bool { return records[i].record.Path < records[j].record.Path })
|
2026-09-07 03:35:46 +00:00
|
|
|
return records, nil
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-07 03:54:41 +00:00
|
|
|
func appendProjectionRecords(records *[]projectedPipelineValue, segments []string, value any) {
|
2026-09-07 03:35:46 +00:00
|
|
|
if mapping, ok := value.(map[string]any); ok && len(mapping) > 0 {
|
|
|
|
|
keys := make([]string, 0, len(mapping))
|
|
|
|
|
for key := range mapping {
|
|
|
|
|
keys = append(keys, key)
|
|
|
|
|
}
|
|
|
|
|
sort.Strings(keys)
|
|
|
|
|
for _, key := range keys {
|
2026-09-07 03:54:41 +00:00
|
|
|
child := append(append([]string(nil), segments...), key)
|
|
|
|
|
appendProjectionRecords(records, child, mapping[key])
|
2026-09-07 03:35:46 +00:00
|
|
|
}
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
encoded, _ := json.Marshal(value)
|
2026-09-07 03:54:41 +00:00
|
|
|
path := strings.Join(segments, ".")
|
|
|
|
|
*records = append(*records, projectedPipelineValue{
|
|
|
|
|
record: EffectivePipelineValueRecord{Path: path, Value: string(encoded)},
|
|
|
|
|
segments: append([]string(nil), segments...),
|
|
|
|
|
})
|
2026-09-07 03:35:46 +00:00
|
|
|
}
|