346 lines
12 KiB
Go
346 lines
12 KiB
Go
package config
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
|
|
configkit "go.maximumdirect.net/configkit"
|
|
)
|
|
|
|
type pipelineResolutionMetadata struct {
|
|
rootPath string
|
|
imports []string
|
|
sources []string
|
|
selectedOverlay *pipelineOverlaySelection
|
|
effectiveDigest string
|
|
logicalNotariusConfig string
|
|
logicalNotariusWorking string
|
|
logicalNotariusCaptured bool
|
|
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
|
artifactFamiliesExpanded bool
|
|
configkitMetadata configkit.Metadata
|
|
configkitSources []configkit.SourceInfo
|
|
}
|
|
|
|
type pipelineOverlaySelection struct {
|
|
name string
|
|
source string
|
|
overlayPath string
|
|
}
|
|
|
|
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
|
|
overlay := selectedPipelineOverlayName(opts)
|
|
result, err := loadPipelineConfigKitResult(path, overlay)
|
|
if err != nil {
|
|
return nil, describePipelineLoadError(err)
|
|
}
|
|
cfg := result.Config
|
|
cfg.resolution = pipelineMetadataFromConfigKit(result, opts)
|
|
return &cfg, nil
|
|
}
|
|
|
|
func loadPipelineConfigKitResult(path, overlay string) (*configkit.Result[PipelineConfig], error) {
|
|
return configkit.Load(configkit.File(path), configkit.Options[PipelineConfig]{
|
|
Imports: true,
|
|
Overlay: overlay,
|
|
Definition: configkit.Definition[PipelineConfig]{
|
|
Defaults: pipelineConfigDefaults,
|
|
Resolve: resolvePipelineConfig,
|
|
Validate: validatePipelineConfigKit,
|
|
},
|
|
})
|
|
}
|
|
|
|
func describePipelineLoadError(err error) error {
|
|
return describeConfigKitLoadError("pipeline", err)
|
|
}
|
|
|
|
func describeConfigKitLoadError(kind string, err error) error {
|
|
var loadError *configkit.LoadError
|
|
if !errors.As(err, &loadError) {
|
|
return fmt.Errorf("load %s config: %w", kind, err)
|
|
}
|
|
var diagnostics bytes.Buffer
|
|
resolved := loadError.Diagnostics()
|
|
sourcePaths := make(map[string]string, len(loadError.Sources()))
|
|
for _, source := range loadError.Sources() {
|
|
if source.Path != "" {
|
|
sourcePaths[source.ID] = source.Path
|
|
} else if source.Label != "" {
|
|
sourcePaths[source.ID] = source.Label
|
|
}
|
|
}
|
|
for index := range resolved {
|
|
if resolved[index].Primary != nil {
|
|
if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" {
|
|
resolved[index].Primary.SourceID = name
|
|
}
|
|
}
|
|
for relatedIndex := range resolved[index].Related {
|
|
if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" {
|
|
resolved[index].Related[relatedIndex].SourceID = name
|
|
}
|
|
}
|
|
}
|
|
if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil {
|
|
return fmt.Errorf("load %s config: %w", kind, err)
|
|
}
|
|
return fmt.Errorf("load %s config: strict decode failed or validation failed: %w: %s", kind, err, strings.TrimSpace(diagnostics.String()))
|
|
}
|
|
|
|
func selectedPipelineOverlayName(opts PipelineLoadOptions) string {
|
|
if opts.Overlay == nil || *opts.Overlay == "" || *opts.Overlay == "default" {
|
|
return ""
|
|
}
|
|
return *opts.Overlay
|
|
}
|
|
|
|
func pipelineConfigDefaults() PipelineConfig {
|
|
return PipelineConfig{
|
|
Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot},
|
|
Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot},
|
|
Storage: StorageConfig{Backend: StorageBackendLocal},
|
|
Spool: SpoolConfig{Root: DefaultSpoolRoot},
|
|
Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)},
|
|
Publish: &PublishConfig{
|
|
Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun),
|
|
Outputs: defaultPublishOutputs(),
|
|
},
|
|
WhisperX: WhisperXConfig{
|
|
Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout,
|
|
RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency),
|
|
Retries: intPtr(DefaultWhisperXRetries),
|
|
},
|
|
Seriatim: SeriatimConfig{
|
|
Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout,
|
|
OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap),
|
|
Report: boolPtr(DefaultSeriatimReport),
|
|
},
|
|
Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout},
|
|
Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)},
|
|
Trim: &TrimConfig{
|
|
Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath,
|
|
Bounds: TrimBoundsConfig{
|
|
PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName,
|
|
OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout,
|
|
},
|
|
Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)},
|
|
},
|
|
Render: &RenderConfig{
|
|
Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle,
|
|
IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs),
|
|
},
|
|
Notification: NotificationConfig{Mode: DefaultNotificationMode},
|
|
}
|
|
}
|
|
|
|
func defaultPublishOutputs() []PublishOutputRule {
|
|
outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...)
|
|
for index := range outputs {
|
|
outputs[index].Required = boolPtr(true)
|
|
}
|
|
return outputs
|
|
}
|
|
|
|
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error {
|
|
backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend))
|
|
if backend != cfg.Storage.Backend {
|
|
configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend")
|
|
}
|
|
if cfg.Storage.S3 != nil {
|
|
path := configkit.Keys("storage", "s3")
|
|
configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix")
|
|
}
|
|
if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil {
|
|
value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout}
|
|
configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium")
|
|
}
|
|
if cfg.Scriptorium != nil {
|
|
path := configkit.Keys("scriptorium")
|
|
configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary")
|
|
configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout")
|
|
}
|
|
if cfg.Notarius != nil {
|
|
path := configkit.Keys("notarius")
|
|
configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary")
|
|
configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout")
|
|
if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" {
|
|
configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory,
|
|
filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path"))
|
|
}
|
|
}
|
|
if cfg.Publish != nil {
|
|
defaultedOutput := false
|
|
for index := range cfg.Publish.Outputs {
|
|
output := cfg.Publish.Outputs[index]
|
|
path := configkit.Keys("publish", "outputs").Index(index).Key("required")
|
|
if !resolution.IsExplicit(path) {
|
|
defaultedOutput = true
|
|
}
|
|
configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default")
|
|
cfg.Publish.Outputs[index] = output
|
|
}
|
|
if defaultedOutput {
|
|
resolution.RecordDerivedDefault(configkit.Keys("publish", "outputs"), "publish output entry defaults")
|
|
}
|
|
}
|
|
before := cloneNotariusForResolution(cfg.Notarius)
|
|
if err := NormalizePipelineConfig(cfg); err != nil {
|
|
resolution.Error(configkit.Keys("notarius"), err.Error())
|
|
return nil
|
|
}
|
|
if !reflect.DeepEqual(before, cfg.Notarius) {
|
|
resolution.RecordNormalization(configkit.Keys("notarius"), "canonical Notarius configuration")
|
|
}
|
|
if cfg.Publish != nil {
|
|
destinations := deriveMissingPublishDestinations(cfg.Publish, cfg.Scriptorium, cfg.Notarius)
|
|
for _, derived := range destinations {
|
|
output := cfg.Publish.Outputs[derived.index]
|
|
configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(derived.index).Key("dest"), &output.Dest, derived.dest, "derived from publish source")
|
|
cfg.Publish.Outputs[derived.index] = output
|
|
}
|
|
if len(destinations) > 0 {
|
|
// ConfigKit compares sequences atomically; child records alone do not
|
|
// cover the list mutation when requiredness was already explicit.
|
|
resolution.RecordNormalization(configkit.Keys("publish", "outputs"), "derived publish destinations")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
clone := *value
|
|
clone.References = cloneStringMap(value.References)
|
|
if value.Outputs != nil {
|
|
clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs))
|
|
for key, output := range value.Outputs {
|
|
clone.Outputs[key] = output
|
|
}
|
|
}
|
|
return &clone
|
|
}
|
|
|
|
func cloneStringMap(value map[string]string) map[string]string {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
clone := make(map[string]string, len(value))
|
|
for key, item := range value {
|
|
clone[key] = item
|
|
}
|
|
return clone
|
|
}
|
|
|
|
func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) {
|
|
// Checks that depend on family expansion or campaign/session context remain
|
|
// at Narratio's explicit resolution boundary. This early definition check is
|
|
// deliberately limited to an invariant that is wholly file-local.
|
|
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
|
validation.Error(configkit.Keys("workspace", "root"), "workspace root is required")
|
|
}
|
|
}
|
|
|
|
func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) *pipelineResolutionMetadata {
|
|
sources := result.Sources()
|
|
resolved := &pipelineResolutionMetadata{
|
|
configkitMetadata: result.Metadata(), configkitSources: sources,
|
|
}
|
|
for index, source := range sources {
|
|
path := source.Path
|
|
if path == "" {
|
|
path = source.Label
|
|
}
|
|
resolved.sources = append(resolved.sources, path)
|
|
if index == 0 {
|
|
resolved.rootPath = path
|
|
} else if source.Layer == configkit.LayerMain {
|
|
resolved.imports = append(resolved.imports, path)
|
|
}
|
|
if source.Layer == configkit.LayerOverlay {
|
|
resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path}
|
|
}
|
|
}
|
|
return resolved
|
|
}
|
|
|
|
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
|
|
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if cfg.Scriptorium == nil {
|
|
return effective, nil
|
|
}
|
|
families := cfg.Scriptorium.ArtifactFamilies
|
|
if cfg.resolution != nil && len(cfg.resolution.artifactFamilies) != 0 {
|
|
families = cfg.resolution.artifactFamilies
|
|
}
|
|
if len(families) == 0 {
|
|
return effective, nil
|
|
}
|
|
detailedFamilies, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": families}})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return append(effective, detailedFamilies...), nil
|
|
}
|
|
|
|
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
|
|
if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) {
|
|
return []string{pipelineDefaultOwnershipSource}
|
|
}
|
|
if origin.Layer == configkit.LayerResolution {
|
|
return []string{"resolution"}
|
|
}
|
|
seen := map[string]struct{}{}
|
|
var sources []string
|
|
add := func(location configkit.Location) {
|
|
value := pathsByID[location.SourceID]
|
|
if value == "" {
|
|
value = location.SourceID
|
|
}
|
|
if _, ok := seen[value]; !ok {
|
|
seen[value] = struct{}{}
|
|
sources = append(sources, value)
|
|
}
|
|
}
|
|
for _, location := range origin.Locations {
|
|
add(location)
|
|
}
|
|
for _, inheritance := range origin.Inheritance {
|
|
for _, location := range inheritance.Locations {
|
|
add(location)
|
|
}
|
|
}
|
|
if len(sources) == 0 {
|
|
return []string{pipelineDefaultOwnershipSource}
|
|
}
|
|
return sources
|
|
}
|
|
|
|
func pipelineSourcePaths(cfg *PipelineConfig) map[string]string {
|
|
paths := make(map[string]string)
|
|
if cfg == nil || cfg.resolution == nil {
|
|
return paths
|
|
}
|
|
for _, source := range cfg.resolution.configkitSources {
|
|
path := source.Path
|
|
if path == "" {
|
|
path = source.Label
|
|
}
|
|
paths[source.ID] = path
|
|
}
|
|
return paths
|
|
}
|
|
|
|
func intPtr(value int) *int { return &value }
|
|
func float64Ptr(value float64) *float64 { return &value }
|
|
func boolPtr(value bool) *bool { return &value }
|