package config import ( "bytes" "errors" "fmt" "os" "path/filepath" "reflect" "strings" configkit "go.maximumdirect.net/configkit" ) type pipelineResolutionMetadata struct { rootPath string imports []string sources []string selectedOverlay *pipelineOverlaySelection effectiveDigest string artifactFamilies map[string]ScriptoriumArtifactFamilyConfig artifactFamiliesExpanded bool configkitMetadata configkit.Metadata configkitSources []configkit.SourceInfo inspection *configkit.Result[effectivePipelineProjection] authoring *configkit.Result[PipelineConfig] } type pipelineOverlaySelection struct { name string source string overlayPath string } func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) { overlay := selectedPipelineOverlayName(opts) source := configkit.File(strings.TrimSpace(path)) if strings.TrimSpace(path) == "" { search := configkit.Searches(configkit.WorkingDirectory(), configkit.UserConfigDirectory(), configkit.UnixSystemDirectories()) if opts.Search != nil { search = *opts.Search } source = configkit.SearchConfig("narratio", search, configkit.SearchOptions{Required: true}) } result, err := loadPipelineConfigKitResult(source, overlay) if err != nil { return nil, describePipelineLoadError(err) } cfg := result.Config cfg.resolution = pipelineMetadataFromConfigKit(result, opts) return &cfg, nil } func loadPipelineConfigKitResult(source configkit.Source, overlay string) (*configkit.Result[PipelineConfig], error) { invocationDir, err := os.Getwd() if err != nil { return nil, fmt.Errorf("capture invocation directory: %w", err) } return configkit.Load(source, configkit.Options[PipelineConfig]{ Imports: true, Overlay: overlay, Definition: configkit.Definition[PipelineConfig]{ Defaults: pipelineConfigDefaults, Resolve: func(cfg *PipelineConfig, resolution *configkit.Resolution) error { return resolvePipelineConfig(cfg, resolution, invocationDir) }, Validate: validatePipelineConfigKit, }, }) } func describePipelineLoadError(err error) error { return describeConfigKitLoadError("pipeline", err) } func describeConfigKitLoadError(kind string, err error) error { var loadError *configkit.LoadError if !errors.As(err, &loadError) { return fmt.Errorf("load %s config: %w", kind, err) } var diagnostics bytes.Buffer resolved := loadError.Diagnostics() sourcePaths := make(map[string]string, len(loadError.Sources())) for _, source := range loadError.Sources() { if source.Path != "" { sourcePaths[source.ID] = source.Path } else if source.Label != "" { sourcePaths[source.ID] = source.Label } } for index := range resolved { if resolved[index].Primary != nil { if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" { resolved[index].Primary.SourceID = name } } for relatedIndex := range resolved[index].Related { if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" { resolved[index].Related[relatedIndex].SourceID = name } } } if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil { return fmt.Errorf("load %s config: %w", kind, err) } return fmt.Errorf("load %s config: strict decode failed or validation failed: %w: %s", kind, err, strings.TrimSpace(diagnostics.String())) } func selectedPipelineOverlayName(opts PipelineLoadOptions) string { if opts.Overlay == nil || *opts.Overlay == "" || *opts.Overlay == "default" { return "" } return *opts.Overlay } func pipelineConfigDefaults() PipelineConfig { return PipelineConfig{ Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot}, Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot}, Storage: StorageConfig{Backend: StorageBackendLocal}, Spool: SpoolConfig{Root: DefaultSpoolRoot}, Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)}, Publish: &PublishConfig{ Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun), Outputs: defaultPublishOutputs(), }, WhisperX: WhisperXConfig{ Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout, RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency), Retries: intPtr(DefaultWhisperXRetries), }, Seriatim: SeriatimConfig{ Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout, OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap), Report: boolPtr(DefaultSeriatimReport), }, Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout}, Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)}, Trim: &TrimConfig{ Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath, Bounds: TrimBoundsConfig{ PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName, OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout, }, Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)}, }, Render: &RenderConfig{ Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle, IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs), }, Notification: NotificationConfig{Mode: DefaultNotificationMode}, } } func defaultPublishOutputs() []PublishOutputRule { outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...) for index := range outputs { outputs[index].Required = boolPtr(true) } return outputs } func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution, invocationDir string) error { resolveHostResource(resolution, configkit.Keys("workspace", "root"), &cfg.Workspace.Root, invocationDir) resolveHostResource(resolution, configkit.Keys("campaigns", "root"), &cfg.Campaigns.Root, invocationDir) resolveHostResource(resolution, configkit.Keys("spool", "root"), &cfg.Spool.Root, invocationDir) resolveHostResource(resolution, configkit.Keys("cache", "root"), &cfg.Cache.Root, invocationDir) resolveHostResource(resolution, configkit.Keys("audita", "config_path"), &cfg.Audita.ConfigPath, invocationDir) if cfg.Scriptorium != nil { resolveHostResource(resolution, configkit.Keys("scriptorium", "config_path"), &cfg.Scriptorium.ConfigPath, invocationDir) } if cfg.Notarius != nil { resolveHostResource(resolution, configkit.Keys("notarius", "config_path"), &cfg.Notarius.ConfigPath, invocationDir) resolveHostResource(resolution, configkit.Keys("notarius", "working_directory"), &cfg.Notarius.WorkingDirectory, invocationDir) } backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend)) if backend != cfg.Storage.Backend { configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend") } if cfg.Storage.S3 != nil { path := configkit.Keys("storage", "s3") configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix") } if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil { value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout} configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium") } if cfg.Scriptorium != nil { path := configkit.Keys("scriptorium") configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary") configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout") } if cfg.Notarius != nil { path := configkit.Keys("notarius") configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary") configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout") if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" { configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory, filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path")) } } if cfg.Publish != nil { for index := range cfg.Publish.Outputs { output := cfg.Publish.Outputs[index] path := configkit.Keys("publish", "outputs").Index(index).Key("required") configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default") cfg.Publish.Outputs[index] = output } } before := cloneNotariusForResolution(cfg.Notarius) if err := NormalizePipelineConfig(cfg); err != nil { resolution.Error(configkit.Keys("notarius"), err.Error()) return nil } if !reflect.DeepEqual(before, cfg.Notarius) { resolution.RecordNormalization(configkit.Keys("notarius"), "canonical Notarius configuration") } if cfg.Publish != nil { destinations := deriveMissingPublishDestinations(cfg.Publish, cfg.Scriptorium, cfg.Notarius) for _, derived := range destinations { output := cfg.Publish.Outputs[derived.index] configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(derived.index).Key("dest"), &output.Dest, derived.dest, "derived from publish source") cfg.Publish.Outputs[derived.index] = output } } return nil } func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig { if value == nil { return nil } clone := *value clone.References = cloneStringMap(value.References) if value.Outputs != nil { clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs)) for key, output := range value.Outputs { clone.Outputs[key] = output } } return &clone } func cloneStringMap(value map[string]string) map[string]string { if value == nil { return nil } clone := make(map[string]string, len(value)) for key, item := range value { clone[key] = item } return clone } func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) { // Checks that depend on family expansion or campaign/session context remain // at Narratio's explicit resolution boundary. This early definition check is // deliberately limited to an invariant that is wholly file-local. if strings.TrimSpace(cfg.Workspace.Root) == "" { validation.Error(configkit.Keys("workspace", "root"), "workspace root is required") } } func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) *pipelineResolutionMetadata { sources := result.Sources() resolved := &pipelineResolutionMetadata{ configkitMetadata: result.Metadata(), configkitSources: sources, authoring: result, } for index, source := range sources { path := source.Path if path == "" { path = source.Label } resolved.sources = append(resolved.sources, path) if index == 0 { resolved.rootPath = path } else if source.Layer == configkit.LayerMain { resolved.imports = append(resolved.imports, path) } if source.Layer == configkit.LayerOverlay { resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path} } } return resolved } func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) { effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg)) if err != nil { return nil, err } if cfg.Scriptorium == nil { return effective, nil } families := cfg.Scriptorium.ArtifactFamilies if cfg.resolution != nil && len(cfg.resolution.artifactFamilies) != 0 { families = cfg.resolution.artifactFamilies } if len(families) == 0 { return effective, nil } detailedFamilies, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": families}}) if err != nil { return nil, err } return append(effective, detailedFamilies...), nil } func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string { if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) { return []string{pipelineDefaultOwnershipSource} } if origin.Layer == configkit.LayerResolution { return []string{"resolution"} } seen := map[string]struct{}{} var sources []string add := func(location configkit.Location) { value := pathsByID[location.SourceID] if value == "" { value = location.SourceID } if _, ok := seen[value]; !ok { seen[value] = struct{}{} sources = append(sources, value) } } for _, location := range origin.Locations { add(location) } for _, inheritance := range origin.Inheritance { for _, location := range inheritance.Locations { add(location) } } if len(sources) == 0 { return []string{pipelineDefaultOwnershipSource} } return sources } func pipelineSourcePaths(cfg *PipelineConfig) map[string]string { paths := make(map[string]string) if cfg == nil || cfg.resolution == nil { return paths } for _, source := range cfg.resolution.configkitSources { if cfg.resolution.authoring != nil { if found, ok := cfg.resolution.authoring.Source(source.ID); ok { source = found } } path := source.Path if path == "" { path = source.Label } paths[source.ID] = path } return paths } func intPtr(value int) *int { return &value } func float64Ptr(value float64) *float64 { return &value } func boolPtr(value bool) *bool { return &value }