Files
narratio/internal/config/pipeline_configkit.go

371 lines
14 KiB
Go

package config
import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
configkit "go.maximumdirect.net/configkit"
)
type pipelineResolutionMetadata struct {
rootPath string
imports []string
sources []string
selectedOverlay *pipelineOverlaySelection
effectiveDigest string
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
configkitMetadata configkit.Metadata
configkitSources []configkit.SourceInfo
inspection *configkit.Result[effectivePipelineProjection]
}
type pipelineOverlaySelection struct {
name string
source string
overlayPath string
}
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
overlay := selectedPipelineOverlayName(opts)
source := configkit.File(strings.TrimSpace(path))
if strings.TrimSpace(path) == "" {
search := configkit.Searches(configkit.WorkingDirectory(), configkit.UserConfigDirectory(), configkit.UnixSystemDirectories())
if opts.Search != nil {
search = *opts.Search
}
source = configkit.SearchConfig("narratio", search, configkit.SearchOptions{Required: true})
}
result, err := loadPipelineConfigKitResult(source, overlay)
if err != nil {
return nil, describePipelineLoadError(err)
}
cfg := result.Config
cfg.resolution = pipelineMetadataFromConfigKit(result, opts)
return &cfg, nil
}
func loadPipelineConfigKitResult(source configkit.Source, overlay string) (*configkit.Result[PipelineConfig], error) {
invocationDir, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("capture invocation directory: %w", err)
}
return configkit.Load(source, configkit.Options[PipelineConfig]{
Imports: true,
Overlay: overlay,
Definition: configkit.Definition[PipelineConfig]{
Defaults: pipelineConfigDefaults,
Resolve: func(cfg *PipelineConfig, resolution *configkit.Resolution) error {
return resolvePipelineConfig(cfg, resolution, invocationDir)
},
Validate: validatePipelineConfigKit,
},
})
}
func describePipelineLoadError(err error) error {
return describeConfigKitLoadError("pipeline", err)
}
func describeConfigKitLoadError(kind string, err error) error {
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
return fmt.Errorf("load %s config: %w", kind, err)
}
var diagnostics bytes.Buffer
resolved := loadError.Diagnostics()
sourcePaths := make(map[string]string, len(loadError.Sources()))
for _, source := range loadError.Sources() {
if source.Path != "" {
sourcePaths[source.ID] = source.Path
} else if source.Label != "" {
sourcePaths[source.ID] = source.Label
}
}
for index := range resolved {
if resolved[index].Primary != nil {
if name := sourcePaths[resolved[index].Primary.SourceID]; name != "" {
resolved[index].Primary.SourceID = name
}
}
for relatedIndex := range resolved[index].Related {
if name := sourcePaths[resolved[index].Related[relatedIndex].SourceID]; name != "" {
resolved[index].Related[relatedIndex].SourceID = name
}
}
}
if writeErr := configkit.WriteDiagnostics(&diagnostics, resolved); writeErr != nil {
return fmt.Errorf("load %s config: %w", kind, err)
}
return fmt.Errorf("load %s config: strict decode failed or validation failed: %w: %s", kind, err, strings.TrimSpace(diagnostics.String()))
}
func selectedPipelineOverlayName(opts PipelineLoadOptions) string {
if opts.Overlay == nil || *opts.Overlay == "" || *opts.Overlay == "default" {
return ""
}
return *opts.Overlay
}
func pipelineConfigDefaults() PipelineConfig {
return PipelineConfig{
Workspace: WorkspaceConfig{Root: DefaultWorkspaceRoot},
Campaigns: CampaignsConfig{Root: DefaultCampaignsRoot},
Storage: StorageConfig{Backend: StorageBackendLocal},
Spool: SpoolConfig{Root: DefaultSpoolRoot},
Cache: CacheConfig{Root: DefaultCacheRoot, S3Audio: boolPtr(DefaultCacheS3Audio)},
Publish: &PublishConfig{
Enabled: boolPtr(DefaultArchiveEnabled), UploadRun: boolPtr(DefaultArchiveUploadRun),
Outputs: defaultPublishOutputs(),
},
WhisperX: WhisperXConfig{
Language: DefaultWhisperXLanguage, Timeout: DefaultWhisperXTimeout,
RetryDelay: DefaultWhisperXRetryDelay, Concurrency: intPtr(DefaultWhisperXConcurrency),
Retries: intPtr(DefaultWhisperXRetries),
},
Seriatim: SeriatimConfig{
Binary: DefaultSeriatimBinary, Timeout: DefaultSeriatimTimeout,
OutputSchema: DefaultSeriatimOutputSchema, CoalesceGap: float64Ptr(DefaultSeriatimCoalesceGap),
Report: boolPtr(DefaultSeriatimReport),
},
Audita: AuditaConfig{Binary: DefaultAuditaBinary, Timeout: DefaultAuditaTimeout},
Normalize: &NormalizeConfig{OutputPath: DefaultNormalizeOutputPath, OutputSchema: DefaultNormalizeOutputSchema, Report: boolPtr(DefaultNormalizeReport)},
Trim: &TrimConfig{
Enabled: boolPtr(DefaultTrimEnabled), OutputPath: DefaultTrimOutputPath,
Bounds: TrimBoundsConfig{
PromptID: DefaultTrimBoundsPromptID, TranscriptInputName: DefaultTrimBoundsTranscriptInputName,
OutputPath: DefaultTrimBoundsOutputPath, Timeout: DefaultTrimBoundsTimeout,
},
Seriatim: TrimSeriatimConfig{Report: boolPtr(DefaultTrimSeriatimReport)},
},
Render: &RenderConfig{
Enabled: boolPtr(DefaultRenderEnabled), Format: DefaultRenderFormat, Title: DefaultRenderTitle,
IncludeTimestamps: boolPtr(DefaultRenderTimestamps), IncludeSegmentIDs: boolPtr(DefaultRenderSegmentIDs),
},
Notification: NotificationConfig{Mode: DefaultNotificationMode},
}
}
func defaultPublishOutputs() []PublishOutputRule {
outputs := append([]PublishOutputRule(nil), DefaultPublishOutputs...)
for index := range outputs {
outputs[index].Required = boolPtr(true)
}
return outputs
}
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution, invocationDir string) error {
resolveHostResource(resolution, configkit.Keys("workspace", "root"), &cfg.Workspace.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("campaigns", "root"), &cfg.Campaigns.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("spool", "root"), &cfg.Spool.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("cache", "root"), &cfg.Cache.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("audita", "config_path"), &cfg.Audita.ConfigPath, invocationDir)
if cfg.Scriptorium != nil {
resolveHostResource(resolution, configkit.Keys("scriptorium", "config_path"), &cfg.Scriptorium.ConfigPath, invocationDir)
}
if cfg.Notarius != nil {
resolveHostResource(resolution, configkit.Keys("notarius", "config_path"), &cfg.Notarius.ConfigPath, invocationDir)
resolveHostResource(resolution, configkit.Keys("notarius", "working_directory"), &cfg.Notarius.WorkingDirectory, invocationDir)
}
backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend))
if backend != cfg.Storage.Backend {
configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend")
}
if cfg.Storage.S3 != nil {
path := configkit.Keys("storage", "s3")
configkit.Default(resolution, path.Key("root_prefix"), &cfg.Storage.S3.RootPrefix, DefaultStorageS3RootPrefix, "default S3 root prefix")
}
if cfg.Trim != nil && cfg.Trim.Enabled != nil && *cfg.Trim.Enabled && cfg.Scriptorium == nil {
value := &ScriptoriumConfig{Binary: DefaultScriptoriumBinary, Timeout: DefaultScriptoriumTimeout}
configkit.Default(resolution, configkit.Keys("scriptorium"), &cfg.Scriptorium, value, "trim requires Scriptorium")
}
if cfg.Scriptorium != nil {
path := configkit.Keys("scriptorium")
configkit.Default(resolution, path.Key("binary"), &cfg.Scriptorium.Binary, DefaultScriptoriumBinary, "default Scriptorium binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Scriptorium.Timeout, DefaultScriptoriumTimeout, "default Scriptorium timeout")
}
if cfg.Notarius != nil {
path := configkit.Keys("notarius")
configkit.Default(resolution, path.Key("binary"), &cfg.Notarius.Binary, DefaultNotariusBinary, "default Notarius binary")
configkit.Default(resolution, path.Key("timeout"), &cfg.Notarius.Timeout, DefaultNotariusTimeout, "default Notarius timeout")
if cfg.Notarius.Enabled && cfg.Notarius.ConfigPath != "" {
configkit.Default(resolution, path.Key("working_directory"), &cfg.Notarius.WorkingDirectory,
filepath.Dir(cfg.Notarius.ConfigPath), "derived from Notarius config path", path.Key("config_path"))
}
}
if cfg.Publish != nil {
defaultedOutput := false
for index := range cfg.Publish.Outputs {
output := cfg.Publish.Outputs[index]
path := configkit.Keys("publish", "outputs").Index(index).Key("required")
if !resolution.IsExplicit(path) {
defaultedOutput = true
}
configkit.Default(resolution, path, &output.Required, boolPtr(true), "publish outputs are required by default")
cfg.Publish.Outputs[index] = output
}
if defaultedOutput {
resolution.RecordDerivedDefault(configkit.Keys("publish", "outputs"), "publish output entry defaults")
}
}
before := cloneNotariusForResolution(cfg.Notarius)
if err := NormalizePipelineConfig(cfg); err != nil {
resolution.Error(configkit.Keys("notarius"), err.Error())
return nil
}
if !reflect.DeepEqual(before, cfg.Notarius) {
resolution.RecordNormalization(configkit.Keys("notarius"), "canonical Notarius configuration")
}
if cfg.Publish != nil {
destinations := deriveMissingPublishDestinations(cfg.Publish, cfg.Scriptorium, cfg.Notarius)
for _, derived := range destinations {
output := cfg.Publish.Outputs[derived.index]
configkit.Normalize(resolution, configkit.Keys("publish", "outputs").Index(derived.index).Key("dest"), &output.Dest, derived.dest, "derived from publish source")
cfg.Publish.Outputs[derived.index] = output
}
if len(destinations) > 0 {
// ConfigKit compares sequences atomically; child records alone do not
// cover the list mutation when requiredness was already explicit.
resolution.RecordNormalization(configkit.Keys("publish", "outputs"), "derived publish destinations")
}
}
return nil
}
func cloneNotariusForResolution(value *NotariusConfig) *NotariusConfig {
if value == nil {
return nil
}
clone := *value
clone.References = cloneStringMap(value.References)
if value.Outputs != nil {
clone.Outputs = make(map[string]NotariusOutputConfig, len(value.Outputs))
for key, output := range value.Outputs {
clone.Outputs[key] = output
}
}
return &clone
}
func cloneStringMap(value map[string]string) map[string]string {
if value == nil {
return nil
}
clone := make(map[string]string, len(value))
for key, item := range value {
clone[key] = item
}
return clone
}
func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Validation) {
// Checks that depend on family expansion or campaign/session context remain
// at Narratio's explicit resolution boundary. This early definition check is
// deliberately limited to an invariant that is wholly file-local.
if strings.TrimSpace(cfg.Workspace.Root) == "" {
validation.Error(configkit.Keys("workspace", "root"), "workspace root is required")
}
}
func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) *pipelineResolutionMetadata {
sources := result.Sources()
resolved := &pipelineResolutionMetadata{
configkitMetadata: result.Metadata(), configkitSources: sources,
}
for index, source := range sources {
path := source.Path
if path == "" {
path = source.Label
}
resolved.sources = append(resolved.sources, path)
if index == 0 {
resolved.rootPath = path
} else if source.Layer == configkit.LayerMain {
resolved.imports = append(resolved.imports, path)
}
if source.Layer == configkit.LayerOverlay {
resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path}
}
}
return resolved
}
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg))
if err != nil {
return nil, err
}
if cfg.Scriptorium == nil {
return effective, nil
}
families := cfg.Scriptorium.ArtifactFamilies
if cfg.resolution != nil && len(cfg.resolution.artifactFamilies) != 0 {
families = cfg.resolution.artifactFamilies
}
if len(families) == 0 {
return effective, nil
}
detailedFamilies, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": families}})
if err != nil {
return nil, err
}
return append(effective, detailedFamilies...), nil
}
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
if origin.Layer == configkit.LayerDefault || (origin.Layer == configkit.LayerResolution && origin.Operation == configkit.ChangeDerivedDefault) {
return []string{pipelineDefaultOwnershipSource}
}
if origin.Layer == configkit.LayerResolution {
return []string{"resolution"}
}
seen := map[string]struct{}{}
var sources []string
add := func(location configkit.Location) {
value := pathsByID[location.SourceID]
if value == "" {
value = location.SourceID
}
if _, ok := seen[value]; !ok {
seen[value] = struct{}{}
sources = append(sources, value)
}
}
for _, location := range origin.Locations {
add(location)
}
for _, inheritance := range origin.Inheritance {
for _, location := range inheritance.Locations {
add(location)
}
}
if len(sources) == 0 {
return []string{pipelineDefaultOwnershipSource}
}
return sources
}
func pipelineSourcePaths(cfg *PipelineConfig) map[string]string {
paths := make(map[string]string)
if cfg == nil || cfg.resolution == nil {
return paths
}
for _, source := range cfg.resolution.configkitSources {
path := source.Path
if path == "" {
path = source.Label
}
paths[source.ID] = path
}
return paths
}
func intPtr(value int) *int { return &value }
func float64Ptr(value float64) *float64 { return &value }
func boolPtr(value bool) *bool { return &value }