Resolve pipeline resources from their source locations

This commit is contained in:
2026-11-17 22:09:58 +00:00
parent 6593ccbeaf
commit c3023cb609
10 changed files with 209 additions and 88 deletions

View File

@@ -127,7 +127,7 @@ directories and temporary native search descriptions. Test selected invalid
input, no source, bootstrap override/default behavior, and diff bootstrap once.
Keep tests isolated from the operator's system configuration and secrets.
## Stage 3: Resolve pipeline resources from native origin context
## Stage 3: Resolve pipeline resources from native origin context [x]
Read `resolvePipelineConfig`, `resolveNotariusPaths`,
`finalizeLoadedPipeline`, `effective_digest.go`, and the upstream source-aware

View File

@@ -4,8 +4,6 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"path/filepath"
"strings"
)
const (
@@ -53,7 +51,7 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
if cfg == nil || cfg.resolution == nil {
return fmt.Errorf("pipeline resolution metadata is required")
}
canonical, err := projectionJSON(projectEffectivePipeline(cfg, true))
canonical, err := projectionJSON(projectEffectivePipeline(cfg))
if err != nil {
return err
}
@@ -62,25 +60,3 @@ func recomputePipelineEffectiveDigest(cfg *PipelineConfig) error {
cfg.resolution.effectiveDigest = hex.EncodeToString(digest[:])
return nil
}
// captureLogicalNotariusPaths retains normalized user-facing path semantics
// before runtime resolution makes relative paths depend on the checkout or
// installation directory. Runtime paths remain absolute; provenance does not.
func captureLogicalNotariusPaths(cfg *PipelineConfig) {
if cfg == nil || cfg.resolution == nil || cfg.Notarius == nil {
return
}
configPath := normalizeLogicalFilesystemPath(cfg.Notarius.ConfigPath)
workingDirectory := normalizeLogicalFilesystemPath(cfg.Notarius.WorkingDirectory)
cfg.resolution.logicalNotariusConfig = configPath
cfg.resolution.logicalNotariusWorking = workingDirectory
cfg.resolution.logicalNotariusCaptured = true
}
func normalizeLogicalFilesystemPath(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
return filepath.ToSlash(filepath.Clean(value))
}

View File

@@ -63,7 +63,7 @@ func EffectivePipelineValues(cfg *PipelineConfig) ([]EffectivePipelineValueRecor
if cfg == nil {
return nil, fmt.Errorf("pipeline config is required")
}
return projectionValueRecords(projectEffectivePipeline(cfg, true))
return projectionValueRecords(projectEffectivePipeline(cfg))
}
// EffectivePipelineSources projects pipeline ownership after defaults and
@@ -157,7 +157,7 @@ func MarshalEffectivePipeline(cfg *PipelineConfig) ([]byte, error) {
return nil, fmt.Errorf("pipeline config is required")
}
value, err := projectionValueTree(projectEffectivePipeline(cfg, true))
value, err := projectionValueTree(projectEffectivePipeline(cfg))
if err != nil {
return nil, err
}

View File

@@ -43,7 +43,7 @@ type projectedPipelineValue struct {
segments []string
}
func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effectivePipelineProjection {
func projectEffectivePipeline(cfg *PipelineConfig) effectivePipelineProjection {
projection := effectivePipelineProjection{
Workspace: cfg.Workspace, Campaigns: cfg.Campaigns, Storage: cfg.Storage,
Spool: cfg.Spool, Cache: cfg.Cache, Publish: cfg.Publish,
@@ -59,12 +59,6 @@ func projectEffectivePipeline(cfg *PipelineConfig, logicalNotarius bool) effecti
Artifacts: cfg.Scriptorium.Artifacts,
}
}
if logicalNotarius && cfg.Notarius != nil && cfg.resolution != nil && cfg.resolution.logicalNotariusCaptured {
notarius := *cfg.Notarius
notarius.ConfigPath = cfg.resolution.logicalNotariusConfig
notarius.WorkingDirectory = cfg.resolution.logicalNotariusWorking
projection.Notarius = &notarius
}
return projection
}

View File

@@ -30,7 +30,7 @@ func LoadPipelineWithOptions(path string, opts PipelineLoadOptions) (*PipelineCo
if err != nil {
return nil, err
}
return finalizeLoadedPipeline(EffectivePipelineRootPath(cfg), cfg)
return finalizeLoadedPipeline(cfg)
}
// LoadPipelineOverlayPair resolves two overlays as independent ConfigKit loads.
@@ -51,11 +51,7 @@ func LoadPipelineOverlayPair(path, leftOverlay, rightOverlay string) (*PipelineC
return left, right, nil
}
func finalizeLoadedPipeline(path string, cfg *PipelineConfig) (*PipelineConfig, error) {
captureLogicalNotariusPaths(cfg)
if err := resolveNotariusPaths(cfg, path); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
func finalizeLoadedPipeline(cfg *PipelineConfig) (*PipelineConfig, error) {
finalizeArtifactDependencies(cfg)
if err := finalizePipelineResolution(cfg); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
@@ -427,36 +423,3 @@ func shortName(path, fallback string) string {
}
return base
}
func resolveNotariusPaths(cfg *PipelineConfig, pipelinePath string) error {
if cfg == nil || cfg.Notarius == nil || !cfg.Notarius.Enabled {
return nil
}
pipelineAbs, err := filepath.Abs(pipelinePath)
if err != nil {
return fmt.Errorf("resolve pipeline config path %q: %w", pipelinePath, err)
}
baseDir := filepath.Dir(pipelineAbs)
resolve := func(value string) (string, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return "", nil
}
if !filepath.IsAbs(trimmed) {
trimmed = filepath.Join(baseDir, trimmed)
}
return filepath.Abs(trimmed)
}
resolvedConfigPath, err := resolve(cfg.Notarius.ConfigPath)
if err != nil {
return fmt.Errorf("resolve pipeline.notarius.config_path: %w", err)
}
cfg.Notarius.ConfigPath = resolvedConfigPath
workingDirectory, err := resolve(cfg.Notarius.WorkingDirectory)
if err != nil {
return fmt.Errorf("resolve pipeline.notarius.working_directory: %w", err)
}
cfg.Notarius.WorkingDirectory = workingDirectory
return nil
}

View File

@@ -51,8 +51,8 @@ notarius:
if disabled.Notarius.Resume || disabled.Notarius.ChunkCache != "" {
t.Fatalf("disabled cache-control defaults = %#v", disabled.Notarius)
}
if disabled.Notarius.ConfigPath != "relative/notarius.yml" || disabled.Notarius.WorkingDirectory != "" {
t.Fatalf("disabled paths were resolved unexpectedly: %#v", disabled.Notarius)
if disabled.Notarius.ConfigPath != filepath.Join(dir, "relative/notarius.yml") || disabled.Notarius.WorkingDirectory != "" {
t.Fatalf("disabled resource paths = %#v", disabled.Notarius)
}
}
@@ -294,7 +294,7 @@ func TestNotariusCacheControlsSafeSerialization(t *testing.T) {
}
}
func TestNotariusRelativePathsDoNotMakeEffectiveDigestLocationDependent(t *testing.T) {
func TestNotariusRelativePathsAffectEffectiveDigestWhenResourcesDiffer(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
notarius:
enabled: true
@@ -325,8 +325,8 @@ notarius:
configs[0].Notarius.WorkingDirectory == configs[1].Notarius.WorkingDirectory {
t.Fatalf("runtime Notarius paths should remain location-specific: %#v / %#v", configs[0].Notarius, configs[1].Notarius)
}
if first, second := EffectivePipelineDigest(configs[0]), EffectivePipelineDigest(configs[1]); first == "" || first != second {
t.Fatalf("relocated logical configuration digests = %q / %q, want equal non-empty values", first, second)
if first, second := EffectivePipelineDigest(configs[0]), EffectivePipelineDigest(configs[1]); first == "" || first == second {
t.Fatalf("different resource digests = %q / %q, want distinct non-empty values", first, second)
}
}

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
@@ -17,9 +18,6 @@ type pipelineResolutionMetadata struct {
sources []string
selectedOverlay *pipelineOverlaySelection
effectiveDigest string
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
configkitMetadata configkit.Metadata
@@ -52,12 +50,18 @@ func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*Pipeline
}
func loadPipelineConfigKitResult(source configkit.Source, overlay string) (*configkit.Result[PipelineConfig], error) {
invocationDir, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("capture invocation directory: %w", err)
}
return configkit.Load(source, configkit.Options[PipelineConfig]{
Imports: true,
Overlay: overlay,
Definition: configkit.Definition[PipelineConfig]{
Defaults: pipelineConfigDefaults,
Resolve: resolvePipelineConfig,
Resolve: func(cfg *PipelineConfig, resolution *configkit.Resolution) error {
return resolvePipelineConfig(cfg, resolution, invocationDir)
},
Validate: validatePipelineConfigKit,
},
})
@@ -154,7 +158,19 @@ func defaultPublishOutputs() []PublishOutputRule {
return outputs
}
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution) error {
func resolvePipelineConfig(cfg *PipelineConfig, resolution *configkit.Resolution, invocationDir string) error {
resolveHostResource(resolution, configkit.Keys("workspace", "root"), &cfg.Workspace.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("campaigns", "root"), &cfg.Campaigns.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("spool", "root"), &cfg.Spool.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("cache", "root"), &cfg.Cache.Root, invocationDir)
resolveHostResource(resolution, configkit.Keys("audita", "config_path"), &cfg.Audita.ConfigPath, invocationDir)
if cfg.Scriptorium != nil {
resolveHostResource(resolution, configkit.Keys("scriptorium", "config_path"), &cfg.Scriptorium.ConfigPath, invocationDir)
}
if cfg.Notarius != nil {
resolveHostResource(resolution, configkit.Keys("notarius", "config_path"), &cfg.Notarius.ConfigPath, invocationDir)
resolveHostResource(resolution, configkit.Keys("notarius", "working_directory"), &cfg.Notarius.WorkingDirectory, invocationDir)
}
backend := strings.ToLower(strings.TrimSpace(cfg.Storage.Backend))
if backend != cfg.Storage.Backend {
configkit.Normalize(resolution, configkit.Keys("storage", "backend"), &cfg.Storage.Backend, backend, "canonical storage backend")
@@ -279,7 +295,7 @@ func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opt
}
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg, false))
effective, err := projectionDetailedRecords(projectEffectivePipeline(cfg))
if err != nil {
return nil, err
}

View File

@@ -338,7 +338,7 @@ whisperx:
}
}
func TestInheritedNotariusPathsKeepPipelineRootBase(t *testing.T) {
func TestInheritedNotariusPathsFollowSupplyingSource(t *testing.T) {
dir := t.TempDir()
pipelinePath := writePipelineFixture(t, dir, "pipeline.yml", `config:
imports:
@@ -364,9 +364,9 @@ notarius:
if err != nil {
t.Fatalf("LoadPipeline() error = %v", err)
}
wantConfigPath := filepath.Join(dir, "notarius", "config.yml")
wantConfigPath := filepath.Join(dir, "profiles", "notarius", "config.yml")
if cfg.Notarius.ConfigPath != wantConfigPath {
t.Fatalf("imported Notarius config_path = %q, want pipeline-root path %q", cfg.Notarius.ConfigPath, wantConfigPath)
t.Fatalf("imported Notarius config_path = %q, want source-relative path %q", cfg.Notarius.ConfigPath, wantConfigPath)
}
if got, want := cfg.Notarius.WorkingDirectory, filepath.Dir(wantConfigPath); got != want {
t.Fatalf("derived Notarius working_directory = %q, want pipeline-root directory %q", got, want)

View File

@@ -0,0 +1,45 @@
package config
import (
"path/filepath"
"strings"
configkit "go.maximumdirect.net/configkit"
)
// resourceBase chooses the effective host source, falling back to the main
// source and then the captured invocation directory for contextless inputs.
func resourceBase(resolution *configkit.Resolution, path configkit.Path, invocationDir string) string {
info, found := resolution.EntryInfo(path)
if found && info.Origin.Layer == configkit.LayerCLI {
return invocationDir
}
if found && len(info.Origin.Locations) == 1 {
if base, ok := hostSourceBase(resolution, info.Origin.Locations[0].SourceID); ok {
return base
}
}
if base, ok := hostSourceBase(resolution, "source-1"); ok {
return base
}
return invocationDir
}
func hostSourceBase(resolution *configkit.Resolution, sourceID string) (string, bool) {
source, ok := resolution.Source(sourceID)
if !ok || source.Kind != configkit.SourceKindFile || !filepath.IsAbs(source.Path) {
return "", false
}
if _, ok := resolution.SourceContext(sourceID); !ok {
return "", false
}
return filepath.Dir(source.Path), true
}
func resolveHostResource(resolution *configkit.Resolution, path configkit.Path, value *string, invocationDir string) {
trimmed := strings.TrimSpace(*value)
if trimmed == "" || filepath.IsAbs(trimmed) {
return
}
configkit.Normalize(resolution, path, value, filepath.Join(resourceBase(resolution, path, invocationDir), trimmed), "resolved relative resource from its source")
}

View File

@@ -0,0 +1,127 @@
package config
import (
"os"
"path/filepath"
"testing"
configkit "go.maximumdirect.net/configkit"
)
func TestPipelineResourcePathsFollowEffectiveSources(t *testing.T) {
dir := t.TempDir()
root := writePipelineFixture(t, dir, "pipeline.yml", `config:
imports:
- path: fragments/shared.yml
workspace: {root: root-work}
`)
writePipelineFixture(t, dir, "fragments/shared.yml", `campaigns: {root: campaigns}
spool: {root: /absolute/spool}
cache: {root: cache}
audita: {config_path: config/tool.yml}
scriptorium: {extends: audita}
notarius:
extends: audita
enabled: true
pipeline_id: dnd-session
`)
writePipelineFixture(t, dir, "overlays/testing.yml", "audita:\n config_path: custom.yml\n")
base, err := LoadPipeline(root)
if err != nil {
t.Fatal(err)
}
wantConfig := filepath.Join(dir, "fragments", "config", "tool.yml")
if base.Audita.ConfigPath != wantConfig || base.Scriptorium.ConfigPath != wantConfig || base.Notarius.ConfigPath != wantConfig {
t.Fatalf("resource paths: audita=%q scriptorium=%q notarius=%q", base.Audita.ConfigPath, base.Scriptorium.ConfigPath, base.Notarius.ConfigPath)
}
if base.Notarius.WorkingDirectory != filepath.Dir(wantConfig) || base.Workspace.Root != filepath.Join(dir, "root-work") || base.Campaigns.Root != filepath.Join(dir, "fragments", "campaigns") || base.Cache.Root != filepath.Join(dir, "fragments", "cache") || base.Spool.Root != "/absolute/spool" {
t.Fatalf("resolved paths: %#v", base)
}
overlay := "testing"
updated, err := LoadPipelineWithOptions(root, PipelineLoadOptions{Overlay: &overlay})
if err != nil {
t.Fatal(err)
}
wantOverlay := filepath.Join(dir, "overlays", "custom.yml")
if updated.Audita.ConfigPath != wantOverlay || updated.Scriptorium.ConfigPath != wantOverlay || updated.Notarius.ConfigPath != wantOverlay {
t.Fatalf("overlay resource paths: audita=%q scriptorium=%q notarius=%q", updated.Audita.ConfigPath, updated.Scriptorium.ConfigPath, updated.Notarius.ConfigPath)
}
}
func TestResourceBaseUsesInvocationDirectoryForContextlessAndCLIValues(t *testing.T) {
invocation := t.TempDir()
type document struct {
Path string `yaml:"path"`
}
for _, overrides := range [][]configkit.Override{nil, {{Path: configkit.Keys("path"), Value: "cli.txt", Flag: "--path"}}} {
result, err := configkit.Load(configkit.Bytes("remote snapshot", []byte("path: input.txt\n")), configkit.Options[document]{
Overrides: overrides,
Definition: configkit.Definition[document]{Resolve: func(doc *document, resolution *configkit.Resolution) error {
resolveHostResource(resolution, configkit.Keys("path"), &doc.Path, invocation)
return nil
}},
})
if err != nil {
t.Fatal(err)
}
name := "input.txt"
if len(overrides) > 0 {
name = "cli.txt"
}
if got, want := result.Config.Path, filepath.Join(invocation, name); got != want {
t.Fatalf("resolved %q, want %q", got, want)
}
}
}
func TestEquivalentPipelineResourceLayoutsHaveEqualDigests(t *testing.T) {
dir := t.TempDir()
first := writePipelineFixture(t, dir, "first.yml", "audita: {config_path: shared/tool.yml}\n")
second := writePipelineFixture(t, dir, "second.yml", "config:\n imports:\n - path: shared/settings.yml\n")
writePipelineFixture(t, dir, "shared/settings.yml", "audita: {config_path: tool.yml}\n")
a, err := LoadPipeline(first)
if err != nil {
t.Fatal(err)
}
b, err := LoadPipeline(second)
if err != nil {
t.Fatal(err)
}
if a.Audita.ConfigPath != b.Audita.ConfigPath || EffectivePipelineDigest(a) != EffectivePipelineDigest(b) {
t.Fatalf("equivalent resources differ: %q / %q", a.Audita.ConfigPath, b.Audita.ConfigPath)
}
if _, err := os.Stat(filepath.Join(dir, "shared", "tool.yml")); !os.IsNotExist(err) {
t.Fatalf("fixture unexpectedly exists: %v", err)
}
}
func TestResourceBaseFallsBackToMainForCompositeAndDefaultOrigins(t *testing.T) {
dir := t.TempDir()
root := writePipelineFixture(t, dir, "pipeline.yml", "config:\n imports:\n - path: fragment/part.yml\nvalues: {first: one}\n")
writePipelineFixture(t, dir, "fragment/part.yml", "values: {second: two}\n")
type document struct {
Values map[string]string `yaml:"values"`
Default string `yaml:"default"`
}
var compositeBase, defaultBase string
var contributors int
_, err := configkit.Load(configkit.File(root), configkit.Options[document]{
Imports: true,
Definition: configkit.Definition[document]{
Defaults: func() document { return document{Default: "default.txt"} },
Resolve: func(_ *document, resolution *configkit.Resolution) error {
info, _ := resolution.EntryInfo(configkit.Keys("values"))
contributors = len(info.Origin.Locations)
compositeBase = resourceBase(resolution, configkit.Keys("values"), t.TempDir())
defaultBase = resourceBase(resolution, configkit.Keys("default"), t.TempDir())
return nil
},
},
})
if err != nil {
t.Fatal(err)
}
if contributors < 2 || compositeBase != dir || defaultBase != dir {
t.Fatalf("fallback bases = %q, %q; want %q", compositeBase, defaultBase, dir)
}
}