2026-11-17 20:48:05 +00:00
|
|
|
package config
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
configkit "go.maximumdirect.net/configkit"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func TestStepCompositionAcrossSourcesAndOverlays(t *testing.T) {
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
write := func(name, data string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
path := filepath.Join(root, name)
|
|
|
|
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
write("notarius.yml", `version: 5
|
|
|
|
|
config:
|
|
|
|
|
imports:
|
|
|
|
|
- path: one/pipeline.yml
|
|
|
|
|
required: true
|
|
|
|
|
- path: two/pipeline.yml
|
|
|
|
|
required: true
|
|
|
|
|
`)
|
|
|
|
|
write("one/pipeline.yml", `pipelines:
|
|
|
|
|
session:
|
|
|
|
|
input: {module: input}
|
|
|
|
|
steps:
|
|
|
|
|
- id: first
|
|
|
|
|
references: {source: {path: ./first.txt}}
|
|
|
|
|
artifacts: {first: {extract: {module: extract, retries: 4}}}
|
|
|
|
|
`)
|
|
|
|
|
write("two/pipeline.yml", `pipelines:
|
|
|
|
|
session:
|
|
|
|
|
steps:
|
|
|
|
|
- id: second
|
|
|
|
|
references: {source: {path: ./second.txt}}
|
|
|
|
|
artifacts: {second: {extract: {module: extract}}}
|
|
|
|
|
`)
|
|
|
|
|
write("overlays/upsert.yml", `pipelines:
|
|
|
|
|
session:
|
|
|
|
|
steps:
|
|
|
|
|
- id: first
|
|
|
|
|
artifacts: {replacement: {extract: {module: extract}}}
|
|
|
|
|
`)
|
|
|
|
|
write("overlays/replace.yml", `config:
|
|
|
|
|
sequences:
|
|
|
|
|
- destination: [pipelines, session, steps]
|
|
|
|
|
mode: replace
|
|
|
|
|
pipelines:
|
|
|
|
|
session:
|
|
|
|
|
steps:
|
|
|
|
|
- id: third
|
|
|
|
|
artifacts: {third: {extract: {module: extract}}}
|
|
|
|
|
`)
|
|
|
|
|
write("overlays/append.yml", `config:
|
|
|
|
|
sequences:
|
|
|
|
|
- destination: [pipelines, session, steps]
|
|
|
|
|
mode: append
|
|
|
|
|
pipelines:
|
|
|
|
|
session:
|
|
|
|
|
steps:
|
|
|
|
|
- id: third
|
|
|
|
|
artifacts: {third: {extract: {module: extract}}}
|
|
|
|
|
`)
|
|
|
|
|
write("overlays/clear.yml", `config:
|
|
|
|
|
sequences:
|
|
|
|
|
- destination: [pipelines, session, steps]
|
|
|
|
|
mode: replace
|
|
|
|
|
pipelines:
|
|
|
|
|
session: {steps: []}
|
|
|
|
|
`)
|
|
|
|
|
main := filepath.Join(root, "notarius.yml")
|
|
|
|
|
loaded, err := Load(main, LoadOptions{})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
steps := loaded.Config.Pipelines["session"].Steps
|
|
|
|
|
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || steps[0].References["source"].Path != filepath.Join(root, "one", "first.txt") || steps[1].References["source"].Path != filepath.Join(root, "two", "second.txt") {
|
|
|
|
|
t.Fatalf("composed steps = %#v", steps)
|
|
|
|
|
}
|
|
|
|
|
for _, tc := range []struct {
|
|
|
|
|
overlay string
|
|
|
|
|
ids []string
|
|
|
|
|
}{{"upsert", []string{"first", "second"}}, {"replace", []string{"third"}}, {"append", []string{"first", "second", "third"}}} {
|
|
|
|
|
t.Run(tc.overlay, func(t *testing.T) {
|
|
|
|
|
got, err := Load(main, LoadOptions{Overlay: tc.overlay})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
steps := got.Config.Pipelines["session"].Steps
|
|
|
|
|
if len(steps) != len(tc.ids) {
|
|
|
|
|
t.Fatalf("steps = %#v", steps)
|
|
|
|
|
}
|
|
|
|
|
for i, id := range tc.ids {
|
|
|
|
|
if steps[i].ID != id {
|
|
|
|
|
t.Fatalf("steps = %#v", steps)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if tc.overlay == "upsert" && (len(steps[0].References) != 0 || steps[0].Artifacts["replacement"].Extract.Module != "extract") {
|
|
|
|
|
t.Fatalf("upsert did not replace record: %#v", steps[0])
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
if _, err := Load(main, LoadOptions{Overlay: "clear"}); err == nil {
|
|
|
|
|
t.Fatal("domain validation accepted empty steps")
|
|
|
|
|
}
|
|
|
|
|
overrides := []configkit.Override{{Path: configkit.Keys("pipelines", "session", "steps"), Value: []any{map[string]any{"id": "override", "artifacts": map[string]any{"result": map[string]any{"extract": map[string]any{"module": "extract"}}}}}, Flag: "--steps"}}
|
|
|
|
|
replaced, err := Load(main, LoadOptions{Overrides: overrides})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if got := replaced.Config.Pipelines["session"].Steps; len(got) != 1 || got[0].ID != "override" {
|
|
|
|
|
t.Fatalf("override steps = %#v", got)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestStepIdentityRejectsDuplicatePeersWithoutRegisteringOtherLists(t *testing.T) {
|
|
|
|
|
for _, tc := range []struct {
|
|
|
|
|
name, first, second string
|
|
|
|
|
}{
|
|
|
|
|
{"duplicate steps", "steps: [{id: same, artifacts: {a: {extract: {module: extract}}}}]", "steps: [{id: same, artifacts: {b: {extract: {module: extract}}}}]"},
|
|
|
|
|
{"unregistered validators", "input: {module: input, validators: [{module: one}]}", "input: {validators: [{module: two}]}"},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
for name, data := range map[string]string{
|
|
|
|
|
"main.yml": "version: 5\nconfig:\n imports:\n - path: one.yml\n - path: two.yml\n",
|
|
|
|
|
"one.yml": "pipelines:\n session:\n " + tc.first + "\n",
|
|
|
|
|
"two.yml": "pipelines:\n session:\n " + tc.second + "\n",
|
|
|
|
|
} {
|
|
|
|
|
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if _, err := Load(filepath.Join(root, "main.yml"), LoadOptions{}); err == nil {
|
|
|
|
|
t.Fatal("conflicting peer lists were accepted")
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestStepRecordInheritanceReplacesMatchingRecord(t *testing.T) {
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
main := filepath.Join(root, "main.yml")
|
|
|
|
|
if err := os.WriteFile(main, []byte(`version: 5
|
|
|
|
|
pipelines:
|
|
|
|
|
parent:
|
|
|
|
|
input: {module: input}
|
|
|
|
|
steps:
|
|
|
|
|
- id: first
|
|
|
|
|
artifacts: {old: {extract: {module: extract, retries: 4}}}
|
|
|
|
|
- id: second
|
|
|
|
|
artifacts: {second: {extract: {module: extract}}}
|
|
|
|
|
child:
|
|
|
|
|
extends: parent
|
|
|
|
|
steps:
|
|
|
|
|
- id: first
|
|
|
|
|
artifacts: {new: {extract: {module: extract}}}
|
|
|
|
|
`), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
loaded, err := Load(main, LoadOptions{})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
steps := loaded.Config.Pipelines["child"].Steps
|
|
|
|
|
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || len(steps[0].Artifacts) != 1 || steps[0].Artifacts["new"].Extract.Module != "extract" {
|
|
|
|
|
t.Fatalf("inherited steps = %#v", steps)
|
|
|
|
|
}
|
|
|
|
|
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)}); err != nil {
|
|
|
|
|
t.Fatalf("resolve composed pipeline: %v", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-17 20:50:07 +00:00
|
|
|
|
|
|
|
|
func TestEmptyStepOperationsAndOptionalImports(t *testing.T) {
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
main := filepath.Join(root, "main.yml")
|
|
|
|
|
if err := os.WriteFile(main, []byte(`version: 5
|
|
|
|
|
config:
|
|
|
|
|
imports:
|
|
|
|
|
- path: absent.yml
|
|
|
|
|
pipelines:
|
|
|
|
|
session:
|
|
|
|
|
input: {module: input}
|
|
|
|
|
steps:
|
|
|
|
|
- id: first
|
|
|
|
|
artifacts: {result: {extract: {module: extract}}}
|
|
|
|
|
`), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
for _, tc := range []struct {
|
|
|
|
|
mode, value string
|
|
|
|
|
wantError bool
|
|
|
|
|
}{
|
|
|
|
|
{"append", "[]", false}, {"upsert", "[]", false}, {"replace", "[]", true}, {"replace", "null", true},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(tc.mode+tc.value, func(t *testing.T) {
|
|
|
|
|
name := tc.mode + map[string]string{"[]": "empty", "null": "null"}[tc.value]
|
|
|
|
|
path := filepath.Join(root, "overlays", name+".yml")
|
|
|
|
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
data := "config:\n sequences:\n - destination: [pipelines, session, steps]\n mode: " + tc.mode + "\npipelines:\n session: {steps: " + tc.value + "}\n"
|
|
|
|
|
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
loaded, err := Load(main, LoadOptions{Overlay: name})
|
|
|
|
|
if tc.wantError {
|
|
|
|
|
if err == nil {
|
|
|
|
|
t.Fatal("invalid steps were accepted")
|
|
|
|
|
}
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
steps := loaded.Config.Pipelines["session"].Steps
|
|
|
|
|
if len(steps) != 1 || steps[0].ID != "first" {
|
|
|
|
|
t.Fatalf("steps = %#v", steps)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|