Compose pipeline steps using native identities
This commit is contained in:
180
internal/core/config/collection_composition_test.go
Normal file
180
internal/core/config/collection_composition_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestStepCompositionAcrossSourcesAndOverlays(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write := func(name, data string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write("notarius.yml", `version: 5
|
||||
config:
|
||||
imports:
|
||||
- path: one/pipeline.yml
|
||||
required: true
|
||||
- path: two/pipeline.yml
|
||||
required: true
|
||||
`)
|
||||
write("one/pipeline.yml", `pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: first
|
||||
references: {source: {path: ./first.txt}}
|
||||
artifacts: {first: {extract: {module: extract, retries: 4}}}
|
||||
`)
|
||||
write("two/pipeline.yml", `pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: second
|
||||
references: {source: {path: ./second.txt}}
|
||||
artifacts: {second: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/upsert.yml", `pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: first
|
||||
artifacts: {replacement: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/replace.yml", `config:
|
||||
sequences:
|
||||
- destination: [pipelines, session, steps]
|
||||
mode: replace
|
||||
pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: third
|
||||
artifacts: {third: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/append.yml", `config:
|
||||
sequences:
|
||||
- destination: [pipelines, session, steps]
|
||||
mode: append
|
||||
pipelines:
|
||||
session:
|
||||
steps:
|
||||
- id: third
|
||||
artifacts: {third: {extract: {module: extract}}}
|
||||
`)
|
||||
write("overlays/clear.yml", `config:
|
||||
sequences:
|
||||
- destination: [pipelines, session, steps]
|
||||
mode: replace
|
||||
pipelines:
|
||||
session: {steps: []}
|
||||
`)
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
loaded, err := Load(main, LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := loaded.Config.Pipelines["session"].Steps
|
||||
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || steps[0].References["source"].Path != filepath.Join(root, "one", "first.txt") || steps[1].References["source"].Path != filepath.Join(root, "two", "second.txt") {
|
||||
t.Fatalf("composed steps = %#v", steps)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
overlay string
|
||||
ids []string
|
||||
}{{"upsert", []string{"first", "second"}}, {"replace", []string{"third"}}, {"append", []string{"first", "second", "third"}}} {
|
||||
t.Run(tc.overlay, func(t *testing.T) {
|
||||
got, err := Load(main, LoadOptions{Overlay: tc.overlay})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := got.Config.Pipelines["session"].Steps
|
||||
if len(steps) != len(tc.ids) {
|
||||
t.Fatalf("steps = %#v", steps)
|
||||
}
|
||||
for i, id := range tc.ids {
|
||||
if steps[i].ID != id {
|
||||
t.Fatalf("steps = %#v", steps)
|
||||
}
|
||||
}
|
||||
if tc.overlay == "upsert" && (len(steps[0].References) != 0 || steps[0].Artifacts["replacement"].Extract.Module != "extract") {
|
||||
t.Fatalf("upsert did not replace record: %#v", steps[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := Load(main, LoadOptions{Overlay: "clear"}); err == nil {
|
||||
t.Fatal("domain validation accepted empty steps")
|
||||
}
|
||||
overrides := []configkit.Override{{Path: configkit.Keys("pipelines", "session", "steps"), Value: []any{map[string]any{"id": "override", "artifacts": map[string]any{"result": map[string]any{"extract": map[string]any{"module": "extract"}}}}}, Flag: "--steps"}}
|
||||
replaced, err := Load(main, LoadOptions{Overrides: overrides})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := replaced.Config.Pipelines["session"].Steps; len(got) != 1 || got[0].ID != "override" {
|
||||
t.Fatalf("override steps = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepIdentityRejectsDuplicatePeersWithoutRegisteringOtherLists(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, first, second string
|
||||
}{
|
||||
{"duplicate steps", "steps: [{id: same, artifacts: {a: {extract: {module: extract}}}}]", "steps: [{id: same, artifacts: {b: {extract: {module: extract}}}}]"},
|
||||
{"unregistered validators", "input: {module: input, validators: [{module: one}]}", "input: {validators: [{module: two}]}"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for name, data := range map[string]string{
|
||||
"main.yml": "version: 5\nconfig:\n imports:\n - path: one.yml\n - path: two.yml\n",
|
||||
"one.yml": "pipelines:\n session:\n " + tc.first + "\n",
|
||||
"two.yml": "pipelines:\n session:\n " + tc.second + "\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := Load(filepath.Join(root, "main.yml"), LoadOptions{}); err == nil {
|
||||
t.Fatal("conflicting peer lists were accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepRecordInheritanceReplacesMatchingRecord(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
pipelines:
|
||||
parent:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: first
|
||||
artifacts: {old: {extract: {module: extract, retries: 4}}}
|
||||
- id: second
|
||||
artifacts: {second: {extract: {module: extract}}}
|
||||
child:
|
||||
extends: parent
|
||||
steps:
|
||||
- id: first
|
||||
artifacts: {new: {extract: {module: extract}}}
|
||||
`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := Load(main, LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
steps := loaded.Config.Pipelines["child"].Steps
|
||||
if len(steps) != 2 || steps[0].ID != "first" || steps[1].ID != "second" || len(steps[0].Artifacts) != 1 || steps[0].Artifacts["new"].Extract.Module != "extract" {
|
||||
t.Fatalf("inherited steps = %#v", steps)
|
||||
}
|
||||
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "child", Catalog: effectiveCatalog(t)}); err != nil {
|
||||
t.Fatalf("resolve composed pipeline: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user