Migrate run state and reference contracts to native execution

This commit is contained in:
2026-11-22 05:24:30 +00:00
parent 5cfcf10a8b
commit 936fc1e72a
7 changed files with 450 additions and 1765 deletions

View File

@@ -2,54 +2,25 @@ package cli
import (
"bytes"
"errors"
"os"
"strings"
"testing"
)
func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
func TestConfigCommandsDoNotAllocateRunState(t *testing.T) {
for _, args := range [][]string{
{"config", "validate", "--config"},
{"pipelines", "list", "--config"},
{"config", "validate"}, {"config", "show"}, {"pipeline", "list"}, {"pipeline", "validate", "dnd-session"},
} {
name := strings.Join(args[:2], "-")
t.Run(name, func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") }
command := append([]string(nil), args...)
command = append(command, roots.config)
var stdout, stderr bytes.Buffer
code := RunWithOptions(command, &stdout, &stderr, opts)
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
fixture := newStateRunFixture(t)
var stdout, stderr bytes.Buffer
command := append(append([]string(nil), args...), "--config", fixture.configPath)
code := RunWithOptions(command, &stdout, &stderr, fixture.options)
if code != 0 {
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
assertAbsent(t, fixture.outputRoot)
assertAbsent(t, fixture.cacheRoot)
assertAbsent(t, fixture.debugRoot)
if len(fixture.client.requestsSnapshot()) != 0 {
t.Fatal("inspection invoked provider")
}
}
}
func replaceStateTestConfigLine(t *testing.T, path, old, new string) {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
text := replaceRequiredOnce(t, string(data), old, new)
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
t.Fatal(err)
}
}
func removeStateTestConfigLine(t *testing.T, path, line string) {
replaceStateTestConfigLine(t, path, line, "")
}
func assertNoRunState(t *testing.T, roots stateTestRoots) {
t.Helper()
assertAbsent(t, roots.output)
assertAbsent(t, roots.plans)
assertAbsent(t, roots.checkpoints)
assertAbsent(t, roots.debug)
}

View File

@@ -404,3 +404,59 @@ type exampleOutputIndexEntry struct {
func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput {
return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)}
}
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
catalog := catalogFromRegistries(productionTestComponents(t).registries)
sources := map[string]string{
"party": "/references/party.txt",
"players": "/references/players.txt",
"glossary": "/references/glossary.txt",
"spell_catalog": "/references/spells.json",
}
requests := make([]cliReferenceRequest, 0, len(sources))
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
requests = append(requests, cliReferenceRequest{
Selector: mustParseReferenceSelector(t, slot, "--reference"),
Source: sources[slot],
})
}
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
if err != nil {
t.Fatalf("expand complete D&D references: %v", err)
}
if len(unbinds) != 0 {
t.Fatalf("unbinds = %#v, want none", unbinds)
}
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
for _, binding := range overrides {
actual[referenceContractBindingLabel(binding)] = binding
}
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
if err != nil {
t.Fatal(err)
}
matched := make(map[string]int, len(sources))
for _, target := range targets {
for slot, sourcePath := range sources {
if _, ok := target.slots[slot]; !ok {
continue
}
matched[slot]++
label := targetLabel(target) + "." + slot
binding, ok := actual[label]
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
}
}
}
for slot := range sources {
if matched[slot] < 2 {
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
}
}
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
t.Fatalf("resolve complete D&D CLI references: %v", err)
}
}

View File

@@ -86,62 +86,6 @@ func TestReferenceSelectorSpecificityAndFinalOccurrenceChooseConcreteBindings(t
}
}
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
catalog := catalogFromRegistries(productionTestComponents(t).registries)
sources := map[string]string{
"party": "/references/party.txt",
"players": "/references/players.txt",
"glossary": "/references/glossary.txt",
"spell_catalog": "/references/spells.json",
}
requests := make([]cliReferenceRequest, 0, len(sources))
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
requests = append(requests, cliReferenceRequest{
Selector: mustParseReferenceSelector(t, slot, "--reference"),
Source: sources[slot],
})
}
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
if err != nil {
t.Fatalf("expand complete D&D references: %v", err)
}
if len(unbinds) != 0 {
t.Fatalf("unbinds = %#v, want none", unbinds)
}
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
for _, binding := range overrides {
actual[referenceContractBindingLabel(binding)] = binding
}
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
if err != nil {
t.Fatal(err)
}
matched := make(map[string]int, len(sources))
for _, target := range targets {
for slot, sourcePath := range sources {
if _, ok := target.slots[slot]; !ok {
continue
}
matched[slot]++
label := targetLabel(target) + "." + slot
binding, ok := actual[label]
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
}
}
}
for slot := range sources {
if matched[slot] < 2 {
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
}
}
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
t.Fatalf("resolve complete D&D CLI references: %v", err)
}
}
func TestReferenceSelectorsRejectInvalidOrUnselectedScopesBeforeMaterialization(t *testing.T) {
cfg := referenceContractConfig()
catalog := referenceContractCatalog(t, true, true)
@@ -173,14 +117,14 @@ func TestReferenceSyntaxErrorsReturnTwo(t *testing.T) {
name string
args []string
}{
{name: "reference missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference"}},
{name: "reference missing selector", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "=path.txt"}},
{name: "reference missing separator", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot"}},
{name: "reference missing path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot="}},
{name: "reference excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "a.b.c.d=path.txt"}},
{name: "unbind with path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "slot=path.txt"}},
{name: "unbind excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "a.b.c.d"}},
{name: "unbind missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference"}},
{name: "reference missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference"}},
{name: "reference missing selector", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "=path.txt"}},
{name: "reference missing separator", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "slot"}},
{name: "reference missing path", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "slot="}},
{name: "reference excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "a.b.c.d=path.txt"}},
{name: "unbind with path", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference", "slot=path.txt"}},
{name: "unbind excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference", "a.b.c.d"}},
{name: "unbind missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -289,7 +233,7 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T)
workingDir := t.TempDir()
cfg := referenceContractConfig()
configPath := filepath.Join(configDir, "config.yml")
if err := os.WriteFile(configPath, []byte("version: 5\n"), 0o600); err != nil {
if err := os.WriteFile(configPath, []byte("version: 6\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(configDir, "required.txt"), []byte("config reference"), 0o600); err != nil {
@@ -348,10 +292,9 @@ func TestPipelineScopedCLIReferenceProtectsGeneratedHandoff(t *testing.T) {
Selector: mustParseReferenceSelector(t, "shared", "--reference"),
Source: filepath.Join(t.TempDir(), "never-read.json"),
}}, nil)
if err != nil {
t.Fatalf("expand CLI reference: %v", err)
if err == nil {
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
}
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
if err == nil || !strings.Contains(err.Error(), "conflicting generated and external bindings") || strings.Contains(err.Error(), "never-read.json") {
t.Fatalf("resolve error = %v, want generated/external conflict before file access", err)
}
@@ -482,24 +425,55 @@ func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormal
t.Fatal(err)
}
}
register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }))
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{}))
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{}))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[referenceContractArtifact], error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[referenceContractArtifact], error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[referenceContractArtifact], error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
if includeBetaMerger {
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[referenceContractArtifact], error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
}
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[referenceContractArtifact], error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
if includeBetaNormalizer {
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[referenceContractArtifact], error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
}
register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{}, nil }))
register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
t.Fatal("reference selection constructed a module")
return nil, nil
}))
return catalogFromRegistries(registries)
}
type referenceContractArtifact struct {
Value string `json:"value"`
}
type referenceContractCodecB struct{}
type referenceContractCodecA struct{}
@@ -509,16 +483,16 @@ func (referenceContractCodecA) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "reference.alpha", Name: "reference_alpha", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (referenceContractCodecA) MediaType() string { return "application/json" }
func (referenceContractCodecA) EncodeCandidate(stateTestArtifact) ([]byte, error) {
func (referenceContractCodecA) EncodeCandidate(referenceContractArtifact) ([]byte, error) {
return []byte(`{"value":"ok"}`), nil
}
func (referenceContractCodecA) Encode(stateTestArtifact) ([]byte, error) {
func (referenceContractCodecA) Encode(referenceContractArtifact) ([]byte, error) {
return []byte(`{"value":"ok"}`), nil
}
func (referenceContractCodecA) Decode([]byte) (stateTestArtifact, error) {
return stateTestArtifact{Value: "ok"}, nil
func (referenceContractCodecA) Decode([]byte) (referenceContractArtifact, error) {
return referenceContractArtifact{Value: "ok"}, nil
}
func (codec referenceContractCodecA) DecodeCandidate(content []byte) (stateTestArtifact, error) {
func (codec referenceContractCodecA) DecodeCandidate(content []byte) (referenceContractArtifact, error) {
return codec.Decode(content)
}
@@ -527,16 +501,16 @@ func (referenceContractCodecB) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "reference.beta", Name: "reference_beta", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (referenceContractCodecB) MediaType() string { return "application/json" }
func (referenceContractCodecB) EncodeCandidate(stateTestArtifact) ([]byte, error) {
func (referenceContractCodecB) EncodeCandidate(referenceContractArtifact) ([]byte, error) {
return []byte(`{"value":"ok"}`), nil
}
func (referenceContractCodecB) Encode(stateTestArtifact) ([]byte, error) {
func (referenceContractCodecB) Encode(referenceContractArtifact) ([]byte, error) {
return []byte(`{"value":"ok"}`), nil
}
func (referenceContractCodecB) Decode([]byte) (stateTestArtifact, error) {
return stateTestArtifact{Value: "ok"}, nil
func (referenceContractCodecB) Decode([]byte) (referenceContractArtifact, error) {
return referenceContractArtifact{Value: "ok"}, nil
}
func (codec referenceContractCodecB) DecodeCandidate(content []byte) (stateTestArtifact, error) {
func (codec referenceContractCodecB) DecodeCandidate(content []byte) (referenceContractArtifact, error) {
return codec.Decode(content)
}

View File

@@ -2,750 +2,110 @@ package cli
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
tests := []struct {
name string
args func(stateTestRoots) []string
}{
{name: "missing pipeline", args: func(roots stateTestRoots) []string {
return []string{"run", "--config", roots.config, "--input", roots.input}
}},
{name: "missing input", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config}
}},
{name: "unknown flag", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--unknown"}
}},
{name: "blank output directory", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--output-dir", ""}
}},
{name: "blank debug directory", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""}
}},
{name: "missing configuration overlay", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--config-overlay"}
}},
{name: "debug directory without debug", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")}
}},
{name: "blank session ID", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""}
}},
{name: "blank LLM profile", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", ""}
}},
{name: "whitespace LLM profile", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", " \t "}
}},
{name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input}
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
var stdout, stderr bytes.Buffer
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertAbsent(t, roots.output)
assertAbsent(t, roots.debug)
})
}
}
func TestRecomputeStepCLIContract(t *testing.T) {
tests := []struct {
name string
configure func(*testing.T, stateTestRoots)
flags []string
wantCode int
wantOutput string
wantError string
}{
{
name: "explicit step",
configure: func(t *testing.T, roots stateTestRoots) {
replaceStateTestConfigLine(t, roots.config, " artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n", " steps:\n - id: chosen\n artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n")
},
flags: []string{"--resume", "--recompute-step", "chosen"},
wantCode: 0,
wantOutput: "outputs=1",
},
{name: "implicit default step", flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 0, wantOutput: "outputs=1"},
{name: "repeated flag", flags: []string{"--resume", "--recompute-step", "default", "--recompute-step", "default"}, wantCode: 2, wantError: "specified only once"},
{name: "empty step", flags: []string{"--resume", "--recompute-step", ""}, wantCode: 2, wantError: "must not be empty"},
{name: "unknown step", flags: []string{"--resume", "--recompute-step", "missing"}, wantCode: 1, wantError: "unknown pipeline step"},
{name: "without resume", flags: []string{"--recompute-step", "default"}, wantCode: 2, wantError: "requires --resume"},
{
name: "checkpoint recording disabled",
configure: func(t *testing.T, roots stateTestRoots) {
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
},
flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 1, wantError: "cache.checkpoints.enabled",
},
{name: "with only", flags: []string{"--resume", "--recompute-step", "default", "--only", "items"}, wantCode: 2, wantError: "cannot be combined with --only"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
if tt.configure != nil {
tt.configure(t, roots)
}
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
args = append(args, tt.flags...)
var stdout, stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
if code != tt.wantCode || (tt.wantOutput != "" && !strings.Contains(stdout.String(), tt.wantOutput)) || (tt.wantError != "" && !strings.Contains(stderr.String(), tt.wantError)) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestRunValidFailuresClassifyAndReportDebug(t *testing.T) {
tests := []struct {
name string
args func(stateTestRoots) []string
wantError string
wantDebug bool
}{
{name: "unknown pipeline", args: func(roots stateTestRoots) []string {
return []string{"run", "missing", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
}, wantError: `pipeline "missing"`},
{name: "unknown lane", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "missing", "--chunk_cache", "bypass", "--debug"}
}, wantError: `lane "missing"`, wantDebug: true},
{name: "unreadable input", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", filepath.Join(filepath.Dir(roots.input), "unreadable.txt"), "--chunk_cache", "bypass", "--debug"}
}, wantError: "read input", wantDebug: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
var stdout, stderr bytes.Buffer
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if tt.wantDebug {
if !strings.Contains(stderr.String(), "debug=") {
t.Fatalf("stderr=%q, want debug path", stderr.String())
}
onlyChildDir(t, roots.debug)
} else {
assertAbsent(t, roots.debug)
}
assertAbsent(t, roots.output)
})
}
}
func TestRunOnlyExecutesSelectedLanes(t *testing.T) {
roots := newStateTestRoots(t)
data, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
data = []byte(replaceRequiredOnce(t, string(data), " output:\n module: test/output\n", " other:\n extract:\n module: test/extract\n output:\n module: test/output\n"))
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
t.Fatal(err)
}
harness := newStateTestHarness()
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "items", "--chunk_cache", "bypass"}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
harness.mu.Lock()
extractCalls := harness.extractCalls
harness.mu.Unlock()
if extractCalls != 1 {
t.Fatalf("extract calls = %d, want only the selected lane", extractCalls)
}
}
func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
t.Run("retired environment roots have no effect", func(t *testing.T) {
roots := newStateTestRoots(t)
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
opts := newStateTestHarness().options()
opts.LookupEnv = lookupRunContractEnv(map[string]string{
"NOTARIUS_OUTPUT_DIR": environmentOutput,
"NOTARIUS_DEBUG_DIR": environmentDebug,
})
result := runWithStateRoots(t, roots, opts, nil)
if result.code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
assertFile(t, filepath.Join(roots.output, filepath.Base(onlyChildDir(t, roots.output)), "result.json"))
onlyChildDir(t, roots.debug)
assertAbsent(t, environmentOutput)
assertAbsent(t, environmentDebug)
})
t.Run("command flags override environment", func(t *testing.T) {
roots := newStateTestRoots(t)
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
flagOutput := filepath.Join(t.TempDir(), "flag-output")
flagDebug := filepath.Join(t.TempDir(), "flag-debug")
opts := newStateTestHarness().options()
opts.LookupEnv = lookupRunContractEnv(map[string]string{
"NOTARIUS_OUTPUT_DIR": environmentOutput,
"NOTARIUS_DEBUG_DIR": environmentDebug,
})
result := runWithStateRoots(t, roots, opts, []string{"--output-dir", flagOutput, "--debug-dir", flagDebug})
if result.code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
assertFile(t, filepath.Join(flagOutput, filepath.Base(onlyChildDir(t, flagOutput)), "result.json"))
onlyChildDir(t, flagDebug)
assertAbsent(t, environmentOutput)
assertAbsent(t, environmentDebug)
})
t.Run("built-in roots", func(t *testing.T) {
roots := newStateTestRoots(t)
data, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
text := string(data)
text = replaceRequiredOnce(t, text, fmt.Sprintf("output:\n directory: %q\n", roots.output), "")
text = replaceRequiredOnce(t, text, fmt.Sprintf("debug:\n directory: %q\n", roots.debug), "")
if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil {
t.Fatal(err)
}
opts := newStateTestHarness().options()
result := runWithStateRoots(t, roots, opts, nil)
if result.code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
configDirectory := filepath.Dir(roots.config)
assertFile(t, filepath.Join(configDirectory, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(configDirectory, "notarius-output"))), "result.json"))
onlyChildDir(t, filepath.Join(configDirectory, "notarius-debug"))
})
}
func TestRunAppliesConfigurationOverlayAndRecordsSelection(t *testing.T) {
roots := newStateTestRoots(t)
overlayDirectory := filepath.Join(filepath.Dir(roots.config), "overlays")
overlayOutput := filepath.Join(overlayDirectory, "overlay-output")
overlayDebug := filepath.Join(overlayDirectory, "overlay-debug")
writeConfigKitFile(t, filepath.Join(overlayDirectory, "development.yml"), "output:\n directory: ./overlay-output\ndebug:\n directory: ./overlay-debug\n")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--config-overlay", "development",
"--input", roots.input, "--chunk_cache", "bypass", "--debug",
}, &stdout, &stderr, newStateTestHarness().options())
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertFile(t, filepath.Join(overlayOutput, filepath.Base(onlyChildDir(t, overlayOutput)), "result.json"))
bundle := onlyChildDir(t, overlayDebug)
var invocation debugbundle.Invocation
readStateTestSummaryJSON(t, bundle, "invocation.json", &invocation)
if invocation.ConfigOverlay != "development" {
t.Fatalf("config overlay = %q, want development", invocation.ConfigOverlay)
}
}
func TestRunRecordsDiscoveredConfigurationPath(t *testing.T) {
roots := newStateTestRoots(t)
work := t.TempDir()
selected := filepath.Join(work, "notarius.yml")
data, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(selected, data, 0600); err != nil {
t.Fatal(err)
}
previous, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
if err := os.Chdir(work); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := os.Chdir(previous); err != nil {
t.Error(err)
}
})
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, newStateTestHarness().options())
if code != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
var invocation debugbundle.Invocation
readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "invocation.json", &invocation)
if invocation.ConfigPath != selected {
t.Fatalf("recorded config path = %q, want %q", invocation.ConfigPath, selected)
}
}
func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) {
for _, overlay := range []string{"", "default"} {
if got := effectiveConfigOverlay(overlay); got != "" {
t.Fatalf("effectiveConfigOverlay(%q) = %q, want empty", overlay, got)
}
}
if got := effectiveConfigOverlay("development"); got != "development" {
t.Fatalf("effectiveConfigOverlay(development) = %q", got)
}
}
func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
t.Run("one effective profile reaches the factory and modules", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "override-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
harness := newStateTestHarness()
var factoryProfiles []string
opts := harness.options()
var factoryOverrides []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
factoryOverrides = append(factoryOverrides, overrides)
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", " override-profile "}, &stdout, &stderr, opts)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
}
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
}
harness.mu.Lock()
profiles := append([]string(nil), harness.moduleProfiles...)
harness.mu.Unlock()
if len(profiles) < 4 {
t.Fatalf("module profiles = %#v, want chunk and lane stage requests", profiles)
}
for _, profile := range profiles {
if profile != "override-profile" {
t.Fatalf("module profiles = %#v, want override on every request", profiles)
}
}
})
t.Run("runtime override applies to validators", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "override-profile", "validator-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
harness := newStateTestHarness()
var validatorProfiles []string
opts := harness.options()
registerRunContractValidator(t, &opts, &validatorProfiles)
factoryProfiles := []string{}
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryProfiles = append(factoryProfiles, profileID)
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts)
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
}
if len(validatorProfiles) != 1 || validatorProfiles[0] != "override-profile" {
t.Fatalf("validator profiles = %#v, want runtime override", validatorProfiles)
}
})
t.Run("unknown profile is rejected without factory access", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "override-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
factoryCalls := 0
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "missing-profile"}, &stdout, &stderr, opts)
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
}
})
t.Run("pipeline default is rejected before factory access", func(t *testing.T) {
roots := newStateTestRoots(t)
profileDir := writeRunContractProfiles(t, "configured-profile")
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
replaceStateTestConfigLine(t, roots.config, " sample:\n", " sample:\n llm_profile: missing-profile\n")
factoryCalls := 0
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, nil
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
}
})
}
func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
tests := []struct {
name string
flags []string
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
var got []LLMRuntimeOverrides
opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
got = append(got, overrides)
return nil, nil, nil
}
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if len(got) != 1 {
t.Fatalf("factory overrides = %#v, want one call", got)
}
if !tt.wantSet {
if got[0].ReasoningEffort != nil {
t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
}
return
}
if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
}
})
}
}
func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
tests := []struct {
name string
flags []string
wantError string
}{
{
name: "mutually exclusive controls",
flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
wantError: "cannot be combined",
},
{
name: "empty replacement",
flags: []string{"--reasoning-effort", " "},
wantError: "must not be empty",
},
{
name: "duplicate replacement",
flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
wantError: "may be specified only once",
},
{
name: "missing replacement",
flags: []string{"--reasoning-effort"},
wantError: "flag needs an argument",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
var stdout, stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertNoRunState(t, roots)
})
}
}
func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
replacement := " focused "
cleared := ""
states := []struct {
name string
overrides LLMRuntimeOverrides
wantValue string
wantSet bool
}{
{name: "inherit"},
{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
}
digests := make(map[string]string, len(states))
for _, state := range states {
fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
var value string
var found bool
for _, fingerprint := range fingerprints {
if fingerprint.Name == "reasoning_effort_override" {
value, found = fingerprint.Value, true
}
}
if found != state.wantSet || (found && value != state.wantValue) {
t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
}
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
RawInputDigest: "sha256:input",
RuntimeOverrides: fingerprints,
})
if err != nil {
t.Fatal(err)
}
digests[state.name] = identity.Digest
}
if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
}
}
func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
resolved := pipeline.ResolvedPipeline{
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},
InputExecutionClass: contracts.ExecutionClassLLMBacked,
Chunk: pipeline.ModuleBinding{LLMProfile: " zeta "},
ChunkExecutionClass: contracts.ExecutionClassLLMBacked,
Steps: []pipeline.ResolvedPipelineStep{{
ID: "default",
ArtifactLanes: []pipeline.ResolvedArtifactLane{{
Extract: pipeline.ModuleBinding{LLMProfile: "alpha"},
ExtractExecutionClass: contracts.ExecutionClassLLMBacked,
Merge: pipeline.ModuleBinding{LLMProfile: "deterministic-merge"},
MergeExecutionClass: contracts.ExecutionClassDeterministic,
Normalize: pipeline.ModuleBinding{LLMProfile: " gamma "},
NormalizeExecutionClass: contracts.ExecutionClassLLMBacked,
}},
}},
ValidatorChains: []pipeline.ResolvedValidatorChain{{Validators: []pipeline.ResolvedValidator{
{Binding: pipeline.ModuleBinding{LLMProfile: "deterministic-profile"}, ExecutionClass: contracts.ExecutionClassDeterministic},
{Binding: pipeline.ModuleBinding{LLMProfile: "beta"}, ExecutionClass: contracts.ExecutionClassLLMBacked},
}}},
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
OutputExecutionClass: contracts.ExecutionClassLLMBacked,
}
got := effectiveLLMProfileIDs(resolved)
want := []string{"alpha", "beta", "gamma", "input-profile", "output-profile", "zeta"}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("effective profiles = %#v, want %#v", got, want)
}
}
func TestRunSessionIDUsesEffectiveValueForPromptRequests(t *testing.T) {
for _, tt := range []struct {
name string
args []string
want string
}{
{name: "derived default"},
{name: "explicit trimmed value", args: []string{"--session-id", " explicit-session "}, want: "explicit-session"},
func TestRunInvalidSyntaxCreatesNoRunState(t *testing.T) {
for _, flags := range [][]string{
{"--unknown"},
{"--output-dir="},
{"--overlay"},
{"--llm-temperature=invalid"},
{"--input-file="},
} {
t.Run(tt.name, func(t *testing.T) {
roots := newStateTestRoots(t)
want := tt.want
if want == "" {
rawInput, err := os.ReadFile(roots.input)
if err != nil {
t.Fatal(err)
}
want, err = resolvePromptSessionID("", "test/input", rawInput)
if err != nil {
t.Fatal(err)
}
t.Run(strings.Join(flags, " "), func(t *testing.T) {
fixture := newStateRunFixture(t)
result := fixture.run(t, flags...)
if result.status != 2 || result.stdout != "" || result.stderr == "" {
t.Fatalf("syntax error=%#v", result)
}
harness := newStateTestHarness()
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, tt.args...)
var stdout, stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, harness.options())
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
harness.mu.Lock()
sessions := append([]string(nil), harness.sessionIDs...)
harness.mu.Unlock()
if len(sessions) < 4 {
t.Fatalf("session IDs = %#v, want all prompt-facing module requests", sessions)
}
for _, session := range sessions {
if session != want {
t.Fatalf("session IDs = %#v, want %q", sessions, want)
}
}
var manifest artifacts.RunManifest
readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "run-manifest.json", &manifest)
if session, ok := manifest.Metadata["session_id"]; !ok || session != want {
t.Fatalf("manifest session = %#v, want %q; metadata = %#v", session, want, manifest.Metadata)
assertAbsent(t, fixture.outputRoot)
assertAbsent(t, fixture.cacheRoot)
assertAbsent(t, fixture.debugRoot)
if len(fixture.client.requestsSnapshot()) != 0 {
t.Fatal("syntax error reached provider")
}
})
}
}
func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
t.Run("LLM factory", func(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
return nil, nil, errors.New("injected LLM factory failure")
func TestRunStateRootsIgnoreRetiredEnvironmentAndHonorNativeFlags(t *testing.T) {
fixture := newStateRunFixture(t)
environmentOutput, environmentDebug := filepath.Join(t.TempDir(), "environment-output"), filepath.Join(t.TempDir(), "environment-debug")
fixture.options.LookupEnv = func(name string) (string, bool) {
switch name {
case "NOTARIUS_OUTPUT_DIR":
return environmentOutput, true
case "NOTARIUS_DEBUG_DIR":
return environmentDebug, true
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
if code != 1 || !strings.Contains(stderr.String(), "injected LLM factory failure") || stdout.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
t.Run("pipeline preparation", func(t *testing.T) {
roots := newStateTestRoots(t)
data, err := os.ReadFile(roots.config)
if err != nil {
t.Fatal(err)
}
data = []byte(replaceRequiredOnce(t, string(data), "module: test/extract", "module: test/failing-extract"))
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
t.Fatal(err)
}
opts := newStateTestHarness().options()
if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) {
return nil, errors.New("injected extractor construction failure")
}); err != nil {
t.Fatal(err)
}
opts.Catalog = catalogFromRegistries(opts.Registries)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
if code != 1 || !strings.Contains(stderr.String(), "injected extractor construction failure") || stdout.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
return "", false
}
first := fixture.run(t, "--debug", "--cache", "bypass")
if first.status != 0 {
t.Fatalf("configured roots=%#v", first)
}
assertFile(t, filepath.Join(first.report.OutputDirectory, "lanes", "spells.json"))
assertFile(t, filepath.Join(first.report.DebugDirectory, "outcome.json"))
flagOutput, flagDebug := filepath.Join(t.TempDir(), "flag-output"), filepath.Join(t.TempDir(), "flag-debug")
second := fixture.run(t, "--debug", "--cache", "bypass", "--output-dir", flagOutput, "--debug-dir", flagDebug)
if second.status != 0 || !strings.HasPrefix(second.report.OutputDirectory, flagOutput+string(os.PathSeparator)) || !strings.HasPrefix(second.report.DebugDirectory, flagDebug+string(os.PathSeparator)) {
t.Fatalf("flag roots=%#v", second)
}
assertAbsent(t, environmentOutput)
assertAbsent(t, environmentDebug)
}
func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.includeWarnings = true
harness.includeWarningFile = true
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
output, err := os.ReadFile(outputPath)
if err != nil || !strings.Contains(string(output), "contract-warning") {
t.Fatalf("durable output = %q, %v", output, err)
}
assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
bundle := onlyChildDir(t, roots.debug)
var diagnostics contracts.DiagnosticCollection
readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
t.Fatalf("debug diagnostics = %#v", diagnostics)
func TestRunDebugDirectoryAloneDoesNotEnableCapture(t *testing.T) {
fixture := newStateRunFixture(t)
debug := filepath.Join(t.TempDir(), "explicit-debug")
result := fixture.run(t, "--cache", "bypass", "--debug-dir", debug)
if result.status != 0 || result.report.DebugDirectory != "" {
t.Fatalf("run=%#v", result)
}
assertAbsent(t, debug)
assertAbsent(t, fixture.debugRoot)
}
func runWithStateRoots(t *testing.T, roots stateTestRoots, opts Options, extra []string) stateTestResult {
t.Helper()
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
args = append(args, extra...)
var stdout, stderr bytes.Buffer
return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
}
func lookupRunContractEnv(values map[string]string) func(string) (string, bool) {
return func(name string) (string, bool) {
value, ok := values[name]
return value, ok
func TestRunUsesDeclarationRelativeOverlayRootsAndCapturesSelection(t *testing.T) {
fixture := newStateRunFixture(t)
directory := filepath.Join(filepath.Dir(fixture.configPath), "overlays")
path := filepath.Join(directory, "development.yml")
writeConfigKitFile(t, path, "output: {directory: ./overlay-output}\ndebug: {directory: ./overlay-debug}\n")
result := fixture.run(t, "--debug", "--cache", "bypass", "--overlay", "development")
if result.status != 0 || filepath.Dir(result.report.OutputDirectory) != filepath.Join(directory, "overlay-output") || filepath.Dir(result.report.DebugDirectory) != filepath.Join(directory, "overlay-debug") {
t.Fatalf("overlay roots=%#v", result)
}
content, err := os.ReadFile(filepath.Join(result.report.DebugDirectory, "configuration", "sources.json"))
if err != nil || !strings.Contains(string(content), path) {
t.Fatalf("captured overlay selection=%s %v", content, err)
}
assertAbsent(t, fixture.outputRoot)
assertAbsent(t, fixture.debugRoot)
}
func prependRunContractConfig(t *testing.T, roots stateTestRoots, prefix string) {
t.Helper()
data, err := os.ReadFile(roots.config)
func TestRunCapturesDiscoveredConfigurationPath(t *testing.T) {
fixture := newStateRunFixture(t)
root := filepath.Dir(fixture.configPath)
selected := filepath.Join(root, "notarius.yml")
content, err := os.ReadFile(fixture.configPath)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(roots.config, append([]byte(prefix), data...), 0o600); err != nil {
if err := os.WriteFile(selected, content, 0600); err != nil {
t.Fatal(err)
}
}
func writeRunContractProfiles(t *testing.T, ids ...string) string {
t.Helper()
dir := t.TempDir()
for _, id := range ids {
profile := fmt.Sprintf("profiles:\n %s:\n endpoint: http://127.0.0.1:1/v1\n model: %s-model\n", id, id)
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
t.Fatal(err)
}
t.Chdir(root)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--input-file", fixture.inputPath, "--debug", "--cache", "bypass", "--format", "json"}, &stdout, &stderr, fixture.options)
if code != 0 {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
return dir
}
func registerRunContractValidator(t *testing.T, opts *Options, profiles *[]string) {
t.Helper()
if err := pipeline.RegisterTypedValidatorBuilder(opts.Registries.Validators, stateTestArtifactKind, pipeline.ValidatorSpec{Key: "run-contract-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.TypedValidator[stateTestArtifact], error) {
return runContractValidator{profiles: profiles}, nil
}); err != nil {
t.Fatal(err)
sourceBytes, err := os.ReadFile(filepath.Join(onlyChildDir(t, fixture.debugRoot), "configuration", "sources.json"))
if err != nil || !strings.Contains(string(sourceBytes), selected) {
t.Fatalf("selected configuration=%s %v", sourceBytes, err)
}
if err := opts.Registries.ValidatorChains.Register(pipeline.ValidatorChainMapping{Stage: pipeline.StageExtract, Module: "test/extract", Validators: []pipeline.ModuleBinding{{Module: "run-contract-validator", LLMProfile: "validator-profile"}}}); err != nil {
t.Fatal(err)
}
opts.Catalog = catalogFromRegistries(opts.Registries)
}
type runContractValidator struct {
profiles *[]string
}
func (v runContractValidator) Name() string { return "run-contract-validator" }
func (v runContractValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (v runContractValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[stateTestArtifact]) (contracts.ValidationResult, error) {
*v.profiles = append(*v.profiles, req.LLMProfile)
return contracts.ValidationResult{Approved: true}, nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -11,28 +11,20 @@ import (
)
func TestRunCreatesReusableGroupAccessibleCacheState(t *testing.T) {
previousUmask := syscall.Umask(0o007)
previousUmask := syscall.Umask(0007)
defer syscall.Umask(previousUmask)
roots := newStateTestRoots(t)
harness := newStateTestHarness()
result := runStateTest(t, roots, harness.options(), false, false, "auto")
if result.code != 0 {
t.Fatalf("initial run code=%d stderr=%q", result.code, result.stderr)
fixture := newStateRunFixture(t)
first := fixture.run(t, "--cache", "auto")
if first.status != 0 {
t.Fatalf("initial run=%#v", first)
}
assertGroupAccessibleCacheTree(t, roots.plans)
assertGroupAccessibleCacheTree(t, roots.checkpoints)
extractCalls := harness.extractCalls
result = runStateTest(t, roots, harness.options(), false, true, "auto")
if result.code != 0 {
t.Fatalf("resume run code=%d stderr=%q", result.code, result.stderr)
}
if harness.extractCalls != extractCalls {
t.Fatalf("extract calls after reuse = %d, want %d", harness.extractCalls, extractCalls)
assertGroupAccessibleCacheTree(t, fixture.cacheRoot)
calls := len(fixture.client.requestsSnapshot())
second := fixture.run(t, "--cache", "auto")
if second.status != 0 || len(fixture.client.requestsSnapshot()) != calls {
t.Fatalf("reuse=%#v calls=%d/%d", second, len(fixture.client.requestsSnapshot()), calls)
}
}
func assertGroupAccessibleCacheTree(t *testing.T, root string) {
t.Helper()
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {