Migrate run state and reference contracts to native execution
This commit is contained in:
@@ -2,54 +2,25 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
|
||||
func TestConfigCommandsDoNotAllocateRunState(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"config", "validate", "--config"},
|
||||
{"pipelines", "list", "--config"},
|
||||
{"config", "validate"}, {"config", "show"}, {"pipeline", "list"}, {"pipeline", "validate", "dnd-session"},
|
||||
} {
|
||||
name := strings.Join(args[:2], "-")
|
||||
t.Run(name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") }
|
||||
command := append([]string(nil), args...)
|
||||
command = append(command, roots.config)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(command, &stdout, &stderr, opts)
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
fixture := newStateRunFixture(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
command := append(append([]string(nil), args...), "--config", fixture.configPath)
|
||||
code := RunWithOptions(command, &stdout, &stderr, fixture.options)
|
||||
if code != 0 {
|
||||
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertAbsent(t, fixture.outputRoot)
|
||||
assertAbsent(t, fixture.cacheRoot)
|
||||
assertAbsent(t, fixture.debugRoot)
|
||||
if len(fixture.client.requestsSnapshot()) != 0 {
|
||||
t.Fatal("inspection invoked provider")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func replaceStateTestConfigLine(t *testing.T, path, old, new string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := replaceRequiredOnce(t, string(data), old, new)
|
||||
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func removeStateTestConfigLine(t *testing.T, path, line string) {
|
||||
replaceStateTestConfigLine(t, path, line, "")
|
||||
}
|
||||
|
||||
func assertNoRunState(t *testing.T, roots stateTestRoots) {
|
||||
t.Helper()
|
||||
assertAbsent(t, roots.output)
|
||||
assertAbsent(t, roots.plans)
|
||||
assertAbsent(t, roots.checkpoints)
|
||||
assertAbsent(t, roots.debug)
|
||||
}
|
||||
|
||||
@@ -404,3 +404,59 @@ type exampleOutputIndexEntry struct {
|
||||
func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput {
|
||||
return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)}
|
||||
}
|
||||
|
||||
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
|
||||
catalog := catalogFromRegistries(productionTestComponents(t).registries)
|
||||
sources := map[string]string{
|
||||
"party": "/references/party.txt",
|
||||
"players": "/references/players.txt",
|
||||
"glossary": "/references/glossary.txt",
|
||||
"spell_catalog": "/references/spells.json",
|
||||
}
|
||||
requests := make([]cliReferenceRequest, 0, len(sources))
|
||||
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
|
||||
requests = append(requests, cliReferenceRequest{
|
||||
Selector: mustParseReferenceSelector(t, slot, "--reference"),
|
||||
Source: sources[slot],
|
||||
})
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expand complete D&D references: %v", err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
|
||||
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
|
||||
for _, binding := range overrides {
|
||||
actual[referenceContractBindingLabel(binding)] = binding
|
||||
}
|
||||
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
matched := make(map[string]int, len(sources))
|
||||
for _, target := range targets {
|
||||
for slot, sourcePath := range sources {
|
||||
if _, ok := target.slots[slot]; !ok {
|
||||
continue
|
||||
}
|
||||
matched[slot]++
|
||||
label := targetLabel(target) + "." + slot
|
||||
binding, ok := actual[label]
|
||||
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
for slot := range sources {
|
||||
if matched[slot] < 2 {
|
||||
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
|
||||
}
|
||||
}
|
||||
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
|
||||
t.Fatalf("resolve complete D&D CLI references: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,62 +86,6 @@ func TestReferenceSelectorSpecificityAndFinalOccurrenceChooseConcreteBindings(t
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteDNDSharedCLIReferencesExpandAcrossCompatibleTargets(t *testing.T) {
|
||||
cfg := loadMaintainedExample(t, repositoryPath("examples", "dnd-complete.config.yml"))
|
||||
catalog := catalogFromRegistries(productionTestComponents(t).registries)
|
||||
sources := map[string]string{
|
||||
"party": "/references/party.txt",
|
||||
"players": "/references/players.txt",
|
||||
"glossary": "/references/glossary.txt",
|
||||
"spell_catalog": "/references/spells.json",
|
||||
}
|
||||
requests := make([]cliReferenceRequest, 0, len(sources))
|
||||
for _, slot := range []string{"party", "players", "glossary", "spell_catalog"} {
|
||||
requests = append(requests, cliReferenceRequest{
|
||||
Selector: mustParseReferenceSelector(t, slot, "--reference"),
|
||||
Source: sources[slot],
|
||||
})
|
||||
}
|
||||
overrides, unbinds, err := resolveCLIReferenceRequests(cfg, "dnd-session", nil, catalog, requests, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expand complete D&D references: %v", err)
|
||||
}
|
||||
if len(unbinds) != 0 {
|
||||
t.Fatalf("unbinds = %#v, want none", unbinds)
|
||||
}
|
||||
|
||||
actual := make(map[string]pipeline.ReferenceBinding, len(overrides))
|
||||
for _, binding := range overrides {
|
||||
actual[referenceContractBindingLabel(binding)] = binding
|
||||
}
|
||||
targets, err := selectedReferenceTargets(cfg, "dnd-session", nil, catalog)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
matched := make(map[string]int, len(sources))
|
||||
for _, target := range targets {
|
||||
for slot, sourcePath := range sources {
|
||||
if _, ok := target.slots[slot]; !ok {
|
||||
continue
|
||||
}
|
||||
matched[slot]++
|
||||
label := targetLabel(target) + "." + slot
|
||||
binding, ok := actual[label]
|
||||
if !ok || binding.Source != sourcePath || binding.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("binding %q = %#v, want CLI source %q", label, binding, sourcePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
for slot := range sources {
|
||||
if matched[slot] < 2 {
|
||||
t.Fatalf("reference %q matched %d target(s), want a shared D&D reference", slot, matched[slot])
|
||||
}
|
||||
}
|
||||
if _, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog, ReferenceOverrides: overrides}); err != nil {
|
||||
t.Fatalf("resolve complete D&D CLI references: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSelectorsRejectInvalidOrUnselectedScopesBeforeMaterialization(t *testing.T) {
|
||||
cfg := referenceContractConfig()
|
||||
catalog := referenceContractCatalog(t, true, true)
|
||||
@@ -173,14 +117,14 @@ func TestReferenceSyntaxErrorsReturnTwo(t *testing.T) {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{name: "reference missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference"}},
|
||||
{name: "reference missing selector", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "=path.txt"}},
|
||||
{name: "reference missing separator", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot"}},
|
||||
{name: "reference missing path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "slot="}},
|
||||
{name: "reference excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--reference", "a.b.c.d=path.txt"}},
|
||||
{name: "unbind with path", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "slot=path.txt"}},
|
||||
{name: "unbind excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference", "a.b.c.d"}},
|
||||
{name: "unbind missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input", "input.txt", "--without-reference"}},
|
||||
{name: "reference missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference"}},
|
||||
{name: "reference missing selector", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "=path.txt"}},
|
||||
{name: "reference missing separator", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "slot"}},
|
||||
{name: "reference missing path", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "slot="}},
|
||||
{name: "reference excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--reference", "a.b.c.d=path.txt"}},
|
||||
{name: "unbind with path", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference", "slot=path.txt"}},
|
||||
{name: "unbind excess segments", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference", "a.b.c.d"}},
|
||||
{name: "unbind missing value", args: []string{"run", "demo", "--config", "missing.yml", "--input-file", "input.txt", "--without-reference"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -289,7 +233,7 @@ func TestReferenceMaterializationSeparatesCLIAndConfigPathOrigins(t *testing.T)
|
||||
workingDir := t.TempDir()
|
||||
cfg := referenceContractConfig()
|
||||
configPath := filepath.Join(configDir, "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("version: 5\n"), 0o600); err != nil {
|
||||
if err := os.WriteFile(configPath, []byte("version: 6\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(configDir, "required.txt"), []byte("config reference"), 0o600); err != nil {
|
||||
@@ -348,10 +292,9 @@ func TestPipelineScopedCLIReferenceProtectsGeneratedHandoff(t *testing.T) {
|
||||
Selector: mustParseReferenceSelector(t, "shared", "--reference"),
|
||||
Source: filepath.Join(t.TempDir(), "never-read.json"),
|
||||
}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expand CLI reference: %v", err)
|
||||
if err == nil {
|
||||
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
|
||||
}
|
||||
_, err = cfg.Resolve(config.ResolveInput{PipelineID: "demo", Catalog: catalog, ReferenceOverrides: overrides})
|
||||
if err == nil || !strings.Contains(err.Error(), "conflicting generated and external bindings") || strings.Contains(err.Error(), "never-read.json") {
|
||||
t.Fatalf("resolve error = %v, want generated/external conflict before file access", err)
|
||||
}
|
||||
@@ -482,24 +425,55 @@ func referenceContractCatalog(t *testing.T, includeBetaMerger, includeBetaNormal
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }))
|
||||
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) { return stateTestChunker{}, nil }))
|
||||
register(registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "reference/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
register(registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "chunk-slot"}, {Name: "required-chunk", Required: true}}}, func() (contracts.Chunker, error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecA{}))
|
||||
register(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, referenceContractCodecB{}))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{}, nil }))
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-alpha", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared", AcceptedArtifactKinds: []contracts.ArtifactKind{referenceContractKindBeta}, AcceptedMediaTypes: []string{"application/json"}}, {Name: "alpha-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[referenceContractArtifact], error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
register(pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "reference/extract-beta", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-slot"}, {Name: "required-extract", Required: true}}}, func() (contracts.Extractor[referenceContractArtifact], error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[referenceContractArtifact], error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
if includeBetaMerger {
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{}, nil }))
|
||||
register(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "reference/shared-merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-merge"}, {Name: "required-merge", Required: true}}}, func() (contracts.Merger[referenceContractArtifact], error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
}
|
||||
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
|
||||
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindAlpha, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "alpha-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[referenceContractArtifact], error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
if includeBetaNormalizer {
|
||||
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{}, nil }))
|
||||
register(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "reference/shared-normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: referenceContractKindBeta, ReferenceSlots: []contracts.ReferenceSlot{{Name: "shared"}, {Name: "beta-normalize"}, {Name: "required-normalize", Required: true}}}, func() (contracts.Normalizer[referenceContractArtifact], error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
}
|
||||
register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{}, nil }))
|
||||
register(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "reference/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
||||
t.Fatal("reference selection constructed a module")
|
||||
return nil, nil
|
||||
}))
|
||||
return catalogFromRegistries(registries)
|
||||
}
|
||||
|
||||
type referenceContractArtifact struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type referenceContractCodecB struct{}
|
||||
|
||||
type referenceContractCodecA struct{}
|
||||
@@ -509,16 +483,16 @@ func (referenceContractCodecA) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "reference.alpha", Name: "reference_alpha", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (referenceContractCodecA) MediaType() string { return "application/json" }
|
||||
func (referenceContractCodecA) EncodeCandidate(stateTestArtifact) ([]byte, error) {
|
||||
func (referenceContractCodecA) EncodeCandidate(referenceContractArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecA) Encode(stateTestArtifact) ([]byte, error) {
|
||||
func (referenceContractCodecA) Encode(referenceContractArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecA) Decode([]byte) (stateTestArtifact, error) {
|
||||
return stateTestArtifact{Value: "ok"}, nil
|
||||
func (referenceContractCodecA) Decode([]byte) (referenceContractArtifact, error) {
|
||||
return referenceContractArtifact{Value: "ok"}, nil
|
||||
}
|
||||
func (codec referenceContractCodecA) DecodeCandidate(content []byte) (stateTestArtifact, error) {
|
||||
func (codec referenceContractCodecA) DecodeCandidate(content []byte) (referenceContractArtifact, error) {
|
||||
return codec.Decode(content)
|
||||
}
|
||||
|
||||
@@ -527,16 +501,16 @@ func (referenceContractCodecB) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "reference.beta", Name: "reference_beta", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (referenceContractCodecB) MediaType() string { return "application/json" }
|
||||
func (referenceContractCodecB) EncodeCandidate(stateTestArtifact) ([]byte, error) {
|
||||
func (referenceContractCodecB) EncodeCandidate(referenceContractArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecB) Encode(stateTestArtifact) ([]byte, error) {
|
||||
func (referenceContractCodecB) Encode(referenceContractArtifact) ([]byte, error) {
|
||||
return []byte(`{"value":"ok"}`), nil
|
||||
}
|
||||
func (referenceContractCodecB) Decode([]byte) (stateTestArtifact, error) {
|
||||
return stateTestArtifact{Value: "ok"}, nil
|
||||
func (referenceContractCodecB) Decode([]byte) (referenceContractArtifact, error) {
|
||||
return referenceContractArtifact{Value: "ok"}, nil
|
||||
}
|
||||
func (codec referenceContractCodecB) DecodeCandidate(content []byte) (stateTestArtifact, error) {
|
||||
func (codec referenceContractCodecB) DecodeCandidate(content []byte) (referenceContractArtifact, error) {
|
||||
return codec.Decode(content)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,750 +2,110 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args func(stateTestRoots) []string
|
||||
}{
|
||||
{name: "missing pipeline", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "--config", roots.config, "--input", roots.input}
|
||||
}},
|
||||
{name: "missing input", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config}
|
||||
}},
|
||||
{name: "unknown flag", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--unknown"}
|
||||
}},
|
||||
{name: "blank output directory", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--output-dir", ""}
|
||||
}},
|
||||
{name: "blank debug directory", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""}
|
||||
}},
|
||||
{name: "missing configuration overlay", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--config-overlay"}
|
||||
}},
|
||||
{name: "debug directory without debug", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")}
|
||||
}},
|
||||
{name: "blank session ID", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""}
|
||||
}},
|
||||
{name: "blank LLM profile", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", ""}
|
||||
}},
|
||||
{name: "whitespace LLM profile", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--llm-profile", " \t "}
|
||||
}},
|
||||
{name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input}
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertAbsent(t, roots.output)
|
||||
assertAbsent(t, roots.debug)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputeStepCLIContract(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*testing.T, stateTestRoots)
|
||||
flags []string
|
||||
wantCode int
|
||||
wantOutput string
|
||||
wantError string
|
||||
}{
|
||||
{
|
||||
name: "explicit step",
|
||||
configure: func(t *testing.T, roots stateTestRoots) {
|
||||
replaceStateTestConfigLine(t, roots.config, " artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n", " steps:\n - id: chosen\n artifacts:\n items:\n extract:\n module: test/extract\n merge:\n module: test/merge\n normalize:\n module: test/normalize\n")
|
||||
},
|
||||
flags: []string{"--resume", "--recompute-step", "chosen"},
|
||||
wantCode: 0,
|
||||
wantOutput: "outputs=1",
|
||||
},
|
||||
{name: "implicit default step", flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 0, wantOutput: "outputs=1"},
|
||||
{name: "repeated flag", flags: []string{"--resume", "--recompute-step", "default", "--recompute-step", "default"}, wantCode: 2, wantError: "specified only once"},
|
||||
{name: "empty step", flags: []string{"--resume", "--recompute-step", ""}, wantCode: 2, wantError: "must not be empty"},
|
||||
{name: "unknown step", flags: []string{"--resume", "--recompute-step", "missing"}, wantCode: 1, wantError: "unknown pipeline step"},
|
||||
{name: "without resume", flags: []string{"--recompute-step", "default"}, wantCode: 2, wantError: "requires --resume"},
|
||||
{
|
||||
name: "checkpoint recording disabled",
|
||||
configure: func(t *testing.T, roots stateTestRoots) {
|
||||
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
||||
},
|
||||
flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 1, wantError: "cache.checkpoints.enabled",
|
||||
},
|
||||
{name: "with only", flags: []string{"--resume", "--recompute-step", "default", "--only", "items"}, wantCode: 2, wantError: "cannot be combined with --only"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
if tt.configure != nil {
|
||||
tt.configure(t, roots)
|
||||
}
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
||||
args = append(args, tt.flags...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != tt.wantCode || (tt.wantOutput != "" && !strings.Contains(stdout.String(), tt.wantOutput)) || (tt.wantError != "" && !strings.Contains(stderr.String(), tt.wantError)) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunValidFailuresClassifyAndReportDebug(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args func(stateTestRoots) []string
|
||||
wantError string
|
||||
wantDebug bool
|
||||
}{
|
||||
{name: "unknown pipeline", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "missing", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
||||
}, wantError: `pipeline "missing"`},
|
||||
{name: "unknown lane", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "missing", "--chunk_cache", "bypass", "--debug"}
|
||||
}, wantError: `lane "missing"`, wantDebug: true},
|
||||
{name: "unreadable input", args: func(roots stateTestRoots) []string {
|
||||
return []string{"run", "sample", "--config", roots.config, "--input", filepath.Join(filepath.Dir(roots.input), "unreadable.txt"), "--chunk_cache", "bypass", "--debug"}
|
||||
}, wantError: "read input", wantDebug: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if tt.wantDebug {
|
||||
if !strings.Contains(stderr.String(), "debug=") {
|
||||
t.Fatalf("stderr=%q, want debug path", stderr.String())
|
||||
}
|
||||
onlyChildDir(t, roots.debug)
|
||||
} else {
|
||||
assertAbsent(t, roots.debug)
|
||||
}
|
||||
assertAbsent(t, roots.output)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOnlyExecutesSelectedLanes(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = []byte(replaceRequiredOnce(t, string(data), " output:\n module: test/output\n", " other:\n extract:\n module: test/extract\n output:\n module: test/output\n"))
|
||||
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
harness := newStateTestHarness()
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "items", "--chunk_cache", "bypass"}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
harness.mu.Lock()
|
||||
extractCalls := harness.extractCalls
|
||||
harness.mu.Unlock()
|
||||
if extractCalls != 1 {
|
||||
t.Fatalf("extract calls = %d, want only the selected lane", extractCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
|
||||
t.Run("retired environment roots have no effect", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
|
||||
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LookupEnv = lookupRunContractEnv(map[string]string{
|
||||
"NOTARIUS_OUTPUT_DIR": environmentOutput,
|
||||
"NOTARIUS_DEBUG_DIR": environmentDebug,
|
||||
})
|
||||
result := runWithStateRoots(t, roots, opts, nil)
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertFile(t, filepath.Join(roots.output, filepath.Base(onlyChildDir(t, roots.output)), "result.json"))
|
||||
onlyChildDir(t, roots.debug)
|
||||
assertAbsent(t, environmentOutput)
|
||||
assertAbsent(t, environmentDebug)
|
||||
})
|
||||
|
||||
t.Run("command flags override environment", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
|
||||
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
|
||||
flagOutput := filepath.Join(t.TempDir(), "flag-output")
|
||||
flagDebug := filepath.Join(t.TempDir(), "flag-debug")
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LookupEnv = lookupRunContractEnv(map[string]string{
|
||||
"NOTARIUS_OUTPUT_DIR": environmentOutput,
|
||||
"NOTARIUS_DEBUG_DIR": environmentDebug,
|
||||
})
|
||||
result := runWithStateRoots(t, roots, opts, []string{"--output-dir", flagOutput, "--debug-dir", flagDebug})
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
assertFile(t, filepath.Join(flagOutput, filepath.Base(onlyChildDir(t, flagOutput)), "result.json"))
|
||||
onlyChildDir(t, flagDebug)
|
||||
assertAbsent(t, environmentOutput)
|
||||
assertAbsent(t, environmentDebug)
|
||||
})
|
||||
|
||||
t.Run("built-in roots", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(data)
|
||||
text = replaceRequiredOnce(t, text, fmt.Sprintf("output:\n directory: %q\n", roots.output), "")
|
||||
text = replaceRequiredOnce(t, text, fmt.Sprintf("debug:\n directory: %q\n", roots.debug), "")
|
||||
if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := newStateTestHarness().options()
|
||||
result := runWithStateRoots(t, roots, opts, nil)
|
||||
if result.code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
||||
}
|
||||
configDirectory := filepath.Dir(roots.config)
|
||||
assertFile(t, filepath.Join(configDirectory, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(configDirectory, "notarius-output"))), "result.json"))
|
||||
onlyChildDir(t, filepath.Join(configDirectory, "notarius-debug"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunAppliesConfigurationOverlayAndRecordsSelection(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
overlayDirectory := filepath.Join(filepath.Dir(roots.config), "overlays")
|
||||
overlayOutput := filepath.Join(overlayDirectory, "overlay-output")
|
||||
overlayDebug := filepath.Join(overlayDirectory, "overlay-debug")
|
||||
writeConfigKitFile(t, filepath.Join(overlayDirectory, "development.yml"), "output:\n directory: ./overlay-output\ndebug:\n directory: ./overlay-debug\n")
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--config-overlay", "development",
|
||||
"--input", roots.input, "--chunk_cache", "bypass", "--debug",
|
||||
}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertFile(t, filepath.Join(overlayOutput, filepath.Base(onlyChildDir(t, overlayOutput)), "result.json"))
|
||||
bundle := onlyChildDir(t, overlayDebug)
|
||||
var invocation debugbundle.Invocation
|
||||
readStateTestSummaryJSON(t, bundle, "invocation.json", &invocation)
|
||||
if invocation.ConfigOverlay != "development" {
|
||||
t.Fatalf("config overlay = %q, want development", invocation.ConfigOverlay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRecordsDiscoveredConfigurationPath(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
work := t.TempDir()
|
||||
selected := filepath.Join(work, "notarius.yml")
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(selected, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chdir(work); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Chdir(previous); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
var invocation debugbundle.Invocation
|
||||
readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "invocation.json", &invocation)
|
||||
if invocation.ConfigPath != selected {
|
||||
t.Fatalf("recorded config path = %q, want %q", invocation.ConfigPath, selected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) {
|
||||
for _, overlay := range []string{"", "default"} {
|
||||
if got := effectiveConfigOverlay(overlay); got != "" {
|
||||
t.Fatalf("effectiveConfigOverlay(%q) = %q, want empty", overlay, got)
|
||||
}
|
||||
}
|
||||
if got := effectiveConfigOverlay("development"); got != "development" {
|
||||
t.Fatalf("effectiveConfigOverlay(development) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
|
||||
t.Run("one effective profile reaches the factory and modules", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "override-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
harness := newStateTestHarness()
|
||||
var factoryProfiles []string
|
||||
opts := harness.options()
|
||||
var factoryOverrides []LLMRuntimeOverrides
|
||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryProfiles = append(factoryProfiles, profileID)
|
||||
factoryOverrides = append(factoryOverrides, overrides)
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", " override-profile "}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
|
||||
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
|
||||
}
|
||||
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
|
||||
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
|
||||
}
|
||||
harness.mu.Lock()
|
||||
profiles := append([]string(nil), harness.moduleProfiles...)
|
||||
harness.mu.Unlock()
|
||||
if len(profiles) < 4 {
|
||||
t.Fatalf("module profiles = %#v, want chunk and lane stage requests", profiles)
|
||||
}
|
||||
for _, profile := range profiles {
|
||||
if profile != "override-profile" {
|
||||
t.Fatalf("module profiles = %#v, want override on every request", profiles)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runtime override applies to validators", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "override-profile", "validator-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
harness := newStateTestHarness()
|
||||
var validatorProfiles []string
|
||||
opts := harness.options()
|
||||
registerRunContractValidator(t, &opts, &validatorProfiles)
|
||||
factoryProfiles := []string{}
|
||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryProfiles = append(factoryProfiles, profileID)
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
|
||||
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
|
||||
}
|
||||
if len(validatorProfiles) != 1 || validatorProfiles[0] != "override-profile" {
|
||||
t.Fatalf("validator profiles = %#v, want runtime override", validatorProfiles)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown profile is rejected without factory access", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "override-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
factoryCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "missing-profile"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pipeline default is rejected before factory access", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
profileDir := writeRunContractProfiles(t, "configured-profile")
|
||||
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
||||
replaceStateTestConfigLine(t, roots.config, " sample:\n", " sample:\n llm_profile: missing-profile\n")
|
||||
factoryCalls := 0
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, nil
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
flags []string
|
||||
wantValue string
|
||||
wantSet bool
|
||||
}{
|
||||
{name: "inherit"},
|
||||
{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
|
||||
{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
var got []LLMRuntimeOverrides
|
||||
opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
got = append(got, overrides)
|
||||
return nil, nil, nil
|
||||
}
|
||||
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("factory overrides = %#v, want one call", got)
|
||||
}
|
||||
if !tt.wantSet {
|
||||
if got[0].ReasoningEffort != nil {
|
||||
t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
|
||||
}
|
||||
return
|
||||
}
|
||||
if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
|
||||
t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
flags []string
|
||||
wantError string
|
||||
}{
|
||||
{
|
||||
name: "mutually exclusive controls",
|
||||
flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
|
||||
wantError: "cannot be combined",
|
||||
},
|
||||
{
|
||||
name: "empty replacement",
|
||||
flags: []string{"--reasoning-effort", " "},
|
||||
wantError: "must not be empty",
|
||||
},
|
||||
{
|
||||
name: "duplicate replacement",
|
||||
flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
|
||||
wantError: "may be specified only once",
|
||||
},
|
||||
{
|
||||
name: "missing replacement",
|
||||
flags: []string{"--reasoning-effort"},
|
||||
wantError: "flag needs an argument",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
assertNoRunState(t, roots)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
|
||||
replacement := " focused "
|
||||
cleared := ""
|
||||
states := []struct {
|
||||
name string
|
||||
overrides LLMRuntimeOverrides
|
||||
wantValue string
|
||||
wantSet bool
|
||||
}{
|
||||
{name: "inherit"},
|
||||
{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
|
||||
{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
|
||||
}
|
||||
digests := make(map[string]string, len(states))
|
||||
for _, state := range states {
|
||||
fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
|
||||
var value string
|
||||
var found bool
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == "reasoning_effort_override" {
|
||||
value, found = fingerprint.Value, true
|
||||
}
|
||||
}
|
||||
if found != state.wantSet || (found && value != state.wantValue) {
|
||||
t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
|
||||
}
|
||||
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
|
||||
RawInputDigest: "sha256:input",
|
||||
RuntimeOverrides: fingerprints,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digests[state.name] = identity.Digest
|
||||
}
|
||||
if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
|
||||
t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
|
||||
resolved := pipeline.ResolvedPipeline{
|
||||
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},
|
||||
InputExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Chunk: pipeline.ModuleBinding{LLMProfile: " zeta "},
|
||||
ChunkExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Steps: []pipeline.ResolvedPipelineStep{{
|
||||
ID: "default",
|
||||
ArtifactLanes: []pipeline.ResolvedArtifactLane{{
|
||||
Extract: pipeline.ModuleBinding{LLMProfile: "alpha"},
|
||||
ExtractExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Merge: pipeline.ModuleBinding{LLMProfile: "deterministic-merge"},
|
||||
MergeExecutionClass: contracts.ExecutionClassDeterministic,
|
||||
Normalize: pipeline.ModuleBinding{LLMProfile: " gamma "},
|
||||
NormalizeExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
}},
|
||||
}},
|
||||
ValidatorChains: []pipeline.ResolvedValidatorChain{{Validators: []pipeline.ResolvedValidator{
|
||||
{Binding: pipeline.ModuleBinding{LLMProfile: "deterministic-profile"}, ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||
{Binding: pipeline.ModuleBinding{LLMProfile: "beta"}, ExecutionClass: contracts.ExecutionClassLLMBacked},
|
||||
}}},
|
||||
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
|
||||
OutputExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
}
|
||||
got := effectiveLLMProfileIDs(resolved)
|
||||
want := []string{"alpha", "beta", "gamma", "input-profile", "output-profile", "zeta"}
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("effective profiles = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSessionIDUsesEffectiveValueForPromptRequests(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "derived default"},
|
||||
{name: "explicit trimmed value", args: []string{"--session-id", " explicit-session "}, want: "explicit-session"},
|
||||
func TestRunInvalidSyntaxCreatesNoRunState(t *testing.T) {
|
||||
for _, flags := range [][]string{
|
||||
{"--unknown"},
|
||||
{"--output-dir="},
|
||||
{"--overlay"},
|
||||
{"--llm-temperature=invalid"},
|
||||
{"--input-file="},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
want := tt.want
|
||||
if want == "" {
|
||||
rawInput, err := os.ReadFile(roots.input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, err = resolvePromptSessionID("", "test/input", rawInput)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Run(strings.Join(flags, " "), func(t *testing.T) {
|
||||
fixture := newStateRunFixture(t)
|
||||
result := fixture.run(t, flags...)
|
||||
if result.status != 2 || result.stdout != "" || result.stderr == "" {
|
||||
t.Fatalf("syntax error=%#v", result)
|
||||
}
|
||||
harness := newStateTestHarness()
|
||||
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, tt.args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, harness.options())
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
harness.mu.Lock()
|
||||
sessions := append([]string(nil), harness.sessionIDs...)
|
||||
harness.mu.Unlock()
|
||||
if len(sessions) < 4 {
|
||||
t.Fatalf("session IDs = %#v, want all prompt-facing module requests", sessions)
|
||||
}
|
||||
for _, session := range sessions {
|
||||
if session != want {
|
||||
t.Fatalf("session IDs = %#v, want %q", sessions, want)
|
||||
}
|
||||
}
|
||||
var manifest artifacts.RunManifest
|
||||
readStateTestSummaryJSON(t, onlyChildDir(t, roots.debug), "run-manifest.json", &manifest)
|
||||
if session, ok := manifest.Metadata["session_id"]; !ok || session != want {
|
||||
t.Fatalf("manifest session = %#v, want %q; metadata = %#v", session, want, manifest.Metadata)
|
||||
assertAbsent(t, fixture.outputRoot)
|
||||
assertAbsent(t, fixture.cacheRoot)
|
||||
assertAbsent(t, fixture.debugRoot)
|
||||
if len(fixture.client.requestsSnapshot()) != 0 {
|
||||
t.Fatal("syntax error reached provider")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
|
||||
t.Run("LLM factory", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
opts := newStateTestHarness().options()
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
return nil, nil, errors.New("injected LLM factory failure")
|
||||
func TestRunStateRootsIgnoreRetiredEnvironmentAndHonorNativeFlags(t *testing.T) {
|
||||
fixture := newStateRunFixture(t)
|
||||
environmentOutput, environmentDebug := filepath.Join(t.TempDir(), "environment-output"), filepath.Join(t.TempDir(), "environment-debug")
|
||||
fixture.options.LookupEnv = func(name string) (string, bool) {
|
||||
switch name {
|
||||
case "NOTARIUS_OUTPUT_DIR":
|
||||
return environmentOutput, true
|
||||
case "NOTARIUS_DEBUG_DIR":
|
||||
return environmentDebug, true
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "injected LLM factory failure") || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pipeline preparation", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
data, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = []byte(replaceRequiredOnce(t, string(data), "module: test/extract", "module: test/failing-extract"))
|
||||
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := newStateTestHarness().options()
|
||||
if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) {
|
||||
return nil, errors.New("injected extractor construction failure")
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "injected extractor construction failure") || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
return "", false
|
||||
}
|
||||
first := fixture.run(t, "--debug", "--cache", "bypass")
|
||||
if first.status != 0 {
|
||||
t.Fatalf("configured roots=%#v", first)
|
||||
}
|
||||
assertFile(t, filepath.Join(first.report.OutputDirectory, "lanes", "spells.json"))
|
||||
assertFile(t, filepath.Join(first.report.DebugDirectory, "outcome.json"))
|
||||
flagOutput, flagDebug := filepath.Join(t.TempDir(), "flag-output"), filepath.Join(t.TempDir(), "flag-debug")
|
||||
second := fixture.run(t, "--debug", "--cache", "bypass", "--output-dir", flagOutput, "--debug-dir", flagDebug)
|
||||
if second.status != 0 || !strings.HasPrefix(second.report.OutputDirectory, flagOutput+string(os.PathSeparator)) || !strings.HasPrefix(second.report.DebugDirectory, flagDebug+string(os.PathSeparator)) {
|
||||
t.Fatalf("flag roots=%#v", second)
|
||||
}
|
||||
assertAbsent(t, environmentOutput)
|
||||
assertAbsent(t, environmentDebug)
|
||||
}
|
||||
|
||||
func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.includeWarnings = true
|
||||
harness.includeWarningFile = true
|
||||
harness.chunkDiagnostics = []contracts.ProducerDiagnostic{stateTestDiagnostic("chunk", "contract-warning", "warning retained")}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning group(s), 1 occurrence(s)") || !strings.Contains(stderr.String(), "warnings.json") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
|
||||
output, err := os.ReadFile(outputPath)
|
||||
if err != nil || !strings.Contains(string(output), "contract-warning") {
|
||||
t.Fatalf("durable output = %q, %v", output, err)
|
||||
}
|
||||
assertFile(t, filepath.Join(filepath.Dir(outputPath), "warnings.json"))
|
||||
bundle := onlyChildDir(t, roots.debug)
|
||||
var diagnostics contracts.DiagnosticCollection
|
||||
readStateTestSummaryJSON(t, bundle, "final-diagnostics.json", &diagnostics)
|
||||
if len(diagnostics.Groups) != 1 || diagnostics.Groups[0].ReasonCode != "contract-warning" {
|
||||
t.Fatalf("debug diagnostics = %#v", diagnostics)
|
||||
func TestRunDebugDirectoryAloneDoesNotEnableCapture(t *testing.T) {
|
||||
fixture := newStateRunFixture(t)
|
||||
debug := filepath.Join(t.TempDir(), "explicit-debug")
|
||||
result := fixture.run(t, "--cache", "bypass", "--debug-dir", debug)
|
||||
if result.status != 0 || result.report.DebugDirectory != "" {
|
||||
t.Fatalf("run=%#v", result)
|
||||
}
|
||||
assertAbsent(t, debug)
|
||||
assertAbsent(t, fixture.debugRoot)
|
||||
}
|
||||
|
||||
func runWithStateRoots(t *testing.T, roots stateTestRoots, opts Options, extra []string) stateTestResult {
|
||||
t.Helper()
|
||||
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
|
||||
args = append(args, extra...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
|
||||
}
|
||||
|
||||
func lookupRunContractEnv(values map[string]string) func(string) (string, bool) {
|
||||
return func(name string) (string, bool) {
|
||||
value, ok := values[name]
|
||||
return value, ok
|
||||
func TestRunUsesDeclarationRelativeOverlayRootsAndCapturesSelection(t *testing.T) {
|
||||
fixture := newStateRunFixture(t)
|
||||
directory := filepath.Join(filepath.Dir(fixture.configPath), "overlays")
|
||||
path := filepath.Join(directory, "development.yml")
|
||||
writeConfigKitFile(t, path, "output: {directory: ./overlay-output}\ndebug: {directory: ./overlay-debug}\n")
|
||||
result := fixture.run(t, "--debug", "--cache", "bypass", "--overlay", "development")
|
||||
if result.status != 0 || filepath.Dir(result.report.OutputDirectory) != filepath.Join(directory, "overlay-output") || filepath.Dir(result.report.DebugDirectory) != filepath.Join(directory, "overlay-debug") {
|
||||
t.Fatalf("overlay roots=%#v", result)
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(result.report.DebugDirectory, "configuration", "sources.json"))
|
||||
if err != nil || !strings.Contains(string(content), path) {
|
||||
t.Fatalf("captured overlay selection=%s %v", content, err)
|
||||
}
|
||||
assertAbsent(t, fixture.outputRoot)
|
||||
assertAbsent(t, fixture.debugRoot)
|
||||
}
|
||||
|
||||
func prependRunContractConfig(t *testing.T, roots stateTestRoots, prefix string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(roots.config)
|
||||
func TestRunCapturesDiscoveredConfigurationPath(t *testing.T) {
|
||||
fixture := newStateRunFixture(t)
|
||||
root := filepath.Dir(fixture.configPath)
|
||||
selected := filepath.Join(root, "notarius.yml")
|
||||
content, err := os.ReadFile(fixture.configPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(roots.config, append([]byte(prefix), data...), 0o600); err != nil {
|
||||
if err := os.WriteFile(selected, content, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeRunContractProfiles(t *testing.T, ids ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, id := range ids {
|
||||
profile := fmt.Sprintf("profiles:\n %s:\n endpoint: http://127.0.0.1:1/v1\n model: %s-model\n", id, id)
|
||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Chdir(root)
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--input-file", fixture.inputPath, "--debug", "--cache", "bypass", "--format", "json"}, &stdout, &stderr, fixture.options)
|
||||
if code != 0 {
|
||||
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func registerRunContractValidator(t *testing.T, opts *Options, profiles *[]string) {
|
||||
t.Helper()
|
||||
if err := pipeline.RegisterTypedValidatorBuilder(opts.Registries.Validators, stateTestArtifactKind, pipeline.ValidatorSpec{Key: "run-contract-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.TypedValidator[stateTestArtifact], error) {
|
||||
return runContractValidator{profiles: profiles}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
sourceBytes, err := os.ReadFile(filepath.Join(onlyChildDir(t, fixture.debugRoot), "configuration", "sources.json"))
|
||||
if err != nil || !strings.Contains(string(sourceBytes), selected) {
|
||||
t.Fatalf("selected configuration=%s %v", sourceBytes, err)
|
||||
}
|
||||
if err := opts.Registries.ValidatorChains.Register(pipeline.ValidatorChainMapping{Stage: pipeline.StageExtract, Module: "test/extract", Validators: []pipeline.ModuleBinding{{Module: "run-contract-validator", LLMProfile: "validator-profile"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
}
|
||||
|
||||
type runContractValidator struct {
|
||||
profiles *[]string
|
||||
}
|
||||
|
||||
func (v runContractValidator) Name() string { return "run-contract-validator" }
|
||||
|
||||
func (v runContractValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (v runContractValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[stateTestArtifact]) (contracts.ValidationResult, error) {
|
||||
*v.profiles = append(*v.profiles, req.LLMProfile)
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,28 +11,20 @@ import (
|
||||
)
|
||||
|
||||
func TestRunCreatesReusableGroupAccessibleCacheState(t *testing.T) {
|
||||
previousUmask := syscall.Umask(0o007)
|
||||
previousUmask := syscall.Umask(0007)
|
||||
defer syscall.Umask(previousUmask)
|
||||
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
result := runStateTest(t, roots, harness.options(), false, false, "auto")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("initial run code=%d stderr=%q", result.code, result.stderr)
|
||||
fixture := newStateRunFixture(t)
|
||||
first := fixture.run(t, "--cache", "auto")
|
||||
if first.status != 0 {
|
||||
t.Fatalf("initial run=%#v", first)
|
||||
}
|
||||
assertGroupAccessibleCacheTree(t, roots.plans)
|
||||
assertGroupAccessibleCacheTree(t, roots.checkpoints)
|
||||
|
||||
extractCalls := harness.extractCalls
|
||||
result = runStateTest(t, roots, harness.options(), false, true, "auto")
|
||||
if result.code != 0 {
|
||||
t.Fatalf("resume run code=%d stderr=%q", result.code, result.stderr)
|
||||
}
|
||||
if harness.extractCalls != extractCalls {
|
||||
t.Fatalf("extract calls after reuse = %d, want %d", harness.extractCalls, extractCalls)
|
||||
assertGroupAccessibleCacheTree(t, fixture.cacheRoot)
|
||||
calls := len(fixture.client.requestsSnapshot())
|
||||
second := fixture.run(t, "--cache", "auto")
|
||||
if second.status != 0 || len(fixture.client.requestsSnapshot()) != calls {
|
||||
t.Fatalf("reuse=%#v calls=%d/%d", second, len(fixture.client.requestsSnapshot()), calls)
|
||||
}
|
||||
}
|
||||
|
||||
func assertGroupAccessibleCacheTree(t *testing.T, root string) {
|
||||
t.Helper()
|
||||
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
|
||||
Reference in New Issue
Block a user