Migrate configuration and command tests to native inspection

This commit is contained in:
2026-11-22 05:16:24 +00:00
parent 21bcb4eb3b
commit 5cfcf10a8b
6 changed files with 339 additions and 713 deletions

View File

@@ -6,48 +6,27 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
)
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
tests := [][]string{nil, {"help"}, {"--help"}, {"-h"}}
for _, args := range tests {
name := "no arguments"
if len(args) > 0 {
name = args[0]
}
t.Run(name, func(t *testing.T) {
func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
for _, args := range [][]string{
{"unknown"},
{"pipeline", "unknown"},
{"config", "validate", "--unknown"},
{"run", "demo", "--input-file", "missing", "--cache", "invalid"},
{"pipeline", "validate"},
{"run", "demo", "--input-file", "missing", "--only", "spells"},
} {
t.Run(strings.Join(args, " "), func(t *testing.T) {
var stdout, stderr bytes.Buffer
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
if code != 0 || !strings.Contains(stdout.String(), "Usage:") || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
tests := []struct {
name string
args []string
want string
}{
{name: "unknown command", args: []string{"unknown"}, want: "unknown command"},
{name: "missing config subcommand", args: []string{"config"}, want: "config requires a subcommand"},
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
code := RunWithOptions(tt.args, &stdout, &stderr, commandContractOptions(t))
if code != 2 || !strings.Contains(stderr.String(), tt.want) || stdout.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
@@ -56,25 +35,21 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
func TestCommandVersionOutput(t *testing.T) {
previous := buildinfo.Override
t.Cleanup(func() { buildinfo.Override = previous })
tests := []struct {
name string
override string
wantCode int
wantStdout string
wantStderr string
for _, test := range []struct {
name, override string
status int
fragment string
}{
{name: "development", wantStdout: "notarius development\n"},
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
buildinfo.Override = tt.override
{name: "development", fragment: "development"},
{name: "release", override: "v1.2.3", fragment: "v1.2.3"},
{name: "invalid release identity", override: "release", status: 1},
} {
t.Run(test.name, func(t *testing.T) {
buildinfo.Override = test.override
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
code := RunWithOptions([]string{"version"}, &stdout, &stderr, Options{})
if code != test.status || test.status == 0 && (!strings.Contains(stdout.String(), test.fragment) || stderr.Len() != 0) || test.status != 0 && (stdout.Len() != 0 || stderr.Len() == 0) {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
@@ -83,143 +58,104 @@ func TestCommandVersionOutput(t *testing.T) {
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
explicit := writeCommandConfig(t, "explicit", "alpha")
environment := writeCommandConfig(t, "environment", "beta")
lookup := func(name string) (string, bool) {
if name == "NOTARIUS_CONFIG" {
return environment, true
}
return "", false
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"pipelines", "list", "--config", explicit}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
if code != 0 || stdout.String() != "alpha\nexplicit\n" || stderr.Len() != 0 {
t.Fatalf("explicit config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"pipelines", "list"}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
if code != 0 || stdout.String() != "beta\nenvironment\n" || stderr.Len() != 0 {
t.Fatalf("environment config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
lookup := func(name string) (string, bool) { return environment, name == "NOTARIUS_CONFIG" }
for _, test := range []struct {
name string
flags []string
want []string
}{
{name: "explicit", flags: []string{"--config", explicit}, want: []string{"alpha", "explicit"}},
{name: "environment", want: []string{"beta", "environment"}},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
args := append([]string{"pipeline", "list", "--format", "json"}, test.flags...)
code := RunWithOptions(args, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
var ids []string
if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil {
t.Fatal(err)
}
if code != 0 || !reflect.DeepEqual(ids, test.want) {
t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String())
}
})
}
}
func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) {
missing := filepath.Join(t.TempDir(), "missing.yml")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", missing}, &stdout, &stderr, commandContractOptions(t))
if code != 1 || !strings.Contains(stderr.String(), missing) || stdout.Len() != 0 {
t.Fatalf("missing config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
malformed := filepath.Join(t.TempDir(), "malformed.yml")
if err := os.WriteFile(malformed, []byte("version: [\n"), 0o600); err != nil {
func TestConfigLoadingFailuresRetainPathContext(t *testing.T) {
root := t.TempDir()
missing := filepath.Join(root, "missing.yml")
malformed := filepath.Join(root, "malformed.yml")
if err := os.WriteFile(malformed, []byte("version: [\n"), 0600); err != nil {
t.Fatal(err)
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"config", "validate", "--config", malformed}, &stdout, &stderr, commandContractOptions(t))
if code != 1 || !strings.Contains(stderr.String(), "configuration contains invalid YAML") || stdout.Len() != 0 {
t.Fatalf("malformed config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
for _, path := range []string{missing, malformed} {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t))
if code == 0 || !strings.Contains(stderr.String(), path) {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
}
func TestConfigValidateResolvesPipelineAndChecksSelection(t *testing.T) {
configPath := writeResolvableCommandConfig(t)
options := commandContractOptions(t)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells"}, &stdout, &stderr, options)
if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 {
t.Fatalf("valid resolution: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "missing"}, &stdout, &stderr, options)
if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") {
t.Fatalf("unknown pipeline: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "missing"}, &stdout, &stderr, options)
if code != 1 || !strings.Contains(stderr.String(), "lane \"missing\"") {
t.Fatalf("unknown lane: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--only", "spells"}, &stdout, &stderr, options)
if code != 2 || !strings.Contains(stderr.String(), "--only requires --pipeline") {
t.Fatalf("missing pipeline for only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells,,other"}, &stdout, &stderr, options)
if code != 2 || !strings.Contains(stderr.String(), "--only must contain") {
t.Fatalf("malformed only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
func TestPipelineValidationSelectsConfiguredCompositionOffline(t *testing.T) {
path := writeResolvableCommandConfig(t)
for _, test := range []struct {
id string
status int
}{
{id: "demo"}, {id: "missing", status: 2},
} {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"pipeline", "validate", test.id, "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t))
if code != test.status {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if code == 0 {
var report struct {
Pipeline string `json:"pipeline"`
Valid bool `json:"valid"`
}
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatal(err)
}
if report.Pipeline != test.id || !report.Valid {
t.Fatalf("validation report=%#v", report)
}
} else if !strings.Contains(stderr.String(), test.id) {
t.Fatalf("missing pipeline context: %s", stderr.String())
}
}
}
func TestConfigValidatePipelineDefaultProfileIsOffline(t *testing.T) {
configPath := writeCommandConfigContent(t, `version: 5
func TestPipelineValidationDefaultProfileNeedsNoCredentialsOrProvider(t *testing.T) {
path := writeCommandConfigContent(t, `version: 6
pipelines:
demo:
llm_profile: dnd-extraction
input:
module: seriatim
llm_profile: notarius/dnd-extraction
input: {module: seriatim}
artifacts:
spells:
extract:
module: dnd/spells
extract: {module: dnd/spells}
`)
client := &diagnosticClient{}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo"}, &stdout, &stderr, Options{})
if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
code := RunWithOptions([]string{"pipeline", "validate", "demo", "--config", path}, &stdout, &stderr, Options{LLMClient: client, LookupEnv: emptyLookup})
if code != 0 || client.calls.Load() != 0 {
t.Fatalf("status=%d calls=%d stderr=%q", code, client.calls.Load(), stderr.String())
}
}
func TestPipelinesListSortsNormalizedIDsInTextAndJSON(t *testing.T) {
configPath := writeCommandConfig(t, " zeta ", "alpha")
options := commandContractOptions(t)
func TestPipelineListSortsNormalizedIDs(t *testing.T) {
path := writeCommandConfig(t, " zeta ", "alpha")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"pipelines", "list", "--config", configPath}, &stdout, &stderr, options)
if code != 0 || stdout.String() != "alpha\nzeta\n" || stderr.Len() != 0 {
t.Fatalf("text list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
code := RunWithOptions([]string{"pipeline", "list", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t))
var ids []string
if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil {
t.Fatal(err)
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"pipelines", "list", "--config", configPath, "--json"}, &stdout, &stderr, options)
var payload struct {
Pipelines []string `json:"pipelines"`
}
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil {
t.Fatalf("JSON list = %q: %v", stdout.String(), err)
}
if code != 0 || len(payload.Pipelines) != 2 || payload.Pipelines[0] != "alpha" || payload.Pipelines[1] != "zeta" || stderr.Len() != 0 {
t.Fatalf("JSON list: code=%d payload=%#v stderr=%q", code, payload, stderr.String())
}
}
func TestRemovedStructuralFlagsAndRuntimeFailuresKeepExitClasses(t *testing.T) {
configPath := writeResolvableCommandConfig(t)
options := commandContractOptions(t)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"run", "demo", "--input", "missing-input", "--config", configPath, "--diagnostics-dir", t.TempDir()}, &stdout, &stderr, options)
if code != 2 || !strings.Contains(stderr.String(), "flag provided but not defined") {
t.Fatalf("removed flag: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
code = RunWithOptions([]string{"run", "missing", "--input", "missing-input", "--config", configPath, "--chunk_cache", "bypass"}, &stdout, &stderr, options)
if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") || stdout.Len() != 0 {
t.Fatalf("valid-runtime failure: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
if code != 0 || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) {
t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String())
}
}
@@ -229,42 +165,29 @@ func commandContractOptions(t *testing.T) Options {
func commandContractOptionsWithLookup(t *testing.T, lookup func(string) (string, bool)) Options {
t.Helper()
components, err := newProductionComponents()
if err != nil {
t.Fatal(err)
}
return Options{
Catalog: catalogFromRegistries(components.registries),
Registries: components.registries,
LookupEnv: lookup,
}
options := productionCLIOptions(t)
options.LookupEnv = lookup
return options
}
func writeCommandConfig(t *testing.T, firstID, secondID string) string {
t.Helper()
content := fmt.Sprintf("version: 5\npipelines:\n %q:\n input:\n module: seriatim\n %q:\n input:\n module: seriatim\n", firstID, secondID)
return writeCommandConfigContent(t, content)
return writeCommandConfigContent(t, fmt.Sprintf("version: 6\npipelines:\n %q:\n input: {module: seriatim}\n %q:\n input: {module: seriatim}\n", firstID, secondID))
}
func writeResolvableCommandConfig(t *testing.T) string {
t.Helper()
return writeCommandConfigContent(t, `version: 5
return writeCommandConfigContent(t, `version: 6
pipelines:
demo:
input:
module: seriatim
input: {module: seriatim}
artifacts:
spells:
extract:
module: dnd/spells
extract: {module: dnd/spells}
`)
}
func writeCommandConfigContent(t *testing.T, content string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return path
return writeProductionContractConfig(t, content)
}

View File

@@ -2,523 +2,206 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestExplainRejectsMalformedPointerBeforeLoadingEitherScope(t *testing.T) {
for _, scope := range []string{"notarius", "promptkit"} {
t.Run(scope, func(t *testing.T) {
var stdout, stderr strings.Builder
options := commandContractOptions(t)
options.LookupEnv = func(string) (string, bool) {
t.Fatal("configuration discovery ran for invalid syntax")
return "", false
}
code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", filepath.Join(t.TempDir(), "missing.yml"), "/profiles/~2"}, &stdout, &stderr, options)
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "canonical JSON Pointer") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
func TestConfigKitCommandsLoadImportsAndOverlays(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
writeConfigKitFile(t, path, "version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: input}\n")
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") {
t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String())
}
for _, selection := range [][]string{nil, {"--config-overlay", ""}, {"--config-overlay", "default"}} {
stdout.Reset()
stderr.Reset()
args := append([]string{"pipelines", "list", "--config", path}, selection...)
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\n" || stderr.Len() != 0 {
t.Fatalf("base list %q = %d, %q, %q", selection, code, stdout.String(), stderr.String())
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: seriatim}\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: seriatim}\n")
writeConfigKitFile(t, path, "version: 6\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: seriatim}\n")
for _, test := range []struct {
flags []string
want []string
}{
{want: []string{"base", "imported"}},
{flags: []string{"--overlay", "development"}, want: []string{"base", "imported", "overlaid"}},
} {
var stdout, stderr bytes.Buffer
args := append([]string{"pipeline", "list", "--config", path, "--format", "json"}, test.flags...)
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
var ids []string
if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil {
t.Fatal(err)
}
if code != 0 || !reflect.DeepEqual(ids, test.want) {
t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String())
}
}
}
func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) {
func TestPipelineInheritanceCommandsInspectDerivedPipeline(t *testing.T) {
t.Parallel()
path := writeCommandConfigContent(t, `version: 5
path := writeCommandConfigContent(t, `version: 6
pipelines:
" parent ":
input:
module: seriatim
input: {module: seriatim}
artifacts:
spells:
extract:
module: dnd/spells
structured_output_repair_attempts: 2
extract: {module: dnd/spells}
max_additional_turns: 2
child:
extends: " parent "
structured_output_repair_attempts: 0
max_additional_turns: 0
`)
options := commandContractOptions(t)
for _, args := range [][]string{
{"pipeline", "validate", "child"},
{"config", "explain", "--path", "/pipelines/child/input/module"},
} {
var stdout, stderr bytes.Buffer
args = append(args, "--config", path)
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
if code != 0 || stdout.Len() == 0 {
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
}
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 {
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
code := RunWithOptions([]string{"config", "show", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t))
var report struct {
Pipelines map[string]struct {
Input struct {
Module string `json:"module"`
} `json:"input"`
Turns int `json:"max_additional_turns"`
} `json:"pipelines"`
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 {
t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String())
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatal(err)
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 {
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 {
t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String())
child, ok := report.Pipelines["child"]
if code != 0 || !ok || child.Input.Module != "seriatim" || child.Turns != 0 {
t.Fatalf("status=%d inherited=%#v stderr=%q", code, report, stderr.String())
}
}
func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) {
func TestPipelineInheritanceFailuresRemainLoadFailures(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name string
content string
args []string
want string
}{
{
name: "missing parent",
content: `version: 5
pipelines:
child:
extends: absent
`,
args: []string{"config", "validate", "--pipeline", "child"},
want: "inheritance.missing",
},
{
name: "cycle",
content: `version: 5
pipelines:
first:
extends: second
second:
extends: first
`,
args: []string{"config", "validate", "--pipeline", "first"},
want: "cycle",
},
{
name: "invalid directive type",
content: `version: 5
pipelines:
child:
extends: 7
`,
args: []string{"config", "validate", "--pipeline", "child"},
want: "extends",
},
{
name: "unselected missing parent",
content: `version: 5
pipelines:
parent:
input:
module: seriatim
child:
extends: parent
unrelated:
extends: absent
`,
args: []string{"config", "validate", "--pipeline", "child"},
want: "inheritance.missing",
},
{
name: "exact sibling mismatch",
content: `version: 5
pipelines:
parent:
input:
module: seriatim
child:
extends: "parent "
`,
args: []string{"pipelines", "list"},
want: "inheritance.missing",
},
for _, test := range []struct{ name, content, fragment string }{
{"missing parent", "pipelines: {child: {extends: absent}}\n", "inheritance.missing"},
{"cycle", "pipelines: {first: {extends: second}, second: {extends: first}}\n", "cycle"},
{"invalid directive", "pipelines: {child: {extends: 7}}\n", "extends"},
{"unselected missing parent", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: parent}, unrelated: {extends: absent}}\n", "inheritance.missing"},
{"exact sibling mismatch", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: 'parent '}}\n", "inheritance.missing"},
} {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
path := writeCommandConfigContent(t, test.content)
args := append([]string{}, test.args...)
args = append(args, "--config", path)
path := writeCommandConfigContent(t, "version: 6\n"+test.content)
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t))
if code == 0 || !strings.Contains(stderr.String(), test.fragment) {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
t.Parallel()
for _, args := range [][]string{
{"run", "sample", "--config-profile=development"},
{"config", "validate", "--config-profile=development"},
{"config", "effective", "--config-profile=development"},
{"config", "explain", "--config-profile=development", ""},
{"pipelines", "list", "--config-profile=development"},
} {
t.Run(strings.Join(args[:2], " "), func(t *testing.T) {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "not defined") {
t.Fatalf("args=%q code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
})
}
}
func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "notarius.yml")
writeConfigKitFile(t, path, "version: 4\n")
var stdout, stderr bytes.Buffer
if code := runPipelinesList([]string{"--config", path}, &stdout, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "version") {
t.Fatalf("list = %d, %q", code, stderr.String())
}
}
func TestConfigurationInspectionCommandsRenderSafeConfigKitViews(t *testing.T) {
func TestConfigurationScopesUseSelectedSourcesAndNativeSecretMarks(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, path, `version: 5
concurrency:
total_llm: 4
pipelines:
"0":
input: {module: input}
`)
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency:\n total_llm: 3\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "retired.yml"), "pipelines: null\n")
providerCalled := false
opts := commandContractOptions(t)
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
providerCalled = true
return nil, nil, errors.New("LLM client must not be constructed")
}
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 {
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
}
if providerCalled {
t.Fatal("config effective constructed an LLM client")
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 {
t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, ""}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: ""`) || stderr.Len() != 0 {
t.Fatalf("explain root = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 {
t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String())
}
}
func TestConfigExplainRejectsInvalidAndUnknownPointers(t *testing.T) {
t.Parallel()
path := writeCommandConfig(t, "session", "other")
for _, test := range []struct {
name string
pointer string
wantCode int
}{
{name: "missing", wantCode: 2},
{name: "malformed escape", pointer: "/pipelines~2session", wantCode: 2},
{name: "unknown", pointer: "/unknown", wantCode: 1},
{name: "invalid sequence index", pointer: "/pipelines/session/steps/+0", wantCode: 1},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
args := []string{"config", "explain", "--config", path}
if test.name != "missing" {
args = append(args, test.pointer)
}
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != test.wantCode || stdout.Len() != 0 || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
})
}
}
func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
t.Parallel()
path := writeCommandConfig(t, "session", "other")
var stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--config", path}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
t.Fatalf("effective writer failure = %d, %q", code, stderr.String())
}
}
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
t.Parallel()
root := t.TempDir()
configPath := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
base:
endpoint: https://example.invalid/v1
model: base-model
extra_params:
Authorization: base-auth-secret
ordinary: visible-value
`)
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
writeConfigKitFile(t, path, "version: 6\nconcurrency: {workers: 4}\npipelines: {demo: {input: {module: seriatim}}}\n")
writeConfigKitFile(t, filepath.Join(root, "prompts.yml"), `config:
imports:
- path: base.yml
profiles:
"escaped/id":
model: imported-model
dnd-extraction:
model: imported-override
- path: profiles/base.yml
required: true
`)
assets, err := productionPromptAssets()
if err != nil {
t.Fatal(err)
}
factoryCalls := 0
opts := commandContractOptions(t)
opts.promptKitAssets = assets
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, errors.New("inspection must not construct an LLM client")
}
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
}
defaultReport := stdout.String()
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
}
if factoryCalls != 0 {
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
}
}
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
t.Parallel()
path := writeCommandConfig(t, "session", "other")
opts := commandContractOptions(t)
for _, test := range []struct {
name string
args []string
want int
}{
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
}
})
}
var stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
}
}
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) {
t.Parallel()
root := t.TempDir()
profilePath := filepath.Join(root, "profiles.yml")
writeConfigKitFile(t, profilePath, `backends:
managed:
endpoint: https://provider.example.invalid/v1
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
profiles:
inherited:
backend: managed
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
- id: base
endpoint: https://example.invalid/v1
model: inherited-model
temperature: 0.4
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
api_key_env: NOTARIUS_TEST_KEY_NOT_REQUIRED_FOR_INSPECTION
api_key_required: true
extra_params:
Authorization: private-authorization
ordinary: visible-value
selected:
extends: inherited
extra_params: {token: private-test-token, ordinary: visible-value}
- id: selected
extends: {id: base}
temperature: 0
extra_params: null
endpoint-only:
endpoint: https://endpoint.example.invalid/v1
model: endpoint-model
api_key_required: false
" literal profile ":
endpoint: https://literal.example.invalid/v1
model: literal-model
`)
configPath := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n")
assets, err := productionPromptAssets()
if err != nil {
t.Fatal(err)
}
lookupCalls := 0
factoryCalls := 0
opts := commandContractOptions(t)
opts.promptKitAssets = assets
opts.LookupEnv = func(string) (string, bool) {
lookupCalls++
return "", false
}
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, errors.New("profile inspection must not construct an LLM client")
}
type profileReport struct {
ProfileID string `json:"profile_id"`
APIKeyRequired bool `json:"api_key_required"`
Target struct {
BackendID string `json:"backend_id"`
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
} `json:"effective_model_params"`
}
run := func(profileID string) (int, string, string, profileReport) {
t.Helper()
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency: {workers: 3}\n")
client := &diagnosticClient{}
options := commandContractOptions(t)
options.LLMClient = client
baseIndex := -1
for _, scope := range []string{"app", "promptkit"} {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts)
var report profileReport
if code == 0 {
if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 {
t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String())
code := RunWithOptions([]string{"config", "show", "--scope", scope, "--config", path, "--overlay", "development", "--format", "json"}, &stdout, &stderr, options)
if code != 0 || !json.Valid(stdout.Bytes()) || !strings.Contains(stdout.String(), "inherited-model") || !strings.Contains(stdout.String(), "visible-value") || strings.Contains(stdout.String(), "private-test-token") {
t.Fatalf("%s status=%d stdout=%q stderr=%q", scope, code, stdout.String(), stderr.String())
}
var report map[string]any
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatal(err)
}
if scope == "app" {
controls, ok := report["concurrency"].(map[string]any)
if !ok || controls["workers"] != float64(3) {
t.Fatalf("overlay missing: %#v", report)
}
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatalf("decode profile JSON %q: %v", stdout.String(), err)
} else {
if report["pipelines"] != nil {
t.Fatalf("focused scope exposed application root: %#v", report)
}
profiles, _ := report["profiles"].([]any)
for index, raw := range profiles {
profile, _ := raw.(map[string]any)
if profile["id"] == "base" {
baseIndex = index
}
}
}
return code, stdout.String(), stderr.String(), report
}
code, output, diagnostic, selected := run("selected")
if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil {
t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected)
if baseIndex < 0 {
t.Fatal("focused configuration omitted the imported profile")
}
code, output, diagnostic, inherited := run("inherited")
if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") {
t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited)
for _, args := range [][]string{
{"config", "sources", "--scope", "promptkit"},
{"config", "explain", "--scope", "promptkit", "--path", fmt.Sprintf("/promptkit/profiles/%d/model", baseIndex)},
} {
var stdout, stderr bytes.Buffer
code := RunWithOptions(append(args, "--config", path), &stdout, &stderr, options)
if code != 0 || !strings.Contains(stdout.String(), "base.yml") {
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
}
code, output, diagnostic, endpointOnly := run("endpoint-only")
if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" {
t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly)
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"promptkit", "show", "profile", "selected", "--config", path, "--format", "json"}, &stdout, &stderr, options)
if code != 0 || !strings.Contains(stdout.String(), "inherited-model") || strings.Contains(stdout.String(), "private-test-token") {
t.Fatalf("profile status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
code, output, diagnostic, literal := run(" literal profile ")
if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" {
t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal)
}
if lookupCalls != 0 || factoryCalls != 0 {
t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls)
if client.calls.Load() != 0 {
t.Fatal("configuration/profile inspection invoked provider")
}
}
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) {
func TestConfigurationCommandsRetainNativeInputAndPresentationFailures(t *testing.T) {
t.Parallel()
configPath := writeCommandConfig(t, "session", "other")
for _, test := range []struct {
name string
args []string
want int
}{
{name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2},
{name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2},
{name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2},
{name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
}
})
for _, scope := range []string{"app", "promptkit"} {
var stdout, stderr bytes.Buffer
missing := filepath.Join(t.TempDir(), "missing.yml")
code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", missing, "--path", "/profiles/~2"}, &stdout, &stderr, commandContractOptions(t))
if code != 2 || stdout.Len() != 0 || strings.Contains(stderr.String(), missing) {
t.Fatalf("invalid path reached loading: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n")
configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
var stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") {
t.Fatalf("profile writer failure = %d, %q", code, stderr.String())
path := writeCommandConfig(t, "session", "other")
for _, args := range [][]string{
{"config", "show"}, {"config", "show", "--scope", "promptkit"}, {"promptkit", "show", "profile", "notarius/dnd-extraction"},
} {
var stderr bytes.Buffer
code := RunWithOptions(append(args, "--config", path), failingInspectionWriter{}, &stderr, commandContractOptions(t))
if code != 1 {
t.Fatalf("writer failure %v: status=%d stderr=%q", args, code, stderr.String())
}
}
}
type failingInspectionWriter struct{}
func (failingInspectionWriter) Write([]byte) (int, error) {
return 0, io.ErrClosedPipe
}
func (failingInspectionWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }

View File

@@ -2,6 +2,7 @@ package cli
import (
"bytes"
"context"
"os"
"path/filepath"
"runtime"
@@ -9,7 +10,10 @@ import (
"strings"
"testing"
configkit "go.maximumdirect.net/configkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
)
func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
@@ -22,41 +26,44 @@ func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
args := []string{"config", "validate", "--config", main}
if overlay {
fragment = filepath.Join(root, "overlays", "development.yml")
writeConfigKitFile(t, main, "version: 5\n")
args = append(args, "--config-overlay", "development")
writeConfigKitFile(t, main, "version: 6\n")
args = append(args, "--overlay", "development")
} else {
writeConfigKitFile(t, main, "version: 5\nconfig:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n required: true\n")
writeConfigKitFile(t, main, "version: 6\nconfig:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n required: true\n")
}
writeConfigKitFile(t, fragment, "concurrency: {total_llm: invalid-private-value}\n")
writeConfigKitFile(t, fragment, "concurrency: {workers: invalid-private-value}\n")
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 {
t.Fatalf("status = %d; %s", code, stderr.String())
}
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
got := stderr.String()
if stdout.Len() != 0 || !strings.Contains(got, "decode.type") ||
!strings.Contains(got, "source-2:1:") ||
!strings.Contains(got, strconv.Quote(fragment)) ||
strings.Contains(got, "invalid-private-value") ||
strings.ContainsRune(got, '\x1b') {
t.Fatalf("stdout = %q; stderr = %q", stdout.String(), got)
if code == 0 || !strings.Contains(got, "decode.type") || !strings.Contains(got, strconv.Quote(fragment)) || strings.Contains(got, "invalid-private-value") || strings.ContainsRune(got, '\x1b') {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), got)
}
})
}
}
func TestLoadConfigKitUsesDiscoveryOverlayAndOverrides(t *testing.T) {
func TestApplicationLoadUsesOverlayAndOverrides(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, path, "version: 5\npipelines:\n base:\n input: {module: input}\n")
overlays := filepath.Join(root, "overlays")
writeConfigKitFile(t, filepath.Join(overlays, "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
loaded, gotPath, err := loadConfigKit(path, "development", []configkit.Override{{Path: configkit.Keys("output", "directory"), Value: "generated", Flag: "--output"}}, commandContractOptions(t))
writeConfigKitFile(t, path, "version: 6\npipelines: {base: {input: {module: input}}}\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines: {overlaid: {input: {module: input}}}\n")
selection := productionSelection(path)
selection.Overlay = commandkit.Supplied[string]{Set: true, Value: "development"}
selectedDirectory := filepath.Join(root, "generated")
selection.Overrides = []configkit.Override{{Path: configkit.Keys("output", "directory"), Value: selectedDirectory, Flag: "--set"}}
loaders := &nativeLoaders{lookupEnv: emptyLookup}
result, err := loaders.application(context.Background(), selection)
if err != nil {
t.Fatalf("loadConfigKit() error = %v", err)
t.Fatal(err)
}
if gotPath != path || loaded.Config.Pipelines["overlaid"].Input.Module != "input" || loaded.Config.Output.Directory == "generated" {
t.Fatalf("loaded = %#v, path=%q", loaded.Config, gotPath)
loaded, err := config.FromResult(result)
if err != nil {
t.Fatal(err)
}
cfg := loaded.RuntimeConfig()
if cfg.Pipelines["overlaid"].Input.Module != "input" || cfg.Output.Directory != selectedDirectory {
t.Fatalf("loaded configuration=%#v", cfg)
}
}
@@ -76,84 +83,82 @@ func TestNativeDiscoverySelectionAndTerminalFailures(t *testing.T) {
default:
t.Setenv("XDG_CONFIG_HOME", user)
}
previous, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
if err := os.Chdir(work); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := os.Chdir(previous); err != nil {
t.Error(err)
}
})
t.Chdir(work)
userFile := filepath.Join(user, "notarius", "config.yml")
writeConfigKitFile(t, userFile, "version: 5\npipelines: {user: {input: {module: input}}}\n")
writeConfigKitFile(t, filepath.Join(work, "config.yml"), "version: 5\npipelines: {generic: {input: {module: input}}}\n")
opts := commandContractOptions(t)
loaded, path, err := loadConfigKit("", "", nil, opts)
if err != nil || path != userFile || loaded.Config.Pipelines["user"].Input.Module != "input" {
t.Fatalf("user discovery = %q, %v", path, err)
writeConfigKitFile(t, userFile, "version: 6\npipelines: {user: {input: {module: input}}}\n")
writeConfigKitFile(t, filepath.Join(work, "config.yml"), "version: 6\npipelines: {generic: {input: {module: input}}}\n")
load := func(selection ckconfig.Selection, lookup func(string) (string, bool)) (*configkit.Result[config.Document], error) {
loaders := &nativeLoaders{lookupEnv: lookup}
return loaders.application(context.Background(), selection)
}
workingYAML := filepath.Join(work, "notarius.yaml")
workingYML := filepath.Join(work, "notarius.yml")
writeConfigKitFile(t, workingYAML, "version: 5\npipelines: {yaml: {input: {module: input}}}\n")
writeConfigKitFile(t, workingYML, "version: 5\npipelines: {yml: {input: {module: input}}}\n")
loaded, path, err = loadConfigKit("", "", nil, opts)
if err != nil || path != workingYML || loaded.Config.Pipelines["yml"].Input.Module != "input" || loaded.Config.Output.Directory != filepath.Join(work, "notarius-output") {
t.Fatalf("working discovery = %q, %v", path, err)
}
for _, args := range [][]string{{"pipelines", "list"}, {"config", "validate"}, {"config", "effective"}, {"config", "explain", "/version"}} {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 || stdout.Len() == 0 {
t.Fatalf("%v: code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
assertSelection := func(wantPath, wantID string) {
t.Helper()
result, err := load(ckconfig.Selection{}, emptyLookup)
if err != nil {
t.Fatal(err)
}
found := false
for _, source := range result.Sources() {
if source.Layer == configkit.LayerMain && source.Path == wantPath {
found = true
}
}
if !found || result.Config.Pipelines[wantID].Input.Module != "input" {
t.Fatalf("discovery missed %s/%s: sources=%#v config=%#v", wantPath, wantID, result.Sources(), result.Config.Pipelines)
}
}
missingEnv := opts
missingEnv.LookupEnv = func(name string) (string, bool) { return filepath.Join(root, "missing.yml"), name == "NOTARIUS_CONFIG" }
if _, _, err := loadConfigKit("", "", nil, missingEnv); err == nil {
assertSelection(userFile, "user")
workingYAML, workingYML := filepath.Join(work, "notarius.yaml"), filepath.Join(work, "notarius.yml")
writeConfigKitFile(t, workingYAML, "version: 6\npipelines: {yaml: {input: {module: input}}}\n")
writeConfigKitFile(t, workingYML, "version: 6\npipelines: {yml: {input: {module: input}}}\n")
assertSelection(workingYML, "yml")
for _, args := range [][]string{{"pipeline", "list"}, {"config", "validate"}, {"config", "show"}, {"config", "explain", "--path", "/version"}} {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.Len() == 0 {
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
}
missingEnv := func(name string) (string, bool) { return filepath.Join(root, "missing.yml"), name == "NOTARIUS_CONFIG" }
if _, err := load(ckconfig.Selection{}, missingEnv); err == nil {
t.Fatal("missing environment selection fell back")
}
if err := os.WriteFile(workingYML, []byte("version: [\n"), 0600); err != nil {
t.Fatal(err)
}
if _, _, err := loadConfigKit("", "", nil, opts); err == nil {
if _, err := load(ckconfig.Selection{}, emptyLookup); err == nil {
t.Fatal("invalid selected file fell back")
}
if _, _, err := loadConfigKit(filepath.Join(root, "missing.yml"), "", nil, opts); err == nil {
t.Fatal("missing explicit file fell back")
if _, err := load(productionSelection(filepath.Join(root, "missing.yml")), emptyLookup); err == nil {
t.Fatal("missing explicit selection fell back")
}
if err := os.Remove(workingYML); err != nil {
t.Fatal(err)
}
loaded, path, err = loadConfigKit("", "", nil, opts)
if err != nil || path != workingYAML || loaded.Config.Pipelines["yaml"].Input.Module != "input" {
t.Fatalf("alternate named file = %q, %v", path, err)
}
assertSelection(workingYAML, "yaml")
}
func TestWriteConfigLoadErrorUsesSafeDiagnostics(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "invalid.yml")
writeConfigKitFile(t, path, "version: 4\n")
_, _, err := loadConfigKit(path, "", nil, commandContractOptions(t))
writeConfigKitFile(t, path, "version: 5\n")
loaders := &nativeLoaders{lookupEnv: emptyLookup}
_, err := loaders.application(context.Background(), productionSelection(path))
if err == nil {
t.Fatal("loadConfigKit() error = nil")
t.Fatal("unsupported version loaded")
}
var output bytes.Buffer
writeConfigLoadError(&output, err)
if !strings.Contains(output.String(), "version") || strings.Contains(output.String(), "notarius: configkit") {
t.Fatalf("diagnostics = %q", output.String())
if !strings.Contains(output.String(), "version") || !strings.Contains(output.String(), path) {
t.Fatalf("diagnostics=%q", output.String())
}
}
func writeConfigKitFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
}

View File

@@ -54,7 +54,11 @@ func TestNativeLoadUsesOptionalCompanionsAndStrictPeers(t *testing.T) {
t.Fatal("embedded collection missing")
}
if test.name == "recursive" {
if len(result.Config.PromptKit.Profiles) != 1 || result.Config.PromptKit.Profiles[0].ID != "partial" {
found := false
for _, profile := range result.Config.PromptKit.Profiles {
found = found || profile.ID == "partial"
}
if !found {
t.Fatal("recursive companion not loaded")
}
}

View File

@@ -75,7 +75,7 @@ promptkit:
if client.calls.Load() != 0 {
t.Fatal("inspection implicitly called provider")
}
outcome, out, diagnostics := execute("promptkit", "test", "text-inference", "--llm-profile=ready")
outcome, out, diagnostics := execute("promptkit", "test", string(promptkit.SmokeTestTextInference), "--llm-profile=ready")
if outcome.Status != 0 || client.calls.Load() == 0 {
t.Fatalf("explicit smoke: %#v, %s, %s", outcome, out, diagnostics)
}