Files
notarius/internal/cli/configkit_commands_test.go

208 lines
8.2 KiB
Go
Raw Normal View History

package cli
import (
"bytes"
"encoding/json"
"fmt"
2026-09-13 17:16:06 +00:00
"io"
"path/filepath"
"reflect"
"strings"
"testing"
)
func TestConfigKitCommandsLoadImportsAndOverlays(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: seriatim}\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: seriatim}\n")
writeConfigKitFile(t, path, "version: 6\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: seriatim}\n")
for _, test := range []struct {
flags []string
want []string
}{
{want: []string{"base", "imported"}},
{flags: []string{"--overlay", "development"}, want: []string{"base", "imported", "overlaid"}},
} {
var stdout, stderr bytes.Buffer
args := append([]string{"pipeline", "list", "--config", path, "--format", "json"}, test.flags...)
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
var ids []string
if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil {
t.Fatal(err)
}
if code != 0 || !reflect.DeepEqual(ids, test.want) {
t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String())
}
}
}
func TestPipelineInheritanceCommandsInspectDerivedPipeline(t *testing.T) {
t.Parallel()
path := writeCommandConfigContent(t, `version: 6
pipelines:
" parent ":
input: {module: seriatim}
artifacts:
spells:
extract: {module: dnd/spells}
max_additional_turns: 2
child:
extends: " parent "
max_additional_turns: 0
`)
for _, args := range [][]string{
{"pipeline", "validate", "child"},
{"config", "explain", "--path", "/pipelines/child/input/module"},
} {
var stdout, stderr bytes.Buffer
args = append(args, "--config", path)
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
if code != 0 || stdout.Len() == 0 {
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
2026-09-13 17:16:06 +00:00
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "show", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t))
var report struct {
Pipelines map[string]struct {
Input struct {
Module string `json:"module"`
} `json:"input"`
Turns int `json:"max_additional_turns"`
} `json:"pipelines"`
}
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatal(err)
2026-09-13 17:16:06 +00:00
}
child, ok := report.Pipelines["child"]
if code != 0 || !ok || child.Input.Module != "seriatim" || child.Turns != 0 {
t.Fatalf("status=%d inherited=%#v stderr=%q", code, report, stderr.String())
2026-09-13 17:16:06 +00:00
}
}
func TestPipelineInheritanceFailuresRemainLoadFailures(t *testing.T) {
2026-09-13 17:16:06 +00:00
t.Parallel()
for _, test := range []struct{ name, content, fragment string }{
{"missing parent", "pipelines: {child: {extends: absent}}\n", "inheritance.missing"},
{"cycle", "pipelines: {first: {extends: second}, second: {extends: first}}\n", "cycle"},
{"invalid directive", "pipelines: {child: {extends: 7}}\n", "extends"},
{"unselected missing parent", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: parent}, unrelated: {extends: absent}}\n", "inheritance.missing"},
{"exact sibling mismatch", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: 'parent '}}\n", "inheritance.missing"},
2026-09-13 17:16:06 +00:00
} {
t.Run(test.name, func(t *testing.T) {
path := writeCommandConfigContent(t, "version: 6\n"+test.content)
2026-09-13 17:16:06 +00:00
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t))
if code == 0 || !strings.Contains(stderr.String(), test.fragment) {
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
2026-09-13 17:16:06 +00:00
}
})
}
}
func TestConfigurationScopesUseSelectedSourcesAndNativeSecretMarks(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, path, "version: 6\nconcurrency: {workers: 4}\npipelines: {demo: {input: {module: seriatim}}}\n")
writeConfigKitFile(t, filepath.Join(root, "prompts.yml"), `config:
imports:
- path: profiles/base.yml
required: true
`)
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
- id: base
endpoint: https://example.invalid/v1
model: inherited-model
api_key_env: NOTARIUS_TEST_KEY_NOT_REQUIRED_FOR_INSPECTION
api_key_required: true
extra_params: {token: private-test-token, ordinary: visible-value}
- id: selected
extends: {id: base}
temperature: 0
extra_params: null
`)
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency: {workers: 3}\n")
client := &diagnosticClient{}
options := commandContractOptions(t)
options.LLMClient = client
baseIndex := -1
for _, scope := range []string{"app", "promptkit"} {
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "show", "--scope", scope, "--config", path, "--overlay", "development", "--format", "json"}, &stdout, &stderr, options)
if code != 0 || !json.Valid(stdout.Bytes()) || !strings.Contains(stdout.String(), "inherited-model") || !strings.Contains(stdout.String(), "visible-value") || strings.Contains(stdout.String(), "private-test-token") {
t.Fatalf("%s status=%d stdout=%q stderr=%q", scope, code, stdout.String(), stderr.String())
}
var report map[string]any
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatal(err)
}
if scope == "app" {
controls, ok := report["concurrency"].(map[string]any)
if !ok || controls["workers"] != float64(3) {
t.Fatalf("overlay missing: %#v", report)
}
} else {
if report["pipelines"] != nil {
t.Fatalf("focused scope exposed application root: %#v", report)
}
profiles, _ := report["profiles"].([]any)
for index, raw := range profiles {
profile, _ := raw.(map[string]any)
if profile["id"] == "base" {
baseIndex = index
}
}
}
}
if baseIndex < 0 {
t.Fatal("focused configuration omitted the imported profile")
}
for _, args := range [][]string{
{"config", "sources", "--scope", "promptkit"},
{"config", "explain", "--scope", "promptkit", "--path", fmt.Sprintf("/promptkit/profiles/%d/model", baseIndex)},
} {
var stdout, stderr bytes.Buffer
code := RunWithOptions(append(args, "--config", path), &stdout, &stderr, options)
if code != 0 || !strings.Contains(stdout.String(), "base.yml") {
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
}
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"promptkit", "show", "profile", "selected", "--config", path, "--format", "json"}, &stdout, &stderr, options)
if code != 0 || !strings.Contains(stdout.String(), "inherited-model") || strings.Contains(stdout.String(), "private-test-token") {
t.Fatalf("profile status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
if client.calls.Load() != 0 {
t.Fatal("configuration/profile inspection invoked provider")
}
}
func TestConfigurationCommandsRetainNativeInputAndPresentationFailures(t *testing.T) {
t.Parallel()
for _, scope := range []string{"app", "promptkit"} {
var stdout, stderr bytes.Buffer
missing := filepath.Join(t.TempDir(), "missing.yml")
code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", missing, "--path", "/profiles/~2"}, &stdout, &stderr, commandContractOptions(t))
if code != 2 || stdout.Len() != 0 || strings.Contains(stderr.String(), missing) {
t.Fatalf("invalid path reached loading: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
}
path := writeCommandConfig(t, "session", "other")
for _, args := range [][]string{
{"config", "show"}, {"config", "show", "--scope", "promptkit"}, {"promptkit", "show", "profile", "notarius/dnd-extraction"},
} {
var stderr bytes.Buffer
code := RunWithOptions(append(args, "--config", path), failingInspectionWriter{}, &stderr, commandContractOptions(t))
if code != 1 {
t.Fatalf("writer failure %v: status=%d stderr=%q", args, code, stderr.String())
}
}
}
2026-09-13 17:16:06 +00:00
type failingInspectionWriter struct{}
func (failingInspectionWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }