208 lines
8.2 KiB
Go
208 lines
8.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestConfigKitCommandsLoadImportsAndOverlays(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: seriatim}\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: seriatim}\n")
|
|
writeConfigKitFile(t, path, "version: 6\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: seriatim}\n")
|
|
for _, test := range []struct {
|
|
flags []string
|
|
want []string
|
|
}{
|
|
{want: []string{"base", "imported"}},
|
|
{flags: []string{"--overlay", "development"}, want: []string{"base", "imported", "overlaid"}},
|
|
} {
|
|
var stdout, stderr bytes.Buffer
|
|
args := append([]string{"pipeline", "list", "--config", path, "--format", "json"}, test.flags...)
|
|
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
|
|
var ids []string
|
|
if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if code != 0 || !reflect.DeepEqual(ids, test.want) {
|
|
t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceCommandsInspectDerivedPipeline(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfigContent(t, `version: 6
|
|
pipelines:
|
|
" parent ":
|
|
input: {module: seriatim}
|
|
artifacts:
|
|
spells:
|
|
extract: {module: dnd/spells}
|
|
max_additional_turns: 2
|
|
child:
|
|
extends: " parent "
|
|
max_additional_turns: 0
|
|
`)
|
|
for _, args := range [][]string{
|
|
{"pipeline", "validate", "child"},
|
|
{"config", "explain", "--path", "/pipelines/child/input/module"},
|
|
} {
|
|
var stdout, stderr bytes.Buffer
|
|
args = append(args, "--config", path)
|
|
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
|
|
if code != 0 || stdout.Len() == 0 {
|
|
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"config", "show", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t))
|
|
var report struct {
|
|
Pipelines map[string]struct {
|
|
Input struct {
|
|
Module string `json:"module"`
|
|
} `json:"input"`
|
|
Turns int `json:"max_additional_turns"`
|
|
} `json:"pipelines"`
|
|
}
|
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
child, ok := report.Pipelines["child"]
|
|
if code != 0 || !ok || child.Input.Module != "seriatim" || child.Turns != 0 {
|
|
t.Fatalf("status=%d inherited=%#v stderr=%q", code, report, stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceFailuresRemainLoadFailures(t *testing.T) {
|
|
t.Parallel()
|
|
for _, test := range []struct{ name, content, fragment string }{
|
|
{"missing parent", "pipelines: {child: {extends: absent}}\n", "inheritance.missing"},
|
|
{"cycle", "pipelines: {first: {extends: second}, second: {extends: first}}\n", "cycle"},
|
|
{"invalid directive", "pipelines: {child: {extends: 7}}\n", "extends"},
|
|
{"unselected missing parent", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: parent}, unrelated: {extends: absent}}\n", "inheritance.missing"},
|
|
{"exact sibling mismatch", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: 'parent '}}\n", "inheritance.missing"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
path := writeCommandConfigContent(t, "version: 6\n"+test.content)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t))
|
|
if code == 0 || !strings.Contains(stderr.String(), test.fragment) {
|
|
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigurationScopesUseSelectedSourcesAndNativeSecretMarks(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, path, "version: 6\nconcurrency: {workers: 4}\npipelines: {demo: {input: {module: seriatim}}}\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "prompts.yml"), `config:
|
|
imports:
|
|
- path: profiles/base.yml
|
|
required: true
|
|
`)
|
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
|
|
- id: base
|
|
endpoint: https://example.invalid/v1
|
|
model: inherited-model
|
|
api_key_env: NOTARIUS_TEST_KEY_NOT_REQUIRED_FOR_INSPECTION
|
|
api_key_required: true
|
|
extra_params: {token: private-test-token, ordinary: visible-value}
|
|
- id: selected
|
|
extends: {id: base}
|
|
temperature: 0
|
|
extra_params: null
|
|
`)
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency: {workers: 3}\n")
|
|
client := &diagnosticClient{}
|
|
options := commandContractOptions(t)
|
|
options.LLMClient = client
|
|
baseIndex := -1
|
|
for _, scope := range []string{"app", "promptkit"} {
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"config", "show", "--scope", scope, "--config", path, "--overlay", "development", "--format", "json"}, &stdout, &stderr, options)
|
|
if code != 0 || !json.Valid(stdout.Bytes()) || !strings.Contains(stdout.String(), "inherited-model") || !strings.Contains(stdout.String(), "visible-value") || strings.Contains(stdout.String(), "private-test-token") {
|
|
t.Fatalf("%s status=%d stdout=%q stderr=%q", scope, code, stdout.String(), stderr.String())
|
|
}
|
|
var report map[string]any
|
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if scope == "app" {
|
|
controls, ok := report["concurrency"].(map[string]any)
|
|
if !ok || controls["workers"] != float64(3) {
|
|
t.Fatalf("overlay missing: %#v", report)
|
|
}
|
|
} else {
|
|
if report["pipelines"] != nil {
|
|
t.Fatalf("focused scope exposed application root: %#v", report)
|
|
}
|
|
profiles, _ := report["profiles"].([]any)
|
|
for index, raw := range profiles {
|
|
profile, _ := raw.(map[string]any)
|
|
if profile["id"] == "base" {
|
|
baseIndex = index
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if baseIndex < 0 {
|
|
t.Fatal("focused configuration omitted the imported profile")
|
|
}
|
|
for _, args := range [][]string{
|
|
{"config", "sources", "--scope", "promptkit"},
|
|
{"config", "explain", "--scope", "promptkit", "--path", fmt.Sprintf("/promptkit/profiles/%d/model", baseIndex)},
|
|
} {
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions(append(args, "--config", path), &stdout, &stderr, options)
|
|
if code != 0 || !strings.Contains(stdout.String(), "base.yml") {
|
|
t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"promptkit", "show", "profile", "selected", "--config", path, "--format", "json"}, &stdout, &stderr, options)
|
|
if code != 0 || !strings.Contains(stdout.String(), "inherited-model") || strings.Contains(stdout.String(), "private-test-token") {
|
|
t.Fatalf("profile status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
if client.calls.Load() != 0 {
|
|
t.Fatal("configuration/profile inspection invoked provider")
|
|
}
|
|
}
|
|
|
|
func TestConfigurationCommandsRetainNativeInputAndPresentationFailures(t *testing.T) {
|
|
t.Parallel()
|
|
for _, scope := range []string{"app", "promptkit"} {
|
|
var stdout, stderr bytes.Buffer
|
|
missing := filepath.Join(t.TempDir(), "missing.yml")
|
|
code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", missing, "--path", "/profiles/~2"}, &stdout, &stderr, commandContractOptions(t))
|
|
if code != 2 || stdout.Len() != 0 || strings.Contains(stderr.String(), missing) {
|
|
t.Fatalf("invalid path reached loading: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
path := writeCommandConfig(t, "session", "other")
|
|
for _, args := range [][]string{
|
|
{"config", "show"}, {"config", "show", "--scope", "promptkit"}, {"promptkit", "show", "profile", "notarius/dnd-extraction"},
|
|
} {
|
|
var stderr bytes.Buffer
|
|
code := RunWithOptions(append(args, "--config", path), failingInspectionWriter{}, &stderr, commandContractOptions(t))
|
|
if code != 1 {
|
|
t.Fatalf("writer failure %v: status=%d stderr=%q", args, code, stderr.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
type failingInspectionWriter struct{}
|
|
|
|
func (failingInspectionWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }
|