package cli import ( "bytes" "encoding/json" "fmt" "io" "path/filepath" "reflect" "strings" "testing" ) func TestConfigKitCommandsLoadImportsAndOverlays(t *testing.T) { t.Parallel() root := t.TempDir() path := filepath.Join(root, "notarius.yml") writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: seriatim}\n") writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: seriatim}\n") writeConfigKitFile(t, path, "version: 6\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: seriatim}\n") for _, test := range []struct { flags []string want []string }{ {want: []string{"base", "imported"}}, {flags: []string{"--overlay", "development"}, want: []string{"base", "imported", "overlaid"}}, } { var stdout, stderr bytes.Buffer args := append([]string{"pipeline", "list", "--config", path, "--format", "json"}, test.flags...) code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)) var ids []string if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil { t.Fatal(err) } if code != 0 || !reflect.DeepEqual(ids, test.want) { t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String()) } } } func TestPipelineInheritanceCommandsInspectDerivedPipeline(t *testing.T) { t.Parallel() path := writeCommandConfigContent(t, `version: 6 pipelines: " parent ": input: {module: seriatim} artifacts: spells: extract: {module: dnd/spells} max_additional_turns: 2 child: extends: " parent " max_additional_turns: 0 `) for _, args := range [][]string{ {"pipeline", "validate", "child"}, {"config", "explain", "--path", "/pipelines/child/input/module"}, } { var stdout, stderr bytes.Buffer args = append(args, "--config", path) code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)) if code != 0 || stdout.Len() == 0 { t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) } } var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"config", "show", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t)) var report struct { Pipelines map[string]struct { Input struct { Module string `json:"module"` } `json:"input"` Turns int `json:"max_additional_turns"` } `json:"pipelines"` } if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { t.Fatal(err) } child, ok := report.Pipelines["child"] if code != 0 || !ok || child.Input.Module != "seriatim" || child.Turns != 0 { t.Fatalf("status=%d inherited=%#v stderr=%q", code, report, stderr.String()) } } func TestPipelineInheritanceFailuresRemainLoadFailures(t *testing.T) { t.Parallel() for _, test := range []struct{ name, content, fragment string }{ {"missing parent", "pipelines: {child: {extends: absent}}\n", "inheritance.missing"}, {"cycle", "pipelines: {first: {extends: second}, second: {extends: first}}\n", "cycle"}, {"invalid directive", "pipelines: {child: {extends: 7}}\n", "extends"}, {"unselected missing parent", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: parent}, unrelated: {extends: absent}}\n", "inheritance.missing"}, {"exact sibling mismatch", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: 'parent '}}\n", "inheritance.missing"}, } { t.Run(test.name, func(t *testing.T) { path := writeCommandConfigContent(t, "version: 6\n"+test.content) var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t)) if code == 0 || !strings.Contains(stderr.String(), test.fragment) { t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } } func TestConfigurationScopesUseSelectedSourcesAndNativeSecretMarks(t *testing.T) { t.Parallel() root := t.TempDir() path := filepath.Join(root, "notarius.yml") writeConfigKitFile(t, path, "version: 6\nconcurrency: {workers: 4}\npipelines: {demo: {input: {module: seriatim}}}\n") writeConfigKitFile(t, filepath.Join(root, "prompts.yml"), `config: imports: - path: profiles/base.yml required: true `) writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles: - id: base endpoint: https://example.invalid/v1 model: inherited-model api_key_env: NOTARIUS_TEST_KEY_NOT_REQUIRED_FOR_INSPECTION api_key_required: true extra_params: {token: private-test-token, ordinary: visible-value} - id: selected extends: {id: base} temperature: 0 extra_params: null `) writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency: {workers: 3}\n") client := &diagnosticClient{} options := commandContractOptions(t) options.LLMClient = client baseIndex := -1 for _, scope := range []string{"app", "promptkit"} { var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"config", "show", "--scope", scope, "--config", path, "--overlay", "development", "--format", "json"}, &stdout, &stderr, options) if code != 0 || !json.Valid(stdout.Bytes()) || !strings.Contains(stdout.String(), "inherited-model") || !strings.Contains(stdout.String(), "visible-value") || strings.Contains(stdout.String(), "private-test-token") { t.Fatalf("%s status=%d stdout=%q stderr=%q", scope, code, stdout.String(), stderr.String()) } var report map[string]any if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { t.Fatal(err) } if scope == "app" { controls, ok := report["concurrency"].(map[string]any) if !ok || controls["workers"] != float64(3) { t.Fatalf("overlay missing: %#v", report) } } else { if report["pipelines"] != nil { t.Fatalf("focused scope exposed application root: %#v", report) } profiles, _ := report["profiles"].([]any) for index, raw := range profiles { profile, _ := raw.(map[string]any) if profile["id"] == "base" { baseIndex = index } } } } if baseIndex < 0 { t.Fatal("focused configuration omitted the imported profile") } for _, args := range [][]string{ {"config", "sources", "--scope", "promptkit"}, {"config", "explain", "--scope", "promptkit", "--path", fmt.Sprintf("/promptkit/profiles/%d/model", baseIndex)}, } { var stdout, stderr bytes.Buffer code := RunWithOptions(append(args, "--config", path), &stdout, &stderr, options) if code != 0 || !strings.Contains(stdout.String(), "base.yml") { t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) } } var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"promptkit", "show", "profile", "selected", "--config", path, "--format", "json"}, &stdout, &stderr, options) if code != 0 || !strings.Contains(stdout.String(), "inherited-model") || strings.Contains(stdout.String(), "private-test-token") { t.Fatalf("profile status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } if client.calls.Load() != 0 { t.Fatal("configuration/profile inspection invoked provider") } } func TestConfigurationCommandsRetainNativeInputAndPresentationFailures(t *testing.T) { t.Parallel() for _, scope := range []string{"app", "promptkit"} { var stdout, stderr bytes.Buffer missing := filepath.Join(t.TempDir(), "missing.yml") code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", missing, "--path", "/profiles/~2"}, &stdout, &stderr, commandContractOptions(t)) if code != 2 || stdout.Len() != 0 || strings.Contains(stderr.String(), missing) { t.Fatalf("invalid path reached loading: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } path := writeCommandConfig(t, "session", "other") for _, args := range [][]string{ {"config", "show"}, {"config", "show", "--scope", "promptkit"}, {"promptkit", "show", "profile", "notarius/dnd-extraction"}, } { var stderr bytes.Buffer code := RunWithOptions(append(args, "--config", path), failingInspectionWriter{}, &stderr, commandContractOptions(t)) if code != 1 { t.Fatalf("writer failure %v: status=%d stderr=%q", args, code, stderr.String()) } } } type failingInspectionWriter struct{} func (failingInspectionWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }