diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d2f3c112..f61e0e2c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1718,6 +1718,17 @@ resume and manifest/sidecar assertions and their helpers. Focused assembled spel catalog tests, vet, and repeated race checks pass; remaining CLI fixtures and final compilation are still required. +Configuration/discovery/inspection fixtures now use native scopes, rooted paths, +overlays, supplied overrides, pipeline validation, and profile inspection. +Retained source-relative discovery, inherited pipeline composition, source +diagnostics, secret-aware configuration views, no implicit provider work, and +presentation failures. Native validation may report failed scopes on stdout; +tests no longer impose the retired application's empty-error-output convention. +Fixed stale fixtures to include bundled profiles and use PromptKit's native +smoke-test kind rather than a command spelling. Focused native command/configuration +tests, vet, and repeated race checks pass. Whole CLI compilation still awaits +the remaining run/state, enemy-event, reference, and example fixture cleanup. + **Goal:** delete the old architecture rather than leave dormant alternatives. **Inspect/change:** all production/test callers of the feature roadmap's removal diff --git a/internal/cli/command_contract_test.go b/internal/cli/command_contract_test.go index 279d13ca..a3d31aa7 100644 --- a/internal/cli/command_contract_test.go +++ b/internal/cli/command_contract_test.go @@ -6,48 +6,27 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/buildinfo" ) -func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) { - tests := [][]string{nil, {"help"}, {"--help"}, {"-h"}} - for _, args := range tests { - name := "no arguments" - if len(args) > 0 { - name = args[0] - } - t.Run(name, func(t *testing.T) { +func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) { + for _, args := range [][]string{ + {"unknown"}, + {"pipeline", "unknown"}, + {"config", "validate", "--unknown"}, + {"run", "demo", "--input-file", "missing", "--cache", "invalid"}, + {"pipeline", "validate"}, + {"run", "demo", "--input-file", "missing", "--only", "spells"}, + } { + t.Run(strings.Join(args, " "), func(t *testing.T) { var stdout, stderr bytes.Buffer code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)) - if code != 0 || !strings.Contains(stdout.String(), "Usage:") || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) - } -} - -func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) { - tests := []struct { - name string - args []string - want string - }{ - {name: "unknown command", args: []string{"unknown"}, want: "unknown command"}, - {name: "missing config subcommand", args: []string{"config"}, want: "config requires a subcommand"}, - {name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"}, - {name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"}, - {name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"}, - {name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var stdout, stderr bytes.Buffer - code := RunWithOptions(tt.args, &stdout, &stderr, commandContractOptions(t)) - if code != 2 || !strings.Contains(stderr.String(), tt.want) || stdout.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } @@ -56,25 +35,21 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) { func TestCommandVersionOutput(t *testing.T) { previous := buildinfo.Override t.Cleanup(func() { buildinfo.Override = previous }) - - tests := []struct { - name string - override string - wantCode int - wantStdout string - wantStderr string + for _, test := range []struct { + name, override string + status int + fragment string }{ - {name: "development", wantStdout: "notarius development\n"}, - {name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"}, - {name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buildinfo.Override = tt.override + {name: "development", fragment: "development"}, + {name: "release", override: "v1.2.3", fragment: "v1.2.3"}, + {name: "invalid release identity", override: "release", status: 1}, + } { + t.Run(test.name, func(t *testing.T) { + buildinfo.Override = test.override var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{}) - if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + code := RunWithOptions([]string{"version"}, &stdout, &stderr, Options{}) + if code != test.status || test.status == 0 && (!strings.Contains(stdout.String(), test.fragment) || stderr.Len() != 0) || test.status != 0 && (stdout.Len() != 0 || stderr.Len() == 0) { + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } @@ -83,143 +58,104 @@ func TestCommandVersionOutput(t *testing.T) { func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) { explicit := writeCommandConfig(t, "explicit", "alpha") environment := writeCommandConfig(t, "environment", "beta") - lookup := func(name string) (string, bool) { - if name == "NOTARIUS_CONFIG" { - return environment, true - } - return "", false - } - - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"pipelines", "list", "--config", explicit}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup)) - if code != 0 || stdout.String() != "alpha\nexplicit\n" || stderr.Len() != 0 { - t.Fatalf("explicit config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"pipelines", "list"}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup)) - if code != 0 || stdout.String() != "beta\nenvironment\n" || stderr.Len() != 0 { - t.Fatalf("environment config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + lookup := func(name string) (string, bool) { return environment, name == "NOTARIUS_CONFIG" } + for _, test := range []struct { + name string + flags []string + want []string + }{ + {name: "explicit", flags: []string{"--config", explicit}, want: []string{"alpha", "explicit"}}, + {name: "environment", want: []string{"beta", "environment"}}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + args := append([]string{"pipeline", "list", "--format", "json"}, test.flags...) + code := RunWithOptions(args, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup)) + var ids []string + if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil { + t.Fatal(err) + } + if code != 0 || !reflect.DeepEqual(ids, test.want) { + t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String()) + } + }) } } -func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) { - missing := filepath.Join(t.TempDir(), "missing.yml") - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"config", "validate", "--config", missing}, &stdout, &stderr, commandContractOptions(t)) - if code != 1 || !strings.Contains(stderr.String(), missing) || stdout.Len() != 0 { - t.Fatalf("missing config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - malformed := filepath.Join(t.TempDir(), "malformed.yml") - if err := os.WriteFile(malformed, []byte("version: [\n"), 0o600); err != nil { +func TestConfigLoadingFailuresRetainPathContext(t *testing.T) { + root := t.TempDir() + missing := filepath.Join(root, "missing.yml") + malformed := filepath.Join(root, "malformed.yml") + if err := os.WriteFile(malformed, []byte("version: [\n"), 0600); err != nil { t.Fatal(err) } - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"config", "validate", "--config", malformed}, &stdout, &stderr, commandContractOptions(t)) - if code != 1 || !strings.Contains(stderr.String(), "configuration contains invalid YAML") || stdout.Len() != 0 { - t.Fatalf("malformed config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + for _, path := range []string{missing, malformed} { + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t)) + if code == 0 || !strings.Contains(stderr.String(), path) { + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } } } -func TestConfigValidateResolvesPipelineAndChecksSelection(t *testing.T) { - configPath := writeResolvableCommandConfig(t) - options := commandContractOptions(t) - - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells"}, &stdout, &stderr, options) - if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 { - t.Fatalf("valid resolution: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "missing"}, &stdout, &stderr, options) - if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") { - t.Fatalf("unknown pipeline: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "missing"}, &stdout, &stderr, options) - if code != 1 || !strings.Contains(stderr.String(), "lane \"missing\"") { - t.Fatalf("unknown lane: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--only", "spells"}, &stdout, &stderr, options) - if code != 2 || !strings.Contains(stderr.String(), "--only requires --pipeline") { - t.Fatalf("missing pipeline for only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells,,other"}, &stdout, &stderr, options) - if code != 2 || !strings.Contains(stderr.String(), "--only must contain") { - t.Fatalf("malformed only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) +func TestPipelineValidationSelectsConfiguredCompositionOffline(t *testing.T) { + path := writeResolvableCommandConfig(t) + for _, test := range []struct { + id string + status int + }{ + {id: "demo"}, {id: "missing", status: 2}, + } { + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"pipeline", "validate", test.id, "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t)) + if code != test.status { + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if code == 0 { + var report struct { + Pipeline string `json:"pipeline"` + Valid bool `json:"valid"` + } + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) + } + if report.Pipeline != test.id || !report.Valid { + t.Fatalf("validation report=%#v", report) + } + } else if !strings.Contains(stderr.String(), test.id) { + t.Fatalf("missing pipeline context: %s", stderr.String()) + } } } -func TestConfigValidatePipelineDefaultProfileIsOffline(t *testing.T) { - configPath := writeCommandConfigContent(t, `version: 5 +func TestPipelineValidationDefaultProfileNeedsNoCredentialsOrProvider(t *testing.T) { + path := writeCommandConfigContent(t, `version: 6 pipelines: demo: - llm_profile: dnd-extraction - input: - module: seriatim + llm_profile: notarius/dnd-extraction + input: {module: seriatim} artifacts: spells: - extract: - module: dnd/spells + extract: {module: dnd/spells} `) + client := &diagnosticClient{} var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo"}, &stdout, &stderr, Options{}) - if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + code := RunWithOptions([]string{"pipeline", "validate", "demo", "--config", path}, &stdout, &stderr, Options{LLMClient: client, LookupEnv: emptyLookup}) + if code != 0 || client.calls.Load() != 0 { + t.Fatalf("status=%d calls=%d stderr=%q", code, client.calls.Load(), stderr.String()) } } -func TestPipelinesListSortsNormalizedIDsInTextAndJSON(t *testing.T) { - configPath := writeCommandConfig(t, " zeta ", "alpha") - options := commandContractOptions(t) +func TestPipelineListSortsNormalizedIDs(t *testing.T) { + path := writeCommandConfig(t, " zeta ", "alpha") var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"pipelines", "list", "--config", configPath}, &stdout, &stderr, options) - if code != 0 || stdout.String() != "alpha\nzeta\n" || stderr.Len() != 0 { - t.Fatalf("text list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + code := RunWithOptions([]string{"pipeline", "list", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t)) + var ids []string + if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil { + t.Fatal(err) } - - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"pipelines", "list", "--config", configPath, "--json"}, &stdout, &stderr, options) - var payload struct { - Pipelines []string `json:"pipelines"` - } - if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil { - t.Fatalf("JSON list = %q: %v", stdout.String(), err) - } - if code != 0 || len(payload.Pipelines) != 2 || payload.Pipelines[0] != "alpha" || payload.Pipelines[1] != "zeta" || stderr.Len() != 0 { - t.Fatalf("JSON list: code=%d payload=%#v stderr=%q", code, payload, stderr.String()) - } -} - -func TestRemovedStructuralFlagsAndRuntimeFailuresKeepExitClasses(t *testing.T) { - configPath := writeResolvableCommandConfig(t) - options := commandContractOptions(t) - - var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"run", "demo", "--input", "missing-input", "--config", configPath, "--diagnostics-dir", t.TempDir()}, &stdout, &stderr, options) - if code != 2 || !strings.Contains(stderr.String(), "flag provided but not defined") { - t.Fatalf("removed flag: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"run", "missing", "--input", "missing-input", "--config", configPath, "--chunk_cache", "bypass"}, &stdout, &stderr, options) - if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") || stdout.Len() != 0 { - t.Fatalf("valid-runtime failure: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + if code != 0 || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) { + t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String()) } } @@ -229,42 +165,29 @@ func commandContractOptions(t *testing.T) Options { func commandContractOptionsWithLookup(t *testing.T, lookup func(string) (string, bool)) Options { t.Helper() - components, err := newProductionComponents() - if err != nil { - t.Fatal(err) - } - return Options{ - Catalog: catalogFromRegistries(components.registries), - Registries: components.registries, - LookupEnv: lookup, - } + options := productionCLIOptions(t) + options.LookupEnv = lookup + return options } func writeCommandConfig(t *testing.T, firstID, secondID string) string { t.Helper() - content := fmt.Sprintf("version: 5\npipelines:\n %q:\n input:\n module: seriatim\n %q:\n input:\n module: seriatim\n", firstID, secondID) - return writeCommandConfigContent(t, content) + return writeCommandConfigContent(t, fmt.Sprintf("version: 6\npipelines:\n %q:\n input: {module: seriatim}\n %q:\n input: {module: seriatim}\n", firstID, secondID)) } func writeResolvableCommandConfig(t *testing.T) string { t.Helper() - return writeCommandConfigContent(t, `version: 5 + return writeCommandConfigContent(t, `version: 6 pipelines: demo: - input: - module: seriatim + input: {module: seriatim} artifacts: spells: - extract: - module: dnd/spells + extract: {module: dnd/spells} `) } func writeCommandConfigContent(t *testing.T, content string) string { t.Helper() - path := filepath.Join(t.TempDir(), "config.yml") - if err := os.WriteFile(path, []byte(content), 0o600); err != nil { - t.Fatal(err) - } - return path + return writeProductionContractConfig(t, content) } diff --git a/internal/cli/configkit_commands_test.go b/internal/cli/configkit_commands_test.go index 6cb768cb..aa2bf3eb 100644 --- a/internal/cli/configkit_commands_test.go +++ b/internal/cli/configkit_commands_test.go @@ -2,523 +2,206 @@ package cli import ( "bytes" - "context" "encoding/json" - "errors" + "fmt" "io" "path/filepath" + "reflect" "strings" "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) -func TestExplainRejectsMalformedPointerBeforeLoadingEitherScope(t *testing.T) { - for _, scope := range []string{"notarius", "promptkit"} { - t.Run(scope, func(t *testing.T) { - var stdout, stderr strings.Builder - options := commandContractOptions(t) - options.LookupEnv = func(string) (string, bool) { - t.Fatal("configuration discovery ran for invalid syntax") - return "", false - } - code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", filepath.Join(t.TempDir(), "missing.yml"), "/profiles/~2"}, &stdout, &stderr, options) - if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "canonical JSON Pointer") { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) - } -} - -func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) { +func TestConfigKitCommandsLoadImportsAndOverlays(t *testing.T) { t.Parallel() root := t.TempDir() path := filepath.Join(root, "notarius.yml") - writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n") - writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n") - writeConfigKitFile(t, path, "version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: input}\n") - var stdout, stderr bytes.Buffer - if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" { - t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String()) - } - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") { - t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String()) - } - for _, selection := range [][]string{nil, {"--config-overlay", ""}, {"--config-overlay", "default"}} { - stdout.Reset() - stderr.Reset() - args := append([]string{"pipelines", "list", "--config", path}, selection...) - if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\n" || stderr.Len() != 0 { - t.Fatalf("base list %q = %d, %q, %q", selection, code, stdout.String(), stderr.String()) + writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: seriatim}\n") + writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: seriatim}\n") + writeConfigKitFile(t, path, "version: 6\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: seriatim}\n") + for _, test := range []struct { + flags []string + want []string + }{ + {want: []string{"base", "imported"}}, + {flags: []string{"--overlay", "development"}, want: []string{"base", "imported", "overlaid"}}, + } { + var stdout, stderr bytes.Buffer + args := append([]string{"pipeline", "list", "--config", path, "--format", "json"}, test.flags...) + code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)) + var ids []string + if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil { + t.Fatal(err) + } + if code != 0 || !reflect.DeepEqual(ids, test.want) { + t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String()) } } } -func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) { +func TestPipelineInheritanceCommandsInspectDerivedPipeline(t *testing.T) { t.Parallel() - path := writeCommandConfigContent(t, `version: 5 + path := writeCommandConfigContent(t, `version: 6 pipelines: " parent ": - input: - module: seriatim + input: {module: seriatim} artifacts: spells: - extract: - module: dnd/spells - structured_output_repair_attempts: 2 + extract: {module: dnd/spells} + max_additional_turns: 2 child: extends: " parent " - structured_output_repair_attempts: 0 + max_additional_turns: 0 `) - options := commandContractOptions(t) - + for _, args := range [][]string{ + {"pipeline", "validate", "child"}, + {"config", "explain", "--path", "/pipelines/child/input/module"}, + } { + var stdout, stderr bytes.Buffer + args = append(args, "--config", path) + code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)) + if code != 0 || stdout.Len() == 0 { + t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + } var stdout, stderr bytes.Buffer - if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 { - t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String()) + code := RunWithOptions([]string{"config", "show", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t)) + var report struct { + Pipelines map[string]struct { + Input struct { + Module string `json:"module"` + } `json:"input"` + Turns int `json:"max_additional_turns"` + } `json:"pipelines"` } - - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 { - t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String()) + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) } - - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 { - t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 { - t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String()) + child, ok := report.Pipelines["child"] + if code != 0 || !ok || child.Input.Module != "seriatim" || child.Turns != 0 { + t.Fatalf("status=%d inherited=%#v stderr=%q", code, report, stderr.String()) } } -func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) { +func TestPipelineInheritanceFailuresRemainLoadFailures(t *testing.T) { t.Parallel() - for _, test := range []struct { - name string - content string - args []string - want string - }{ - { - name: "missing parent", - content: `version: 5 -pipelines: - child: - extends: absent -`, - args: []string{"config", "validate", "--pipeline", "child"}, - want: "inheritance.missing", - }, - { - name: "cycle", - content: `version: 5 -pipelines: - first: - extends: second - second: - extends: first -`, - args: []string{"config", "validate", "--pipeline", "first"}, - want: "cycle", - }, - { - name: "invalid directive type", - content: `version: 5 -pipelines: - child: - extends: 7 -`, - args: []string{"config", "validate", "--pipeline", "child"}, - want: "extends", - }, - { - name: "unselected missing parent", - content: `version: 5 -pipelines: - parent: - input: - module: seriatim - child: - extends: parent - unrelated: - extends: absent -`, - args: []string{"config", "validate", "--pipeline", "child"}, - want: "inheritance.missing", - }, - { - name: "exact sibling mismatch", - content: `version: 5 -pipelines: - parent: - input: - module: seriatim - child: - extends: "parent " -`, - args: []string{"pipelines", "list"}, - want: "inheritance.missing", - }, + for _, test := range []struct{ name, content, fragment string }{ + {"missing parent", "pipelines: {child: {extends: absent}}\n", "inheritance.missing"}, + {"cycle", "pipelines: {first: {extends: second}, second: {extends: first}}\n", "cycle"}, + {"invalid directive", "pipelines: {child: {extends: 7}}\n", "extends"}, + {"unselected missing parent", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: parent}, unrelated: {extends: absent}}\n", "inheritance.missing"}, + {"exact sibling mismatch", "pipelines: {parent: {input: {module: seriatim}}, child: {extends: 'parent '}}\n", "inheritance.missing"}, } { t.Run(test.name, func(t *testing.T) { - t.Parallel() - path := writeCommandConfigContent(t, test.content) - args := append([]string{}, test.args...) - args = append(args, "--config", path) + path := writeCommandConfigContent(t, "version: 6\n"+test.content) var stdout, stderr bytes.Buffer - if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t)) + if code == 0 || !strings.Contains(stderr.String(), test.fragment) { + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } } -func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) { - t.Parallel() - - for _, args := range [][]string{ - {"run", "sample", "--config-profile=development"}, - {"config", "validate", "--config-profile=development"}, - {"config", "effective", "--config-profile=development"}, - {"config", "explain", "--config-profile=development", ""}, - {"pipelines", "list", "--config-profile=development"}, - } { - t.Run(strings.Join(args[:2], " "), func(t *testing.T) { - var stdout, stderr bytes.Buffer - if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "not defined") { - t.Fatalf("args=%q code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) - } - }) - } -} - -func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) { - t.Parallel() - path := filepath.Join(t.TempDir(), "notarius.yml") - writeConfigKitFile(t, path, "version: 4\n") - var stdout, stderr bytes.Buffer - if code := runPipelinesList([]string{"--config", path}, &stdout, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "version") { - t.Fatalf("list = %d, %q", code, stderr.String()) - } -} - -func TestConfigurationInspectionCommandsRenderSafeConfigKitViews(t *testing.T) { +func TestConfigurationScopesUseSelectedSourcesAndNativeSecretMarks(t *testing.T) { t.Parallel() root := t.TempDir() path := filepath.Join(root, "notarius.yml") - writeConfigKitFile(t, path, `version: 5 -concurrency: - total_llm: 4 -pipelines: - "0": - input: {module: input} -`) - writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency:\n total_llm: 3\n") - writeConfigKitFile(t, filepath.Join(root, "overlays", "retired.yml"), "pipelines: null\n") - - providerCalled := false - opts := commandContractOptions(t) - opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - providerCalled = true - return nil, nil, errors.New("LLM client must not be constructed") - } - - var stdout, stderr bytes.Buffer - if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 { - t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String()) - } - if providerCalled { - t.Fatal("config effective constructed an LLM client") - } - - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 { - t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "explain", "--config", path, ""}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: ""`) || stderr.Len() != 0 { - t.Fatalf("explain root = %d, %q, %q", code, stdout.String(), stderr.String()) - } - - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 { - t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String()) - } -} - -func TestConfigExplainRejectsInvalidAndUnknownPointers(t *testing.T) { - t.Parallel() - path := writeCommandConfig(t, "session", "other") - for _, test := range []struct { - name string - pointer string - wantCode int - }{ - {name: "missing", wantCode: 2}, - {name: "malformed escape", pointer: "/pipelines~2session", wantCode: 2}, - {name: "unknown", pointer: "/unknown", wantCode: 1}, - {name: "invalid sequence index", pointer: "/pipelines/session/steps/+0", wantCode: 1}, - } { - t.Run(test.name, func(t *testing.T) { - var stdout, stderr bytes.Buffer - args := []string{"config", "explain", "--config", path} - if test.name != "missing" { - args = append(args, test.pointer) - } - if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != test.wantCode || stdout.Len() != 0 || stderr.Len() == 0 { - t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) - } - }) - } -} - -func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) { - t.Parallel() - path := writeCommandConfig(t, "session", "other") - var stderr bytes.Buffer - if code := RunWithOptions([]string{"config", "effective", "--config", path}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") { - t.Fatalf("effective writer failure = %d, %q", code, stderr.String()) - } -} - -func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) { - t.Parallel() - root := t.TempDir() - configPath := filepath.Join(root, "notarius.yml") - writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n") - writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n") - writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles: - base: - endpoint: https://example.invalid/v1 - model: base-model - extra_params: - Authorization: base-auth-secret - ordinary: visible-value -`) - writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit: + writeConfigKitFile(t, path, "version: 6\nconcurrency: {workers: 4}\npipelines: {demo: {input: {module: seriatim}}}\n") + writeConfigKitFile(t, filepath.Join(root, "prompts.yml"), `config: imports: - - path: base.yml -profiles: - "escaped/id": - model: imported-model - dnd-extraction: - model: imported-override + - path: profiles/base.yml + required: true `) - assets, err := productionPromptAssets() - if err != nil { - t.Fatal(err) - } - factoryCalls := 0 - opts := commandContractOptions(t) - opts.promptKitAssets = assets - opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - factoryCalls++ - return nil, nil, errors.New("inspection must not construct an LLM client") - } - var stdout, stderr bytes.Buffer - if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 { - t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String()) - } - defaultReport := stdout.String() - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 { - t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String()) - } - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 { - t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String()) - } - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 { - t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String()) - } - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 { - t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String()) - } - stdout.Reset() - stderr.Reset() - if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 { - t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String()) - } - if factoryCalls != 0 { - t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls) - } -} - -func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) { - t.Parallel() - path := writeCommandConfig(t, "session", "other") - opts := commandContractOptions(t) - for _, test := range []struct { - name string - args []string - want int - }{ - {name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2}, - {name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2}, - {name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2}, - {name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1}, - } { - t.Run(test.name, func(t *testing.T) { - var stdout, stderr bytes.Buffer - if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 { - t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want) - } - }) - } - var stderr bytes.Buffer - if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") { - t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String()) - } -} - -func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) { - t.Parallel() - root := t.TempDir() - profilePath := filepath.Join(root, "profiles.yml") - writeConfigKitFile(t, profilePath, `backends: - managed: - endpoint: https://provider.example.invalid/v1 - api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY -profiles: - inherited: - backend: managed + writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles: + - id: base + endpoint: https://example.invalid/v1 model: inherited-model - temperature: 0.4 - api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY + api_key_env: NOTARIUS_TEST_KEY_NOT_REQUIRED_FOR_INSPECTION api_key_required: true - extra_params: - Authorization: private-authorization - ordinary: visible-value - selected: - extends: inherited + extra_params: {token: private-test-token, ordinary: visible-value} + - id: selected + extends: {id: base} temperature: 0 extra_params: null - endpoint-only: - endpoint: https://endpoint.example.invalid/v1 - model: endpoint-model - api_key_required: false - " literal profile ": - endpoint: https://literal.example.invalid/v1 - model: literal-model `) - configPath := filepath.Join(root, "notarius.yml") - writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n") - writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n") - assets, err := productionPromptAssets() - if err != nil { - t.Fatal(err) - } - lookupCalls := 0 - factoryCalls := 0 - opts := commandContractOptions(t) - opts.promptKitAssets = assets - opts.LookupEnv = func(string) (string, bool) { - lookupCalls++ - return "", false - } - opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - factoryCalls++ - return nil, nil, errors.New("profile inspection must not construct an LLM client") - } - - type profileReport struct { - ProfileID string `json:"profile_id"` - APIKeyRequired bool `json:"api_key_required"` - Target struct { - BackendID string `json:"backend_id"` - Endpoint string `json:"endpoint"` - Model string `json:"model"` - Temperature float64 `json:"temperature"` - APIKeyEnv string `json:"api_key_env"` - ExtraParams map[string]any `json:"extra_params"` - } `json:"effective_model_params"` - } - run := func(profileID string) (int, string, string, profileReport) { - t.Helper() + writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency: {workers: 3}\n") + client := &diagnosticClient{} + options := commandContractOptions(t) + options.LLMClient = client + baseIndex := -1 + for _, scope := range []string{"app", "promptkit"} { var stdout, stderr bytes.Buffer - code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts) - var report profileReport - if code == 0 { - if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 { - t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String()) + code := RunWithOptions([]string{"config", "show", "--scope", scope, "--config", path, "--overlay", "development", "--format", "json"}, &stdout, &stderr, options) + if code != 0 || !json.Valid(stdout.Bytes()) || !strings.Contains(stdout.String(), "inherited-model") || !strings.Contains(stdout.String(), "visible-value") || strings.Contains(stdout.String(), "private-test-token") { + t.Fatalf("%s status=%d stdout=%q stderr=%q", scope, code, stdout.String(), stderr.String()) + } + var report map[string]any + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) + } + if scope == "app" { + controls, ok := report["concurrency"].(map[string]any) + if !ok || controls["workers"] != float64(3) { + t.Fatalf("overlay missing: %#v", report) } - if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { - t.Fatalf("decode profile JSON %q: %v", stdout.String(), err) + } else { + if report["pipelines"] != nil { + t.Fatalf("focused scope exposed application root: %#v", report) + } + profiles, _ := report["profiles"].([]any) + for index, raw := range profiles { + profile, _ := raw.(map[string]any) + if profile["id"] == "base" { + baseIndex = index + } } } - return code, stdout.String(), stderr.String(), report } - code, output, diagnostic, selected := run("selected") - if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil { - t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected) + if baseIndex < 0 { + t.Fatal("focused configuration omitted the imported profile") } - code, output, diagnostic, inherited := run("inherited") - if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") { - t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited) + for _, args := range [][]string{ + {"config", "sources", "--scope", "promptkit"}, + {"config", "explain", "--scope", "promptkit", "--path", fmt.Sprintf("/promptkit/profiles/%d/model", baseIndex)}, + } { + var stdout, stderr bytes.Buffer + code := RunWithOptions(append(args, "--config", path), &stdout, &stderr, options) + if code != 0 || !strings.Contains(stdout.String(), "base.yml") { + t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } } - code, output, diagnostic, endpointOnly := run("endpoint-only") - if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" { - t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly) + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"promptkit", "show", "profile", "selected", "--config", path, "--format", "json"}, &stdout, &stderr, options) + if code != 0 || !strings.Contains(stdout.String(), "inherited-model") || strings.Contains(stdout.String(), "private-test-token") { + t.Fatalf("profile status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } - code, output, diagnostic, literal := run(" literal profile ") - if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" { - t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal) - } - if lookupCalls != 0 || factoryCalls != 0 { - t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls) + if client.calls.Load() != 0 { + t.Fatal("configuration/profile inspection invoked provider") } } -func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) { +func TestConfigurationCommandsRetainNativeInputAndPresentationFailures(t *testing.T) { t.Parallel() - configPath := writeCommandConfig(t, "session", "other") - for _, test := range []struct { - name string - args []string - want int - }{ - {name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2}, - {name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2}, - {name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2}, - {name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1}, - } { - t.Run(test.name, func(t *testing.T) { - var stdout, stderr bytes.Buffer - if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 { - t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want) - } - }) + for _, scope := range []string{"app", "promptkit"} { + var stdout, stderr bytes.Buffer + missing := filepath.Join(t.TempDir(), "missing.yml") + code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", missing, "--path", "/profiles/~2"}, &stdout, &stderr, commandContractOptions(t)) + if code != 2 || stdout.Len() != 0 || strings.Contains(stderr.String(), missing) { + t.Fatalf("invalid path reached loading: status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } } - profilePath := filepath.Join(t.TempDir(), "profiles.yml") - writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n") - configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n") - var stderr bytes.Buffer - if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") { - t.Fatalf("profile writer failure = %d, %q", code, stderr.String()) + path := writeCommandConfig(t, "session", "other") + for _, args := range [][]string{ + {"config", "show"}, {"config", "show", "--scope", "promptkit"}, {"promptkit", "show", "profile", "notarius/dnd-extraction"}, + } { + var stderr bytes.Buffer + code := RunWithOptions(append(args, "--config", path), failingInspectionWriter{}, &stderr, commandContractOptions(t)) + if code != 1 { + t.Fatalf("writer failure %v: status=%d stderr=%q", args, code, stderr.String()) + } } } type failingInspectionWriter struct{} -func (failingInspectionWriter) Write([]byte) (int, error) { - return 0, io.ErrClosedPipe -} +func (failingInspectionWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe } diff --git a/internal/cli/configkit_load_test.go b/internal/cli/configkit_load_test.go index 4a619493..55da3a95 100644 --- a/internal/cli/configkit_load_test.go +++ b/internal/cli/configkit_load_test.go @@ -2,6 +2,7 @@ package cli import ( "bytes" + "context" "os" "path/filepath" "runtime" @@ -9,7 +10,10 @@ import ( "strings" "testing" - configkit "go.maximumdirect.net/configkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/configkit" ) func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) { @@ -22,41 +26,44 @@ func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) { args := []string{"config", "validate", "--config", main} if overlay { fragment = filepath.Join(root, "overlays", "development.yml") - writeConfigKitFile(t, main, "version: 5\n") - args = append(args, "--config-overlay", "development") + writeConfigKitFile(t, main, "version: 6\n") + args = append(args, "--overlay", "development") } else { - writeConfigKitFile(t, main, "version: 5\nconfig:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n required: true\n") + writeConfigKitFile(t, main, "version: 6\nconfig:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n required: true\n") } - writeConfigKitFile(t, fragment, "concurrency: {total_llm: invalid-private-value}\n") + writeConfigKitFile(t, fragment, "concurrency: {workers: invalid-private-value}\n") var stdout, stderr bytes.Buffer - if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 { - t.Fatalf("status = %d; %s", code, stderr.String()) - } + code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)) got := stderr.String() - if stdout.Len() != 0 || !strings.Contains(got, "decode.type") || - !strings.Contains(got, "source-2:1:") || - !strings.Contains(got, strconv.Quote(fragment)) || - strings.Contains(got, "invalid-private-value") || - strings.ContainsRune(got, '\x1b') { - t.Fatalf("stdout = %q; stderr = %q", stdout.String(), got) + if code == 0 || !strings.Contains(got, "decode.type") || !strings.Contains(got, strconv.Quote(fragment)) || strings.Contains(got, "invalid-private-value") || strings.ContainsRune(got, '\x1b') { + t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), got) } }) } } -func TestLoadConfigKitUsesDiscoveryOverlayAndOverrides(t *testing.T) { +func TestApplicationLoadUsesOverlayAndOverrides(t *testing.T) { t.Parallel() root := t.TempDir() path := filepath.Join(root, "notarius.yml") - writeConfigKitFile(t, path, "version: 5\npipelines:\n base:\n input: {module: input}\n") - overlays := filepath.Join(root, "overlays") - writeConfigKitFile(t, filepath.Join(overlays, "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n") - loaded, gotPath, err := loadConfigKit(path, "development", []configkit.Override{{Path: configkit.Keys("output", "directory"), Value: "generated", Flag: "--output"}}, commandContractOptions(t)) + writeConfigKitFile(t, path, "version: 6\npipelines: {base: {input: {module: input}}}\n") + writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines: {overlaid: {input: {module: input}}}\n") + selection := productionSelection(path) + selection.Overlay = commandkit.Supplied[string]{Set: true, Value: "development"} + selectedDirectory := filepath.Join(root, "generated") + selection.Overrides = []configkit.Override{{Path: configkit.Keys("output", "directory"), Value: selectedDirectory, Flag: "--set"}} + loaders := &nativeLoaders{lookupEnv: emptyLookup} + result, err := loaders.application(context.Background(), selection) if err != nil { - t.Fatalf("loadConfigKit() error = %v", err) + t.Fatal(err) } - if gotPath != path || loaded.Config.Pipelines["overlaid"].Input.Module != "input" || loaded.Config.Output.Directory == "generated" { - t.Fatalf("loaded = %#v, path=%q", loaded.Config, gotPath) + loaded, err := config.FromResult(result) + if err != nil { + t.Fatal(err) + } + cfg := loaded.RuntimeConfig() + if cfg.Pipelines["overlaid"].Input.Module != "input" || cfg.Output.Directory != selectedDirectory { + t.Fatalf("loaded configuration=%#v", cfg) } } @@ -76,84 +83,82 @@ func TestNativeDiscoverySelectionAndTerminalFailures(t *testing.T) { default: t.Setenv("XDG_CONFIG_HOME", user) } - previous, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - if err := os.Chdir(work); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - if err := os.Chdir(previous); err != nil { - t.Error(err) - } - }) + t.Chdir(work) userFile := filepath.Join(user, "notarius", "config.yml") - writeConfigKitFile(t, userFile, "version: 5\npipelines: {user: {input: {module: input}}}\n") - writeConfigKitFile(t, filepath.Join(work, "config.yml"), "version: 5\npipelines: {generic: {input: {module: input}}}\n") - opts := commandContractOptions(t) - loaded, path, err := loadConfigKit("", "", nil, opts) - if err != nil || path != userFile || loaded.Config.Pipelines["user"].Input.Module != "input" { - t.Fatalf("user discovery = %q, %v", path, err) + writeConfigKitFile(t, userFile, "version: 6\npipelines: {user: {input: {module: input}}}\n") + writeConfigKitFile(t, filepath.Join(work, "config.yml"), "version: 6\npipelines: {generic: {input: {module: input}}}\n") + load := func(selection ckconfig.Selection, lookup func(string) (string, bool)) (*configkit.Result[config.Document], error) { + loaders := &nativeLoaders{lookupEnv: lookup} + return loaders.application(context.Background(), selection) } - workingYAML := filepath.Join(work, "notarius.yaml") - workingYML := filepath.Join(work, "notarius.yml") - writeConfigKitFile(t, workingYAML, "version: 5\npipelines: {yaml: {input: {module: input}}}\n") - writeConfigKitFile(t, workingYML, "version: 5\npipelines: {yml: {input: {module: input}}}\n") - loaded, path, err = loadConfigKit("", "", nil, opts) - if err != nil || path != workingYML || loaded.Config.Pipelines["yml"].Input.Module != "input" || loaded.Config.Output.Directory != filepath.Join(work, "notarius-output") { - t.Fatalf("working discovery = %q, %v", path, err) - } - for _, args := range [][]string{{"pipelines", "list"}, {"config", "validate"}, {"config", "effective"}, {"config", "explain", "/version"}} { - var stdout, stderr bytes.Buffer - if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 || stdout.Len() == 0 { - t.Fatalf("%v: code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + assertSelection := func(wantPath, wantID string) { + t.Helper() + result, err := load(ckconfig.Selection{}, emptyLookup) + if err != nil { + t.Fatal(err) + } + found := false + for _, source := range result.Sources() { + if source.Layer == configkit.LayerMain && source.Path == wantPath { + found = true + } + } + if !found || result.Config.Pipelines[wantID].Input.Module != "input" { + t.Fatalf("discovery missed %s/%s: sources=%#v config=%#v", wantPath, wantID, result.Sources(), result.Config.Pipelines) } } - missingEnv := opts - missingEnv.LookupEnv = func(name string) (string, bool) { return filepath.Join(root, "missing.yml"), name == "NOTARIUS_CONFIG" } - if _, _, err := loadConfigKit("", "", nil, missingEnv); err == nil { + assertSelection(userFile, "user") + workingYAML, workingYML := filepath.Join(work, "notarius.yaml"), filepath.Join(work, "notarius.yml") + writeConfigKitFile(t, workingYAML, "version: 6\npipelines: {yaml: {input: {module: input}}}\n") + writeConfigKitFile(t, workingYML, "version: 6\npipelines: {yml: {input: {module: input}}}\n") + assertSelection(workingYML, "yml") + for _, args := range [][]string{{"pipeline", "list"}, {"config", "validate"}, {"config", "show"}, {"config", "explain", "--path", "/version"}} { + var stdout, stderr bytes.Buffer + if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.Len() == 0 { + t.Fatalf("%v status=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + } + missingEnv := func(name string) (string, bool) { return filepath.Join(root, "missing.yml"), name == "NOTARIUS_CONFIG" } + if _, err := load(ckconfig.Selection{}, missingEnv); err == nil { t.Fatal("missing environment selection fell back") } if err := os.WriteFile(workingYML, []byte("version: [\n"), 0600); err != nil { t.Fatal(err) } - if _, _, err := loadConfigKit("", "", nil, opts); err == nil { + if _, err := load(ckconfig.Selection{}, emptyLookup); err == nil { t.Fatal("invalid selected file fell back") } - if _, _, err := loadConfigKit(filepath.Join(root, "missing.yml"), "", nil, opts); err == nil { - t.Fatal("missing explicit file fell back") + if _, err := load(productionSelection(filepath.Join(root, "missing.yml")), emptyLookup); err == nil { + t.Fatal("missing explicit selection fell back") } if err := os.Remove(workingYML); err != nil { t.Fatal(err) } - loaded, path, err = loadConfigKit("", "", nil, opts) - if err != nil || path != workingYAML || loaded.Config.Pipelines["yaml"].Input.Module != "input" { - t.Fatalf("alternate named file = %q, %v", path, err) - } + assertSelection(workingYAML, "yaml") } func TestWriteConfigLoadErrorUsesSafeDiagnostics(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "invalid.yml") - writeConfigKitFile(t, path, "version: 4\n") - _, _, err := loadConfigKit(path, "", nil, commandContractOptions(t)) + writeConfigKitFile(t, path, "version: 5\n") + loaders := &nativeLoaders{lookupEnv: emptyLookup} + _, err := loaders.application(context.Background(), productionSelection(path)) if err == nil { - t.Fatal("loadConfigKit() error = nil") + t.Fatal("unsupported version loaded") } var output bytes.Buffer writeConfigLoadError(&output, err) - if !strings.Contains(output.String(), "version") || strings.Contains(output.String(), "notarius: configkit") { - t.Fatalf("diagnostics = %q", output.String()) + if !strings.Contains(output.String(), "version") || !strings.Contains(output.String(), path) { + t.Fatalf("diagnostics=%q", output.String()) } } func writeConfigKitFile(t *testing.T, path, contents string) { t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { t.Fatal(err) } - if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + if err := os.WriteFile(path, []byte(contents), 0600); err != nil { t.Fatal(err) } } diff --git a/internal/cli/native_load_test.go b/internal/cli/native_load_test.go index 091ca29c..b3d6769b 100644 --- a/internal/cli/native_load_test.go +++ b/internal/cli/native_load_test.go @@ -54,7 +54,11 @@ func TestNativeLoadUsesOptionalCompanionsAndStrictPeers(t *testing.T) { t.Fatal("embedded collection missing") } if test.name == "recursive" { - if len(result.Config.PromptKit.Profiles) != 1 || result.Config.PromptKit.Profiles[0].ID != "partial" { + found := false + for _, profile := range result.Config.PromptKit.Profiles { + found = found || profile.ID == "partial" + } + if !found { t.Fatal("recursive companion not loaded") } } diff --git a/internal/cli/promptkit_commands_test.go b/internal/cli/promptkit_commands_test.go index 7bc13bc6..883661b0 100644 --- a/internal/cli/promptkit_commands_test.go +++ b/internal/cli/promptkit_commands_test.go @@ -75,7 +75,7 @@ promptkit: if client.calls.Load() != 0 { t.Fatal("inspection implicitly called provider") } - outcome, out, diagnostics := execute("promptkit", "test", "text-inference", "--llm-profile=ready") + outcome, out, diagnostics := execute("promptkit", "test", string(promptkit.SmokeTestTextInference), "--llm-profile=ready") if outcome.Status != 0 || client.calls.Load() == 0 { t.Fatalf("explicit smoke: %#v, %s, %s", outcome, out, diagnostics) }