From 70f7e96d023eb6602c6078c444169741fa975ba3 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 3 Oct 2026 17:40:35 +0000 Subject: [PATCH] Test CLI inheritance inspection and use --- docs/roadmap/implementation.md | 8 ++- internal/adapter/cli/config_test.go | 86 +++++++++++++++++++++++++++++ internal/adapter/cli/run_test.go | 74 +++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2aa4609..3b454f2 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -206,7 +206,13 @@ No new field ranges, secret handling, or source/path policies appear. ## Stage 3: Protect CLI Inspection And Promptkit Engine Adoption -Status: Pending. Depends on Stage 2. +Status: Complete. Depends on Stage 2. + +Completion evidence: CLI coverage passes for inheritance-free structural +inspection, inherited and replaced values, provenance, source DTO stability, +safe reserved-key diagnostics, render preparation through the inherited child, +and Promptkit validation of a configured parent backend. Focused and full tests, +race tests, vet, build, and `git diff --check` passed. Files: `internal/adapter/cli/config_test.go` and `internal/adapter/cli/run_test.go`; CLI production code only if an actual diff --git a/internal/adapter/cli/config_test.go b/internal/adapter/cli/config_test.go index 9bb5a05..452a308 100644 --- a/internal/adapter/cli/config_test.go +++ b/internal/adapter/cli/config_test.go @@ -73,6 +73,92 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) { } } +func TestConfigCommandsInspectInheritedBackends(t *testing.T) { + const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY" + const keyValue = "synthetic-inheritance-key-bytes" + path := writeAppConfigFile(t, `backends: + base: + endpoint: http://127.0.0.1:1/v1 + api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY + concurrency_limit: 2 + queue_capacity: 4 + extra_params: + enabled: true + shared: visible-value + child: + extends: base + queue_capacity: 0 + extra_params: + enabled: false +`) + t.Setenv(keyName, keyValue) + if err := os.Unsetenv(keyName); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path}) + if code != ExitOK || stdout != "" || stderr != "" { + t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr) + } + if err := os.Setenv(keyName, keyValue); err != nil { + t.Fatal(err) + } + + code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path}) + if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) { + t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr) + } + for _, fragment := range []string{ + "endpoint: http://127.0.0.1:1/v1", + "api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY", + "shared: visible-value", + "enabled: false", + "queue_capacity: 0", + } { + if !strings.Contains(stdout, fragment) { + t.Fatalf("config show omitted %q: %s", fragment, stdout) + } + } + if strings.Contains(stdout, "extends:") { + t.Fatalf("config show exposed the structural directive: %s", stdout) + } + + code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path}) + if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) { + t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr) + } + for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} { + if !strings.Contains(stdout, fragment) { + t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout) + } + } + code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path}) + if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) { + t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr) + } + + code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path}) + if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) { + t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr) + } + var sources []configurationSource + if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" { + t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err) + } +} + +func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) { + path := writeAppConfigFile(t, `backends: + worker: + endpoint: http://127.0.0.1:1/v1 + extra_params: + extends: missing +`) + code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path}) + if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") { + t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr) + } +} + func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) { const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY" const secret = "synthetic-secret-value-should-not-appear" diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 6063e6d..b3a3586 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -1148,6 +1148,59 @@ backends: } } +func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) { + lib := newCLITestLibrary(t) + writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom +version: "1" +default_profile: local-gpu +messages: + - role: user + content: "hello" +output: + format: text + validation_mode: none +`) + if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu +backend: local-gpu +model: local-model +`), 0o644); err != nil { + t.Fatalf("write profile fixture: %v", err) + } + configPath := writeAppConfigFile(t, fmt.Sprintf(` +prompt_dir: %s +profile_dir: %s +backends: + local-base: + endpoint: http://localhost:11434/v1 + api_key_env: LOCAL_GPU_API_KEY + concurrency_limit: 2 + local-gpu: + extends: local-base + queue_capacity: 0 + extra_params: + provider_option: false +`, lib.promptDir, lib.profileDir)) + + code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", configPath, + "--prompt", "custom", + "--format", "json", + }) + if code != ExitOK { + t.Fatalf("expected inherited backend preparation to succeed, got %d stderr=%q", code, stderr) + } + var prepared promptkit.PreparedRun + if err := json.Unmarshal([]byte(stdout), &prepared); err != nil { + t.Fatalf("decode prepared output: %v\n%s", err, stdout) + } + if prepared.SelectedBackendID != "local-gpu" || prepared.EffectiveModelParams.BackendID != "local-gpu" { + t.Fatalf("prepared run did not select the child backend: %+v", prepared) + } + if prepared.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" { + t.Fatalf("prepared run did not use the inherited endpoint: %+v", prepared.EffectiveModelParams) + } +} + func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) { lib := newCLITestLibrary(t) writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom @@ -1974,6 +2027,27 @@ backends: } } +func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) { + lib := newCLITestLibrary(t) + configPath := writeAppConfigFile(t, fmt.Sprintf(` +prompt_dir: %s +profile_dir: %s +backends: + base: + endpoint: not-a-url + child: + extends: base + endpoint: http://localhost:11434/v1 +`, lib.promptDir, lib.profileDir)) + cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"}) + if err != nil { + t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err) + } + if _, err := newEngine(cfg.runConfig.engineSettings()); !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("invalid parent backend was not validated by Promptkit: %v", err) + } +} + func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")