Test CLI inheritance inspection and use

This commit is contained in:
2026-10-03 17:40:35 +00:00
parent ea964707f7
commit 70f7e96d02
3 changed files with 167 additions and 1 deletions

View File

@@ -73,6 +73,92 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
}
}
func TestConfigCommandsInspectInheritedBackends(t *testing.T) {
const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY"
const keyValue = "synthetic-inheritance-key-bytes"
path := writeAppConfigFile(t, `backends:
base:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY
concurrency_limit: 2
queue_capacity: 4
extra_params:
enabled: true
shared: visible-value
child:
extends: base
queue_capacity: 0
extra_params:
enabled: false
`)
t.Setenv(keyName, keyValue)
if err := os.Unsetenv(keyName); err != nil {
t.Fatal(err)
}
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
if code != ExitOK || stdout != "" || stderr != "" {
t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr)
}
if err := os.Setenv(keyName, keyValue); err != nil {
t.Fatal(err)
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path})
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
}
for _, fragment := range []string{
"endpoint: http://127.0.0.1:1/v1",
"api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY",
"shared: visible-value",
"enabled: false",
"queue_capacity: 0",
} {
if !strings.Contains(stdout, fragment) {
t.Fatalf("config show omitted %q: %s", fragment, stdout)
}
}
if strings.Contains(stdout, "extends:") {
t.Fatalf("config show exposed the structural directive: %s", stdout)
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path})
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
}
for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} {
if !strings.Contains(stdout, fragment) {
t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout)
}
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path})
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) {
t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path})
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr)
}
var sources []configurationSource
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" {
t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err)
}
}
func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) {
path := writeAppConfigFile(t, `backends:
worker:
endpoint: http://127.0.0.1:1/v1
extra_params:
extends: missing
`)
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") {
t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr)
}
}
func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) {
const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY"
const secret = "synthetic-secret-value-should-not-appear"

View File

@@ -1148,6 +1148,59 @@ backends:
}
}
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
version: "1"
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
backend: local-gpu
model: local-model
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
local-base:
endpoint: http://localhost:11434/v1
api_key_env: LOCAL_GPU_API_KEY
concurrency_limit: 2
local-gpu:
extends: local-base
queue_capacity: 0
extra_params:
provider_option: false
`, lib.promptDir, lib.profileDir))
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
"--prompt", "custom",
"--format", "json",
})
if code != ExitOK {
t.Fatalf("expected inherited backend preparation to succeed, got %d stderr=%q", code, stderr)
}
var prepared promptkit.PreparedRun
if err := json.Unmarshal([]byte(stdout), &prepared); err != nil {
t.Fatalf("decode prepared output: %v\n%s", err, stdout)
}
if prepared.SelectedBackendID != "local-gpu" || prepared.EffectiveModelParams.BackendID != "local-gpu" {
t.Fatalf("prepared run did not select the child backend: %+v", prepared)
}
if prepared.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" {
t.Fatalf("prepared run did not use the inherited endpoint: %+v", prepared.EffectiveModelParams)
}
}
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
@@ -1974,6 +2027,27 @@ backends:
}
}
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
lib := newCLITestLibrary(t)
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
base:
endpoint: not-a-url
child:
extends: base
endpoint: http://localhost:11434/v1
`, lib.promptDir, lib.profileDir))
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
if err != nil {
t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err)
}
if _, err := newEngine(cfg.runConfig.engineSettings()); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("invalid parent backend was not validated by Promptkit: %v", err)
}
}
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")