Test CLI inheritance inspection and use
This commit is contained in:
@@ -73,6 +73,92 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigCommandsInspectInheritedBackends(t *testing.T) {
|
||||
const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY"
|
||||
const keyValue = "synthetic-inheritance-key-bytes"
|
||||
path := writeAppConfigFile(t, `backends:
|
||||
base:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 4
|
||||
extra_params:
|
||||
enabled: true
|
||||
shared: visible-value
|
||||
child:
|
||||
extends: base
|
||||
queue_capacity: 0
|
||||
extra_params:
|
||||
enabled: false
|
||||
`)
|
||||
t.Setenv(keyName, keyValue)
|
||||
if err := os.Unsetenv(keyName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||
if code != ExitOK || stdout != "" || stderr != "" {
|
||||
t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
if err := os.Setenv(keyName, keyValue); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path})
|
||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
for _, fragment := range []string{
|
||||
"endpoint: http://127.0.0.1:1/v1",
|
||||
"api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY",
|
||||
"shared: visible-value",
|
||||
"enabled: false",
|
||||
"queue_capacity: 0",
|
||||
} {
|
||||
if !strings.Contains(stdout, fragment) {
|
||||
t.Fatalf("config show omitted %q: %s", fragment, stdout)
|
||||
}
|
||||
}
|
||||
if strings.Contains(stdout, "extends:") {
|
||||
t.Fatalf("config show exposed the structural directive: %s", stdout)
|
||||
}
|
||||
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path})
|
||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} {
|
||||
if !strings.Contains(stdout, fragment) {
|
||||
t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout)
|
||||
}
|
||||
}
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path})
|
||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path})
|
||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
var sources []configurationSource
|
||||
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" {
|
||||
t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) {
|
||||
path := writeAppConfigFile(t, `backends:
|
||||
worker:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
extra_params:
|
||||
extends: missing
|
||||
`)
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") {
|
||||
t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) {
|
||||
const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY"
|
||||
const secret = "synthetic-secret-value-should-not-appear"
|
||||
|
||||
@@ -1148,6 +1148,59 @@ backends:
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
||||
version: "1"
|
||||
default_profile: local-gpu
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
|
||||
backend: local-gpu
|
||||
model: local-model
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||
prompt_dir: %s
|
||||
profile_dir: %s
|
||||
backends:
|
||||
local-base:
|
||||
endpoint: http://localhost:11434/v1
|
||||
api_key_env: LOCAL_GPU_API_KEY
|
||||
concurrency_limit: 2
|
||||
local-gpu:
|
||||
extends: local-base
|
||||
queue_capacity: 0
|
||||
extra_params:
|
||||
provider_option: false
|
||||
`, lib.promptDir, lib.profileDir))
|
||||
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", "custom",
|
||||
"--format", "json",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected inherited backend preparation to succeed, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
var prepared promptkit.PreparedRun
|
||||
if err := json.Unmarshal([]byte(stdout), &prepared); err != nil {
|
||||
t.Fatalf("decode prepared output: %v\n%s", err, stdout)
|
||||
}
|
||||
if prepared.SelectedBackendID != "local-gpu" || prepared.EffectiveModelParams.BackendID != "local-gpu" {
|
||||
t.Fatalf("prepared run did not select the child backend: %+v", prepared)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" {
|
||||
t.Fatalf("prepared run did not use the inherited endpoint: %+v", prepared.EffectiveModelParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
||||
@@ -1974,6 +2027,27 @@ backends:
|
||||
}
|
||||
}
|
||||
|
||||
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||
prompt_dir: %s
|
||||
profile_dir: %s
|
||||
backends:
|
||||
base:
|
||||
endpoint: not-a-url
|
||||
child:
|
||||
extends: base
|
||||
endpoint: http://localhost:11434/v1
|
||||
`, lib.promptDir, lib.profileDir))
|
||||
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||
if err != nil {
|
||||
t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err)
|
||||
}
|
||||
if _, err := newEngine(cfg.runConfig.engineSettings()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("invalid parent backend was not validated by Promptkit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||
|
||||
Reference in New Issue
Block a user