Test CLI inheritance inspection and use
This commit is contained in:
@@ -206,7 +206,13 @@ No new field ranges, secret handling, or source/path policies appear.
|
|||||||
|
|
||||||
## Stage 3: Protect CLI Inspection And Promptkit Engine Adoption
|
## Stage 3: Protect CLI Inspection And Promptkit Engine Adoption
|
||||||
|
|
||||||
Status: Pending. Depends on Stage 2.
|
Status: Complete. Depends on Stage 2.
|
||||||
|
|
||||||
|
Completion evidence: CLI coverage passes for inheritance-free structural
|
||||||
|
inspection, inherited and replaced values, provenance, source DTO stability,
|
||||||
|
safe reserved-key diagnostics, render preparation through the inherited child,
|
||||||
|
and Promptkit validation of a configured parent backend. Focused and full tests,
|
||||||
|
race tests, vet, build, and `git diff --check` passed.
|
||||||
|
|
||||||
Files: `internal/adapter/cli/config_test.go` and
|
Files: `internal/adapter/cli/config_test.go` and
|
||||||
`internal/adapter/cli/run_test.go`; CLI production code only if an actual
|
`internal/adapter/cli/run_test.go`; CLI production code only if an actual
|
||||||
|
|||||||
@@ -73,6 +73,92 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigCommandsInspectInheritedBackends(t *testing.T) {
|
||||||
|
const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY"
|
||||||
|
const keyValue = "synthetic-inheritance-key-bytes"
|
||||||
|
path := writeAppConfigFile(t, `backends:
|
||||||
|
base:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 4
|
||||||
|
extra_params:
|
||||||
|
enabled: true
|
||||||
|
shared: visible-value
|
||||||
|
child:
|
||||||
|
extends: base
|
||||||
|
queue_capacity: 0
|
||||||
|
extra_params:
|
||||||
|
enabled: false
|
||||||
|
`)
|
||||||
|
t.Setenv(keyName, keyValue)
|
||||||
|
if err := os.Unsetenv(keyName); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||||
|
if code != ExitOK || stdout != "" || stderr != "" {
|
||||||
|
t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
if err := os.Setenv(keyName, keyValue); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path})
|
||||||
|
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||||
|
t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
for _, fragment := range []string{
|
||||||
|
"endpoint: http://127.0.0.1:1/v1",
|
||||||
|
"api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY",
|
||||||
|
"shared: visible-value",
|
||||||
|
"enabled: false",
|
||||||
|
"queue_capacity: 0",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(stdout, fragment) {
|
||||||
|
t.Fatalf("config show omitted %q: %s", fragment, stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(stdout, "extends:") {
|
||||||
|
t.Fatalf("config show exposed the structural directive: %s", stdout)
|
||||||
|
}
|
||||||
|
|
||||||
|
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path})
|
||||||
|
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||||
|
t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} {
|
||||||
|
if !strings.Contains(stdout, fragment) {
|
||||||
|
t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path})
|
||||||
|
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) {
|
||||||
|
t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path})
|
||||||
|
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||||
|
t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
var sources []configurationSource
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" {
|
||||||
|
t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) {
|
||||||
|
path := writeAppConfigFile(t, `backends:
|
||||||
|
worker:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
extra_params:
|
||||||
|
extends: missing
|
||||||
|
`)
|
||||||
|
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||||
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") {
|
||||||
|
t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) {
|
func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) {
|
||||||
const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY"
|
const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY"
|
||||||
const secret = "synthetic-secret-value-should-not-appear"
|
const secret = "synthetic-secret-value-should-not-appear"
|
||||||
|
|||||||
@@ -1148,6 +1148,59 @@ backends:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
||||||
|
version: "1"
|
||||||
|
default_profile: local-gpu
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hello"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
|
||||||
|
backend: local-gpu
|
||||||
|
model: local-model
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||||
|
prompt_dir: %s
|
||||||
|
profile_dir: %s
|
||||||
|
backends:
|
||||||
|
local-base:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
|
concurrency_limit: 2
|
||||||
|
local-gpu:
|
||||||
|
extends: local-base
|
||||||
|
queue_capacity: 0
|
||||||
|
extra_params:
|
||||||
|
provider_option: false
|
||||||
|
`, lib.promptDir, lib.profileDir))
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", configPath,
|
||||||
|
"--prompt", "custom",
|
||||||
|
"--format", "json",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected inherited backend preparation to succeed, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
var prepared promptkit.PreparedRun
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &prepared); err != nil {
|
||||||
|
t.Fatalf("decode prepared output: %v\n%s", err, stdout)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != "local-gpu" || prepared.EffectiveModelParams.BackendID != "local-gpu" {
|
||||||
|
t.Fatalf("prepared run did not select the child backend: %+v", prepared)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" {
|
||||||
|
t.Fatalf("prepared run did not use the inherited endpoint: %+v", prepared.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
|
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
||||||
@@ -1974,6 +2027,27 @@ backends:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
||||||
|
prompt_dir: %s
|
||||||
|
profile_dir: %s
|
||||||
|
backends:
|
||||||
|
base:
|
||||||
|
endpoint: not-a-url
|
||||||
|
child:
|
||||||
|
extends: base
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
`, lib.promptDir, lib.profileDir))
|
||||||
|
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := newEngine(cfg.runConfig.engineSettings()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("invalid parent backend was not validated by Promptkit: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
|
|||||||
Reference in New Issue
Block a user