Migrate CLI execution tests to native definitions

This commit is contained in:
2026-10-04 12:15:10 +00:00
parent 54c8e9011e
commit 5c302f18ac
2 changed files with 360 additions and 239 deletions

View File

@@ -3,8 +3,8 @@
## Status, Baseline, And Authority
Planned on 2026-10-04 against Scriptorium
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–5 are complete; stages
6–13 are pending.
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–6 are complete; stages
7–13 are pending.
This plan is for a Luna coding agent: implement the stages in sequential order,
normally one stage per implementation prompt. The
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
@@ -381,7 +381,14 @@ are protected without imposing filesystem validation on configured YAML paths.
## Stage 6: Migrate Existing CLI Execution And Definition-Inspection Coverage
Status: Pending. Depends on Stage 5.
Status: Complete. CLI run, render, report, deadline, and definition-inspection
fixtures now use native keyed definitions and ordered framework sources. Backend
definitions live in framework files, built-in backend overlays are accepted,
and tests retain source precedence, retries, version selection, required and
optional inputs, profile inheritance, provider data, and inspection boundaries.
`go test ./internal/adapter/cli ./internal/format` passes, as does the native
build. Repository-wide test, race, and vet checks stop only on removed
PromptKit constructors in HTTP/report fixtures assigned to Stage 7.
Files: remaining execution/inspection sections in
`internal/adapter/cli/run_test.go`; formatting fixtures only if affected.

View File

@@ -886,14 +886,20 @@ func TestDetermineExitCode(t *testing.T) {
}
func TestRunCommandVarsOptional(t *testing.T) {
lib := newCLITestLibrary(t)
configPath := writeAppConfigFile(t, "{}\n")
writePromptFile(t, lib.promptDir, "p", "local-default")
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "m")
inputPath := lib.writeInputFile(t, "t.md", "hello")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := runCommand([]string{
"--promptkit-source", "./prompts",
"--promptkit-source", "./profiles",
"--config", configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "p",
"--input", "transcript=./t.md",
"--input", "transcript=" + inputPath,
"--llm-base-url", "http://[::1",
"--model", "m",
}, &stdout, &stderr)
@@ -921,10 +927,7 @@ func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
`, lib.promptDir, lib.profileDir))
configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, ""))
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--config", configPath,
@@ -951,6 +954,7 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", writeAppConfigFile(t, "{}\n"),
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
@@ -1003,16 +1007,18 @@ func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
profile := `id: local-default
endpoint: http://127.0.0.1:1/v1
model: profile-model
temperature: 0.7
profile := `profiles:
local-default:
endpoint: http://127.0.0.1:1/v1
model: profile-model
temperature: 0.7
`
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
t.Fatalf("failed to write profile fixture: %v", err)
}
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
@@ -1033,10 +1039,7 @@ func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
`, lib.promptDir, lib.profileDir))
configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, ""))
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
@@ -1065,7 +1068,7 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
base := filepath.Join(lib.rootDir, "config.yml")
for path, body := range map[string]string{
base: "configkit:\n imports:\n - path: fragments/sources.yml\n",
filepath.Join(lib.rootDir, "fragments", "sources.yml"): fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\n", lib.promptDir, lib.profileDir),
filepath.Join(lib.rootDir, "fragments", "sources.yml"): sourceSelectionsYAML(lib.promptDir, lib.profileDir, ""),
filepath.Join(lib.rootDir, "overlays", "local.yml"): "defaults:\n render_format: json\n",
} {
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
@@ -1088,33 +1091,32 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
version: "1"
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
custom:
"1":
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
backend: local-gpu
model: local-model
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
local-gpu:
backend: local-gpu
model: local-model
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
configPath := writeCLIFrameworkConfig(t, lib, `backends:
local-gpu:
endpoint: http://localhost:11434/v1
extra_params:
provider_option: enabled
concurrency_limit: 2
queue_capacity: 0
`, lib.promptDir, lib.profileDir))
`, "")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
@@ -1130,26 +1132,25 @@ backends:
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
version: "1"
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
custom:
"1":
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
backend: local-gpu
model: local-model
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
local-gpu:
backend: local-gpu
model: local-model
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
configPath := writeCLIFrameworkConfig(t, lib, `backends:
local-base:
endpoint: http://localhost:11434/v1
api_key_env: LOCAL_GPU_API_KEY
@@ -1159,7 +1160,7 @@ backends:
queue_capacity: 0
extra_params:
provider_option: false
`, lib.promptDir, lib.profileDir))
`, "")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
@@ -1183,17 +1184,18 @@ backends:
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
version: "1"
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
custom:
"1":
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("id: local-gpu\nbackend: local-gpu\nmodel: local-model\n"), 0o644); err != nil {
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n local-gpu:\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil {
t.Fatal(err)
}
const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY"
@@ -1216,15 +1218,13 @@ output:
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"generated"}}]}`))
}))
defer provider.Close()
configPath := writeAppConfigFile(t, fmt.Sprintf(`prompt_dir: %q
profile_dir: %q
backends:
configPath := writeCLIFrameworkConfig(t, lib, fmt.Sprintf(`backends:
local-gpu:
endpoint: %q
api_key_env: %s
extra_params:
synthetic_count: 7
`, lib.promptDir, lib.profileDir, provider.URL+"/v1", credentialName))
`, provider.URL+"/v1", credentialName), "")
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"})
if err != nil {
t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err)
@@ -1241,21 +1241,23 @@ backends:
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
version: "1"
default_profile: local
session_id: definition-session
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
session:
"1":
default_profile: local
session_id: definition-session
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`id: local
endpoint: http://127.0.0.1:1/v1
model: local-model
reasoning_effort: low
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`profiles:
local:
endpoint: http://127.0.0.1:1/v1
model: local-model
reasoning_effort: low
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
@@ -1273,7 +1275,7 @@ reasoning_effort: low
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
} {
t.Run(tc.name, func(t *testing.T) {
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"}
args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"}
args = append(args, tc.args...)
code, stdout, stderr := runCLICommand(t, renderCommand, args)
if code != ExitOK {
@@ -1298,19 +1300,21 @@ reasoning_effort: low
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "plain.yaml", `id: plain
version: "1"
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "plain.yaml", `prompts:
plain:
"1":
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "plain",
@@ -1325,19 +1329,21 @@ output:
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
version: "1"
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
session:
"1":
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
code, _, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "session",
@@ -1353,23 +1359,24 @@ output:
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect
version: "1"
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `prompts:
inspect:
"1":
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
t.Fatalf("unexpected inspection output: %s", stdout)
}
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "missing"})
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "missing"})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -1395,7 +1402,7 @@ func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) {
func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
@@ -1465,7 +1472,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
}))
defer provider.Close()
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\ntransient_retries: 1\n", lib.promptDir, lib.profileDir))
configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")+"transient_retries: 1\n")
args := []string{"--config", configPath, "--prompt", "retry.prompt", "--input", "transcript=" + inputPath}
serve, err := parseServeArgs([]string{"--config", configPath})
@@ -1526,7 +1533,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
for _, tc := range []struct {
name string
flag []string
@@ -1581,6 +1588,7 @@ func TestRunCommandWritesSeparateExecutionReport(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
path := filepath.Join(t.TempDir(), "execution.json")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir,
"--prompt", "reported", "--input", "transcript=" + inputPath,
"--execution-report", path,
@@ -1692,7 +1700,8 @@ func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) {
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
root := t.TempDir()
args := []string{"--promptkit-source", root, "--prompt", "p"}
configPath := writeAppConfigFile(t, "{}\n")
args := []string{"--config", configPath, "--promptkit-source", root, "--prompt", "p"}
path := filepath.Join(root, "execution.json")
for _, tail := range [][]string{
{"--execution-report", ""},
@@ -1733,7 +1742,7 @@ func intPointer(value int) *int { return &value }
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
tests := []struct {
promptID string
version string
@@ -1774,7 +1783,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
tests := []struct {
name string
profileID string
@@ -1813,11 +1822,13 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
})
}
configWithoutPromptDir := writeDefinitionFixtureConfig(t, fixtureRoot, false)
configWithoutPromptDir := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - directory: %q\n - file: %q\n",
filepath.Join(fixtureRoot, "profiles"),
filepath.Join(fixtureRoot, "framework", "backends.yml"),
))
code, _, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configWithoutPromptDir,
"--promptkit-source", filepath.Join(fixtureRoot, "profiles"),
"--profile", "custom-derived",
})
if code != ExitOK {
@@ -1825,20 +1836,67 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
}
}
func TestProfileInspectionUsesCombinedSourcesAndValidatesUnusedPrompts(t *testing.T) {
lib := newCLITestLibrary(t)
combined := `backends:
fixture:
endpoint: http://127.0.0.1:1/v1
profiles:
inspectable:
backend: fixture
model: fixture-model
prompts:
unused:
"1.0.0":
messages:
- role: user
content: "valid unused prompt"
output:
format: text
validation_mode: none
`
frameworkPath := writeFrameworkFile(t, lib.rootDir, "combined.yml", combined)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath))
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile", "--config", configPath, "--profile", "inspectable", "--format", "json",
})
if code != ExitOK {
t.Fatalf("combined-source profile inspection failed: code=%d out=%q err=%q", code, stdout, stderr)
}
var inspection renderformat.ProfileInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode combined-source profile inspection: %v", err)
}
if inspection.EffectiveModelParams.BackendID != "fixture" || inspection.EffectiveModelParams.Model != "fixture-model" {
t.Fatalf("combined source did not provide the selected profile: %+v", inspection.EffectiveModelParams)
}
invalidCombined := strings.Replace(combined, `content: "valid unused prompt"`, `content: "{{"`, 1)
invalidFrameworkPath := writeFrameworkFile(t, lib.rootDir, "combined-invalid.yml", invalidCombined)
invalidConfigPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", invalidFrameworkPath))
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{
"profile", "--config", invalidConfigPath, "--profile", "inspectable",
})
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid unused prompt did not fail construction: code=%d out=%q err=%q", code, stdout, stderr)
}
}
func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.T) {
promptDir := t.TempDir()
profileDir := t.TempDir()
writePromptDefinition(t, promptDir, "maintained.yaml", `id: maintained
version: "1.0.0"
default_profile: promptkit-openrouter-standard
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
writePromptDefinition(t, promptDir, "maintained.yaml", `prompts:
maintained:
"1.0.0":
default_profile: promptkit-openrouter-standard
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\n", promptDir, profileDir))
configPath := writeAppConfigFile(t, sourceSelectionsYAML(promptDir, profileDir, ""))
for _, profileID := range []string{"promptkit-openrouter-standard", "promptkit-openrouter-auto"} {
t.Run(profileID+" inspection", func(t *testing.T) {
@@ -1886,8 +1944,8 @@ output:
})
}
writePromptDefinition(t, profileDir, "derived.yaml", "id: derived-maintained\nbase_profile: promptkit-openrouter-standard\nmodel: fixture-inherited-model\n")
writePromptDefinition(t, profileDir, "replacement.yaml", "id: promptkit-openrouter-auto\nbackend: openrouter\nmodel: fixture-override-model\n")
writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n derived-maintained:\n extends: promptkit-openrouter-standard\n model: fixture-inherited-model\n")
writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n promptkit-openrouter-auto:\n backend: openrouter\n model: fixture-override-model\n")
for _, tc := range []struct{ id, model string }{
{id: "derived-maintained", model: "fixture-inherited-model"},
{id: "promptkit-openrouter-auto", model: "fixture-override-model"},
@@ -1910,7 +1968,7 @@ output:
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
fixtureRoot := definitionFixtureRoot(t)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
overrideDir := t.TempDir()
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
outPath := filepath.Join(t.TempDir(), "inspection.json")
@@ -1918,7 +1976,9 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--promptkit-source", filepath.Join(fixtureRoot, "prompts"),
"--promptkit-source", overrideDir,
"--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"),
"--profile", "custom-derived",
"--format", "json",
"--out", outPath,
@@ -1936,18 +1996,18 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
for _, profileID := range []string{"missing", "invalid"} {
t.Run(profileID, func(t *testing.T) {
profileDir := filepath.Join(fixtureRoot, "profiles")
args := []string{"profile", "--config", configPath, "--profile", profileID}
if profileID == "invalid" {
profileDir = t.TempDir()
writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n")
profileDir := t.TempDir()
writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n invalid:\n endpoint: not-a-url\n model: fixture\n")
args = append(args,
"--promptkit-source", filepath.Join(fixtureRoot, "prompts"),
"--promptkit-source", profileDir,
"--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"),
)
}
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--promptkit-source", profileDir,
"--profile", profileID,
})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitRuntimeError || stdout != "" || (profileID == "invalid" && !strings.Contains(stderr, "endpoint must use http or https")) {
t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
})
@@ -1961,40 +2021,36 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
name string
backend string
}{
{
name: "reserved ID",
backend: `openrouter:
endpoint: http://localhost:11434/v1`,
},
{
name: "invalid endpoint",
backend: `local-gpu:
endpoint: not-a-url`,
backend: `backends:
local-gpu:
endpoint: not-a-url
`,
},
{
name: "invalid capacity relationship",
backend: `local-gpu:
backend: `backends:
local-gpu:
endpoint: http://localhost:11434/v1
queue_capacity: 0`,
queue_capacity: 0
`,
},
{
name: "reserved extra parameter",
backend: `local-gpu:
backend: `backends:
local-gpu:
endpoint: http://localhost:11434/v1
extra_params:
model: forbidden`,
model: forbidden
`,
},
} {
t.Run(tc.name, func(t *testing.T) {
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
%s
`, lib.promptDir, lib.profileDir, tc.backend))
configPath := writeCLIFrameworkConfig(t, lib, tc.backend, "")
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
if err != nil {
t.Fatalf("expected config decoding to succeed, got %v", err)
t.Fatalf("expected application config decoding to succeed, got %v", err)
}
_, err = newEngine(cfg.runConfig.engineSettings())
if !errors.Is(err, promptkit.ErrInvalidConfig) {
@@ -2005,20 +2061,38 @@ backends:
}
})
}
t.Run("built-in backend overlay", func(t *testing.T) {
configPath := writeCLIFrameworkConfig(t, lib, `backends:
openrouter:
endpoint: http://localhost:11434/v1
`, "")
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile", "--config", configPath,
"--profile", "promptkit-openrouter-auto", "--format", "json",
})
if code != ExitOK {
t.Fatalf("built-in backend overlay failed: code=%d out=%q err=%q", code, stdout, stderr)
}
var inspection renderformat.ProfileInspection
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
t.Fatalf("decode built-in profile inspection: %v", err)
}
if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" {
t.Fatalf("caller overlay did not update the built-in backend: %+v", inspection.EffectiveModelParams)
}
})
}
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
lib := newCLITestLibrary(t)
configPath := writeAppConfigFile(t, fmt.Sprintf(`
prompt_dir: %s
profile_dir: %s
backends:
configPath := writeCLIFrameworkConfig(t, lib, `backends:
base:
endpoint: not-a-url
child:
extends: base
endpoint: http://localhost:11434/v1
`, lib.promptDir, lib.profileDir))
`, "")
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
if err != nil {
t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err)
@@ -2036,6 +2110,7 @@ func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
@@ -2058,6 +2133,7 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
@@ -2084,11 +2160,13 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
}
func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
lib := newCLITestLibrary(t)
var stdout bytes.Buffer
var stderr bytes.Buffer
code := renderCommand([]string{
"--promptkit-source", "./prompts",
"--promptkit-source", "./profiles",
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "p",
"--input", "transcript=./x.md",
"--format", "yaml",
@@ -2110,6 +2188,7 @@ func TestRenderCommandOutWritesToFile(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
@@ -2139,6 +2218,7 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
@@ -2165,6 +2245,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--prompt", "prompt.builtin",
"--input", "transcript=" + inputPath,
@@ -2189,6 +2270,7 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
@@ -2212,74 +2294,77 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
lib := newCLITestLibrary(t)
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
writePromptDefinition(t, lib.promptDir, "sole.yaml", `id: sole
version: "1"
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "sole.yaml", `prompts:
sole:
"1":
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "versioned-one.yaml", `id: versioned
version: "1"
default_profile: local
messages:
- role: user
content: "one"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "versioned.yaml", `prompts:
versioned:
"1":
default_profile: local
messages:
- role: user
content: "one"
output:
format: text
validation_mode: none
"2":
default_profile: local
messages:
- role: user
content: "two"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "versioned-two.yaml", `id: versioned
version: "2"
default_profile: local
messages:
- role: user
content: "two"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts:
optional:
"1":
default_profile: local
inputs:
- name: note
required: false
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "optional.yaml", `id: optional
version: "1"
default_profile: local
inputs:
- name: note
required: false
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "required.yaml", `prompts:
required:
"1":
default_profile: local
inputs:
- name: note
required: true
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "required.yaml", `id: required
version: "1"
default_profile: local
inputs:
- name: note
required: true
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "template.yaml", `id: template
version: "1"
default_profile: local
messages:
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: none
writePromptDefinition(t, lib.promptDir, "template.yaml", `prompts:
template:
"1":
default_profile: local
messages:
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: none
`)
baseArgs := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
baseArgs := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
for _, tc := range []struct {
name string
args []string
@@ -2288,7 +2373,7 @@ output:
}{
{name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"},
{name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"},
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "duplicate prompt definition id"},
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "version is ambiguous"},
{name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"},
{name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"},
{name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"},
@@ -2317,6 +2402,7 @@ func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
@@ -2348,6 +2434,7 @@ func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
@@ -2388,6 +2475,7 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--config", lib.configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
@@ -2509,6 +2597,7 @@ type cliTestLibrary struct {
rootDir string
promptDir string
profileDir string
configPath string
}
func newCLITestLibrary(t *testing.T) *cliTestLibrary {
@@ -2518,6 +2607,7 @@ func newCLITestLibrary(t *testing.T) *cliTestLibrary {
rootDir: root,
promptDir: filepath.Join(root, "prompts"),
profileDir: filepath.Join(root, "profiles"),
configPath: writeAppConfigFile(t, "{}\n"),
}
if err := os.MkdirAll(lib.promptDir, 0o755); err != nil {
t.Fatalf("failed to create prompt fixture directory: %v", err)
@@ -2560,22 +2650,19 @@ func definitionFixtureRoot(t *testing.T) string {
return root
}
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string {
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string) string {
t.Helper()
frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml")
if _, err := os.Stat(frameworkPath); err != nil {
t.Fatalf("stat framework backend fixture: %v", err)
}
sources := "{}\n"
if includeSources {
sources = fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n",
filepath.Join(fixtureRoot, "prompts"),
filepath.Join(fixtureRoot, "profiles"),
frameworkPath,
filepath.Join(fixtureRoot, "schemas"),
)
}
sources := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n",
filepath.Join(fixtureRoot, "prompts"),
filepath.Join(fixtureRoot, "profiles"),
frameworkPath,
filepath.Join(fixtureRoot, "schemas"),
)
return writeAppConfigFile(t, sources)
}
@@ -2639,3 +2726,30 @@ func writeAppConfigFile(t *testing.T, content string) string {
}
return path
}
func writeFrameworkFile(t *testing.T, root, name, definition string) string {
t.Helper()
dir := filepath.Join(root, "framework")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("create framework fixture directory: %v", err)
}
path := filepath.Join(dir, name)
if err := os.WriteFile(path, []byte(definition), 0o644); err != nil {
t.Fatalf("write framework fixture: %v", err)
}
return path
}
func sourceSelectionsYAML(promptDir, profileDir, frameworkFile string) string {
selectors := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n", promptDir, profileDir)
if frameworkFile != "" {
selectors += fmt.Sprintf(" - file: %q\n", frameworkFile)
}
return selectors
}
func writeCLIFrameworkConfig(t *testing.T, lib *cliTestLibrary, frameworkDefinition, applicationFields string) string {
t.Helper()
frameworkPath := writeFrameworkFile(t, lib.rootDir, "definitions.yml", frameworkDefinition)
return writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, frameworkPath)+applicationFields)
}