Migrate CLI execution tests to native definitions
This commit is contained in:
@@ -3,8 +3,8 @@
|
|||||||
## Status, Baseline, And Authority
|
## Status, Baseline, And Authority
|
||||||
|
|
||||||
Planned on 2026-10-04 against Scriptorium
|
Planned on 2026-10-04 against Scriptorium
|
||||||
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–5 are complete; stages
|
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–6 are complete; stages
|
||||||
6–13 are pending.
|
7–13 are pending.
|
||||||
This plan is for a Luna coding agent: implement the stages in sequential order,
|
This plan is for a Luna coding agent: implement the stages in sequential order,
|
||||||
normally one stage per implementation prompt. The
|
normally one stage per implementation prompt. The
|
||||||
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
||||||
@@ -381,7 +381,14 @@ are protected without imposing filesystem validation on configured YAML paths.
|
|||||||
|
|
||||||
## Stage 6: Migrate Existing CLI Execution And Definition-Inspection Coverage
|
## Stage 6: Migrate Existing CLI Execution And Definition-Inspection Coverage
|
||||||
|
|
||||||
Status: Pending. Depends on Stage 5.
|
Status: Complete. CLI run, render, report, deadline, and definition-inspection
|
||||||
|
fixtures now use native keyed definitions and ordered framework sources. Backend
|
||||||
|
definitions live in framework files, built-in backend overlays are accepted,
|
||||||
|
and tests retain source precedence, retries, version selection, required and
|
||||||
|
optional inputs, profile inheritance, provider data, and inspection boundaries.
|
||||||
|
`go test ./internal/adapter/cli ./internal/format` passes, as does the native
|
||||||
|
build. Repository-wide test, race, and vet checks stop only on removed
|
||||||
|
PromptKit constructors in HTTP/report fixtures assigned to Stage 7.
|
||||||
|
|
||||||
Files: remaining execution/inspection sections in
|
Files: remaining execution/inspection sections in
|
||||||
`internal/adapter/cli/run_test.go`; formatting fixtures only if affected.
|
`internal/adapter/cli/run_test.go`; formatting fixtures only if affected.
|
||||||
|
|||||||
@@ -886,14 +886,20 @@ func TestDetermineExitCode(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunCommandVarsOptional(t *testing.T) {
|
func TestRunCommandVarsOptional(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
writePromptFile(t, lib.promptDir, "p", "local-default")
|
||||||
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "m")
|
||||||
|
inputPath := lib.writeInputFile(t, "t.md", "hello")
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := runCommand([]string{
|
code := runCommand([]string{
|
||||||
"--promptkit-source", "./prompts",
|
"--config", configPath,
|
||||||
"--promptkit-source", "./profiles",
|
"--promptkit-source", lib.promptDir,
|
||||||
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "transcript=./t.md",
|
"--input", "transcript=" + inputPath,
|
||||||
"--llm-base-url", "http://[::1",
|
"--llm-base-url", "http://[::1",
|
||||||
"--model", "m",
|
"--model", "m",
|
||||||
}, &stdout, &stderr)
|
}, &stdout, &stderr)
|
||||||
@@ -921,10 +927,7 @@ func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
|||||||
|
|
||||||
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.default", "local-default")
|
||||||
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, ""))
|
||||||
prompt_dir: %s
|
|
||||||
profile_dir: %s
|
|
||||||
`, lib.promptDir, lib.profileDir))
|
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
@@ -951,6 +954,7 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", writeAppConfigFile(t, "{}\n"),
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
@@ -1003,7 +1007,8 @@ func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing
|
|||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
|
|
||||||
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
profile := `id: local-default
|
profile := `profiles:
|
||||||
|
local-default:
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
model: profile-model
|
model: profile-model
|
||||||
temperature: 0.7
|
temperature: 0.7
|
||||||
@@ -1013,6 +1018,7 @@ temperature: 0.7
|
|||||||
}
|
}
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
@@ -1033,10 +1039,7 @@ func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
|||||||
|
|
||||||
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, ""))
|
||||||
prompt_dir: %s
|
|
||||||
profile_dir: %s
|
|
||||||
`, lib.promptDir, lib.profileDir))
|
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
@@ -1065,7 +1068,7 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
|
|||||||
base := filepath.Join(lib.rootDir, "config.yml")
|
base := filepath.Join(lib.rootDir, "config.yml")
|
||||||
for path, body := range map[string]string{
|
for path, body := range map[string]string{
|
||||||
base: "configkit:\n imports:\n - path: fragments/sources.yml\n",
|
base: "configkit:\n imports:\n - path: fragments/sources.yml\n",
|
||||||
filepath.Join(lib.rootDir, "fragments", "sources.yml"): fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\n", lib.promptDir, lib.profileDir),
|
filepath.Join(lib.rootDir, "fragments", "sources.yml"): sourceSelectionsYAML(lib.promptDir, lib.profileDir, ""),
|
||||||
filepath.Join(lib.rootDir, "overlays", "local.yml"): "defaults:\n render_format: json\n",
|
filepath.Join(lib.rootDir, "overlays", "local.yml"): "defaults:\n render_format: json\n",
|
||||||
} {
|
} {
|
||||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||||
@@ -1088,8 +1091,9 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
|
|||||||
|
|
||||||
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
|
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
||||||
version: "1"
|
custom:
|
||||||
|
"1":
|
||||||
default_profile: local-gpu
|
default_profile: local-gpu
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -1098,23 +1102,21 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
|
||||||
|
local-gpu:
|
||||||
backend: local-gpu
|
backend: local-gpu
|
||||||
model: local-model
|
model: local-model
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatalf("write profile fixture: %v", err)
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
prompt_dir: %s
|
|
||||||
profile_dir: %s
|
|
||||||
backends:
|
|
||||||
local-gpu:
|
local-gpu:
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
concurrency_limit: 2
|
concurrency_limit: 2
|
||||||
queue_capacity: 0
|
queue_capacity: 0
|
||||||
`, lib.promptDir, lib.profileDir))
|
`, "")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
@@ -1130,8 +1132,9 @@ backends:
|
|||||||
|
|
||||||
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
|
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
||||||
version: "1"
|
custom:
|
||||||
|
"1":
|
||||||
default_profile: local-gpu
|
default_profile: local-gpu
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -1140,16 +1143,14 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
|
||||||
|
local-gpu:
|
||||||
backend: local-gpu
|
backend: local-gpu
|
||||||
model: local-model
|
model: local-model
|
||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatalf("write profile fixture: %v", err)
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
prompt_dir: %s
|
|
||||||
profile_dir: %s
|
|
||||||
backends:
|
|
||||||
local-base:
|
local-base:
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
api_key_env: LOCAL_GPU_API_KEY
|
api_key_env: LOCAL_GPU_API_KEY
|
||||||
@@ -1159,7 +1160,7 @@ backends:
|
|||||||
queue_capacity: 0
|
queue_capacity: 0
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: false
|
provider_option: false
|
||||||
`, lib.promptDir, lib.profileDir))
|
`, "")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
@@ -1183,8 +1184,9 @@ backends:
|
|||||||
|
|
||||||
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
|
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom
|
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
|
||||||
version: "1"
|
custom:
|
||||||
|
"1":
|
||||||
default_profile: local-gpu
|
default_profile: local-gpu
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -1193,7 +1195,7 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("id: local-gpu\nbackend: local-gpu\nmodel: local-model\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n local-gpu:\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY"
|
const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY"
|
||||||
@@ -1216,15 +1218,13 @@ output:
|
|||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"generated"}}]}`))
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"generated"}}]}`))
|
||||||
}))
|
}))
|
||||||
defer provider.Close()
|
defer provider.Close()
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`prompt_dir: %q
|
configPath := writeCLIFrameworkConfig(t, lib, fmt.Sprintf(`backends:
|
||||||
profile_dir: %q
|
|
||||||
backends:
|
|
||||||
local-gpu:
|
local-gpu:
|
||||||
endpoint: %q
|
endpoint: %q
|
||||||
api_key_env: %s
|
api_key_env: %s
|
||||||
extra_params:
|
extra_params:
|
||||||
synthetic_count: 7
|
synthetic_count: 7
|
||||||
`, lib.promptDir, lib.profileDir, provider.URL+"/v1", credentialName))
|
`, provider.URL+"/v1", credentialName), "")
|
||||||
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"})
|
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err)
|
t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err)
|
||||||
@@ -1241,8 +1241,9 @@ backends:
|
|||||||
|
|
||||||
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
|
||||||
version: "1"
|
session:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
session_id: definition-session
|
session_id: definition-session
|
||||||
messages:
|
messages:
|
||||||
@@ -1252,7 +1253,8 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`id: local
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`profiles:
|
||||||
|
local:
|
||||||
endpoint: http://127.0.0.1:1/v1
|
endpoint: http://127.0.0.1:1/v1
|
||||||
model: local-model
|
model: local-model
|
||||||
reasoning_effort: low
|
reasoning_effort: low
|
||||||
@@ -1273,7 +1275,7 @@ reasoning_effort: low
|
|||||||
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
|
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"}
|
args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"}
|
||||||
args = append(args, tc.args...)
|
args = append(args, tc.args...)
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
@@ -1298,8 +1300,9 @@ reasoning_effort: low
|
|||||||
|
|
||||||
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "plain.yaml", `id: plain
|
writePromptDefinition(t, lib.promptDir, "plain.yaml", `prompts:
|
||||||
version: "1"
|
plain:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -1311,6 +1314,7 @@ output:
|
|||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "plain",
|
"--prompt", "plain",
|
||||||
@@ -1325,8 +1329,9 @@ output:
|
|||||||
|
|
||||||
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session
|
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
|
||||||
version: "1"
|
session:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -1338,6 +1343,7 @@ output:
|
|||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
code, _, stderr := runCLICommand(t, renderCommand, []string{
|
code, _, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "session",
|
"--prompt", "session",
|
||||||
@@ -1353,8 +1359,9 @@ output:
|
|||||||
|
|
||||||
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
|
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect
|
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `prompts:
|
||||||
version: "1"
|
inspect:
|
||||||
|
"1":
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
content: "hello"
|
content: "hello"
|
||||||
@@ -1362,14 +1369,14 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
|
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
|
||||||
t.Fatalf("unexpected inspection output: %s", stdout)
|
t.Fatalf("unexpected inspection output: %s", stdout)
|
||||||
}
|
}
|
||||||
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "missing"})
|
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "missing"})
|
||||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||||
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
@@ -1395,7 +1402,7 @@ func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) {
|
|||||||
|
|
||||||
func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
|
func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) {
|
||||||
fixtureRoot := definitionFixtureRoot(t)
|
fixtureRoot := definitionFixtureRoot(t)
|
||||||
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
|
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
|
||||||
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
|
inputPath := filepath.Join(fixtureRoot, "inputs", "source.md")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
@@ -1465,7 +1472,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer provider.Close()
|
defer provider.Close()
|
||||||
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\ntransient_retries: 1\n", lib.promptDir, lib.profileDir))
|
configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")+"transient_retries: 1\n")
|
||||||
args := []string{"--config", configPath, "--prompt", "retry.prompt", "--input", "transcript=" + inputPath}
|
args := []string{"--config", configPath, "--prompt", "retry.prompt", "--input", "transcript=" + inputPath}
|
||||||
|
|
||||||
serve, err := parseServeArgs([]string{"--config", configPath})
|
serve, err := parseServeArgs([]string{"--config", configPath})
|
||||||
@@ -1526,7 +1533,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
|
|||||||
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
|
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
|
args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
flag []string
|
flag []string
|
||||||
@@ -1581,6 +1588,7 @@ func TestRunCommandWritesSeparateExecutionReport(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
||||||
path := filepath.Join(t.TempDir(), "execution.json")
|
path := filepath.Join(t.TempDir(), "execution.json")
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "reported", "--input", "transcript=" + inputPath,
|
"--prompt", "reported", "--input", "transcript=" + inputPath,
|
||||||
"--execution-report", path,
|
"--execution-report", path,
|
||||||
@@ -1692,7 +1700,8 @@ func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
|
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
args := []string{"--promptkit-source", root, "--prompt", "p"}
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
args := []string{"--config", configPath, "--promptkit-source", root, "--prompt", "p"}
|
||||||
path := filepath.Join(root, "execution.json")
|
path := filepath.Join(root, "execution.json")
|
||||||
for _, tail := range [][]string{
|
for _, tail := range [][]string{
|
||||||
{"--execution-report", ""},
|
{"--execution-report", ""},
|
||||||
@@ -1733,7 +1742,7 @@ func intPointer(value int) *int { return &value }
|
|||||||
|
|
||||||
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||||
fixtureRoot := definitionFixtureRoot(t)
|
fixtureRoot := definitionFixtureRoot(t)
|
||||||
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
|
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
promptID string
|
promptID string
|
||||||
version string
|
version string
|
||||||
@@ -1774,7 +1783,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
|||||||
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
|
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
|
||||||
|
|
||||||
fixtureRoot := definitionFixtureRoot(t)
|
fixtureRoot := definitionFixtureRoot(t)
|
||||||
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
|
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
profileID string
|
profileID string
|
||||||
@@ -1813,11 +1822,13 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
configWithoutPromptDir := writeDefinitionFixtureConfig(t, fixtureRoot, false)
|
configWithoutPromptDir := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - directory: %q\n - file: %q\n",
|
||||||
|
filepath.Join(fixtureRoot, "profiles"),
|
||||||
|
filepath.Join(fixtureRoot, "framework", "backends.yml"),
|
||||||
|
))
|
||||||
code, _, stderr := runCLICommand(t, inspectCommand, []string{
|
code, _, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
"profile",
|
"profile",
|
||||||
"--config", configWithoutPromptDir,
|
"--config", configWithoutPromptDir,
|
||||||
"--promptkit-source", filepath.Join(fixtureRoot, "profiles"),
|
|
||||||
"--profile", "custom-derived",
|
"--profile", "custom-derived",
|
||||||
})
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
@@ -1825,11 +1836,58 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProfileInspectionUsesCombinedSourcesAndValidatesUnusedPrompts(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
combined := `backends:
|
||||||
|
fixture:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
profiles:
|
||||||
|
inspectable:
|
||||||
|
backend: fixture
|
||||||
|
model: fixture-model
|
||||||
|
prompts:
|
||||||
|
unused:
|
||||||
|
"1.0.0":
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "valid unused prompt"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`
|
||||||
|
frameworkPath := writeFrameworkFile(t, lib.rootDir, "combined.yml", combined)
|
||||||
|
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath))
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"profile", "--config", configPath, "--profile", "inspectable", "--format", "json",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("combined-source profile inspection failed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
var inspection renderformat.ProfileInspection
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||||
|
t.Fatalf("decode combined-source profile inspection: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.EffectiveModelParams.BackendID != "fixture" || inspection.EffectiveModelParams.Model != "fixture-model" {
|
||||||
|
t.Fatalf("combined source did not provide the selected profile: %+v", inspection.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalidCombined := strings.Replace(combined, `content: "valid unused prompt"`, `content: "{{"`, 1)
|
||||||
|
invalidFrameworkPath := writeFrameworkFile(t, lib.rootDir, "combined-invalid.yml", invalidCombined)
|
||||||
|
invalidConfigPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", invalidFrameworkPath))
|
||||||
|
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{
|
||||||
|
"profile", "--config", invalidConfigPath, "--profile", "inspectable",
|
||||||
|
})
|
||||||
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||||
|
t.Fatalf("invalid unused prompt did not fail construction: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.T) {
|
func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.T) {
|
||||||
promptDir := t.TempDir()
|
promptDir := t.TempDir()
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
writePromptDefinition(t, promptDir, "maintained.yaml", `id: maintained
|
writePromptDefinition(t, promptDir, "maintained.yaml", `prompts:
|
||||||
version: "1.0.0"
|
maintained:
|
||||||
|
"1.0.0":
|
||||||
default_profile: promptkit-openrouter-standard
|
default_profile: promptkit-openrouter-standard
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -1838,7 +1896,7 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\n", promptDir, profileDir))
|
configPath := writeAppConfigFile(t, sourceSelectionsYAML(promptDir, profileDir, ""))
|
||||||
|
|
||||||
for _, profileID := range []string{"promptkit-openrouter-standard", "promptkit-openrouter-auto"} {
|
for _, profileID := range []string{"promptkit-openrouter-standard", "promptkit-openrouter-auto"} {
|
||||||
t.Run(profileID+" inspection", func(t *testing.T) {
|
t.Run(profileID+" inspection", func(t *testing.T) {
|
||||||
@@ -1886,8 +1944,8 @@ output:
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
writePromptDefinition(t, profileDir, "derived.yaml", "id: derived-maintained\nbase_profile: promptkit-openrouter-standard\nmodel: fixture-inherited-model\n")
|
writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n derived-maintained:\n extends: promptkit-openrouter-standard\n model: fixture-inherited-model\n")
|
||||||
writePromptDefinition(t, profileDir, "replacement.yaml", "id: promptkit-openrouter-auto\nbackend: openrouter\nmodel: fixture-override-model\n")
|
writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n promptkit-openrouter-auto:\n backend: openrouter\n model: fixture-override-model\n")
|
||||||
for _, tc := range []struct{ id, model string }{
|
for _, tc := range []struct{ id, model string }{
|
||||||
{id: "derived-maintained", model: "fixture-inherited-model"},
|
{id: "derived-maintained", model: "fixture-inherited-model"},
|
||||||
{id: "promptkit-openrouter-auto", model: "fixture-override-model"},
|
{id: "promptkit-openrouter-auto", model: "fixture-override-model"},
|
||||||
@@ -1910,7 +1968,7 @@ output:
|
|||||||
|
|
||||||
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
|
func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) {
|
||||||
fixtureRoot := definitionFixtureRoot(t)
|
fixtureRoot := definitionFixtureRoot(t)
|
||||||
configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true)
|
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
|
||||||
overrideDir := t.TempDir()
|
overrideDir := t.TempDir()
|
||||||
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
|
writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model")
|
||||||
outPath := filepath.Join(t.TempDir(), "inspection.json")
|
outPath := filepath.Join(t.TempDir(), "inspection.json")
|
||||||
@@ -1918,7 +1976,9 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
|
|||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
"profile",
|
"profile",
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
|
"--promptkit-source", filepath.Join(fixtureRoot, "prompts"),
|
||||||
"--promptkit-source", overrideDir,
|
"--promptkit-source", overrideDir,
|
||||||
|
"--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"),
|
||||||
"--profile", "custom-derived",
|
"--profile", "custom-derived",
|
||||||
"--format", "json",
|
"--format", "json",
|
||||||
"--out", outPath,
|
"--out", outPath,
|
||||||
@@ -1936,18 +1996,18 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
|
|||||||
|
|
||||||
for _, profileID := range []string{"missing", "invalid"} {
|
for _, profileID := range []string{"missing", "invalid"} {
|
||||||
t.Run(profileID, func(t *testing.T) {
|
t.Run(profileID, func(t *testing.T) {
|
||||||
profileDir := filepath.Join(fixtureRoot, "profiles")
|
args := []string{"profile", "--config", configPath, "--profile", profileID}
|
||||||
if profileID == "invalid" {
|
if profileID == "invalid" {
|
||||||
profileDir = t.TempDir()
|
profileDir := t.TempDir()
|
||||||
writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n")
|
writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n invalid:\n endpoint: not-a-url\n model: fixture\n")
|
||||||
}
|
args = append(args,
|
||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
"--promptkit-source", filepath.Join(fixtureRoot, "prompts"),
|
||||||
"profile",
|
|
||||||
"--config", configPath,
|
|
||||||
"--promptkit-source", profileDir,
|
"--promptkit-source", profileDir,
|
||||||
"--profile", profileID,
|
"--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"),
|
||||||
})
|
)
|
||||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
}
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||||
|
if code != ExitRuntimeError || stdout != "" || (profileID == "invalid" && !strings.Contains(stderr, "endpoint must use http or https")) {
|
||||||
t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -1961,40 +2021,36 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
|||||||
name string
|
name string
|
||||||
backend string
|
backend string
|
||||||
}{
|
}{
|
||||||
{
|
|
||||||
name: "reserved ID",
|
|
||||||
backend: `openrouter:
|
|
||||||
endpoint: http://localhost:11434/v1`,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "invalid endpoint",
|
name: "invalid endpoint",
|
||||||
backend: `local-gpu:
|
backend: `backends:
|
||||||
endpoint: not-a-url`,
|
local-gpu:
|
||||||
|
endpoint: not-a-url
|
||||||
|
`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "invalid capacity relationship",
|
name: "invalid capacity relationship",
|
||||||
backend: `local-gpu:
|
backend: `backends:
|
||||||
|
local-gpu:
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
queue_capacity: 0`,
|
queue_capacity: 0
|
||||||
|
`,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "reserved extra parameter",
|
name: "reserved extra parameter",
|
||||||
backend: `local-gpu:
|
backend: `backends:
|
||||||
|
local-gpu:
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
extra_params:
|
extra_params:
|
||||||
model: forbidden`,
|
model: forbidden
|
||||||
|
`,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeCLIFrameworkConfig(t, lib, tc.backend, "")
|
||||||
prompt_dir: %s
|
|
||||||
profile_dir: %s
|
|
||||||
backends:
|
|
||||||
%s
|
|
||||||
`, lib.promptDir, lib.profileDir, tc.backend))
|
|
||||||
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected config decoding to succeed, got %v", err)
|
t.Fatalf("expected application config decoding to succeed, got %v", err)
|
||||||
}
|
}
|
||||||
_, err = newEngine(cfg.runConfig.engineSettings())
|
_, err = newEngine(cfg.runConfig.engineSettings())
|
||||||
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
@@ -2005,20 +2061,38 @@ backends:
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("built-in backend overlay", func(t *testing.T) {
|
||||||
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
|
openrouter:
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
`, "")
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"profile", "--config", configPath,
|
||||||
|
"--profile", "promptkit-openrouter-auto", "--format", "json",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("built-in backend overlay failed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
var inspection renderformat.ProfileInspection
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||||
|
t.Fatalf("decode built-in profile inspection: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" {
|
||||||
|
t.Fatalf("caller overlay did not update the built-in backend: %+v", inspection.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
|
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
configPath := writeAppConfigFile(t, fmt.Sprintf(`
|
configPath := writeCLIFrameworkConfig(t, lib, `backends:
|
||||||
prompt_dir: %s
|
|
||||||
profile_dir: %s
|
|
||||||
backends:
|
|
||||||
base:
|
base:
|
||||||
endpoint: not-a-url
|
endpoint: not-a-url
|
||||||
child:
|
child:
|
||||||
extends: base
|
extends: base
|
||||||
endpoint: http://localhost:11434/v1
|
endpoint: http://localhost:11434/v1
|
||||||
`, lib.promptDir, lib.profileDir))
|
`, "")
|
||||||
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err)
|
t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err)
|
||||||
@@ -2036,6 +2110,7 @@ func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
@@ -2058,6 +2133,7 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
@@ -2084,11 +2160,13 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
|
func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := renderCommand([]string{
|
code := renderCommand([]string{
|
||||||
"--promptkit-source", "./prompts",
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", "./profiles",
|
"--promptkit-source", lib.promptDir,
|
||||||
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "transcript=./x.md",
|
"--input", "transcript=./x.md",
|
||||||
"--format", "yaml",
|
"--format", "yaml",
|
||||||
@@ -2110,6 +2188,7 @@ func TestRenderCommandOutWritesToFile(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
@@ -2139,6 +2218,7 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
@@ -2165,6 +2245,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
|||||||
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--prompt", "prompt.builtin",
|
"--prompt", "prompt.builtin",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
@@ -2189,6 +2270,7 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
|
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
@@ -2212,8 +2294,9 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) {
|
|||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
|
||||||
|
|
||||||
writePromptDefinition(t, lib.promptDir, "sole.yaml", `id: sole
|
writePromptDefinition(t, lib.promptDir, "sole.yaml", `prompts:
|
||||||
version: "1"
|
sole:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -2222,8 +2305,9 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
writePromptDefinition(t, lib.promptDir, "versioned-one.yaml", `id: versioned
|
writePromptDefinition(t, lib.promptDir, "versioned.yaml", `prompts:
|
||||||
version: "1"
|
versioned:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -2231,9 +2315,7 @@ messages:
|
|||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
"2":
|
||||||
writePromptDefinition(t, lib.promptDir, "versioned-two.yaml", `id: versioned
|
|
||||||
version: "2"
|
|
||||||
default_profile: local
|
default_profile: local
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -2242,8 +2324,9 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
writePromptDefinition(t, lib.promptDir, "optional.yaml", `id: optional
|
writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts:
|
||||||
version: "1"
|
optional:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
inputs:
|
inputs:
|
||||||
- name: note
|
- name: note
|
||||||
@@ -2255,8 +2338,9 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
writePromptDefinition(t, lib.promptDir, "required.yaml", `id: required
|
writePromptDefinition(t, lib.promptDir, "required.yaml", `prompts:
|
||||||
version: "1"
|
required:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
inputs:
|
inputs:
|
||||||
- name: note
|
- name: note
|
||||||
@@ -2268,8 +2352,9 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
writePromptDefinition(t, lib.promptDir, "template.yaml", `id: template
|
writePromptDefinition(t, lib.promptDir, "template.yaml", `prompts:
|
||||||
version: "1"
|
template:
|
||||||
|
"1":
|
||||||
default_profile: local
|
default_profile: local
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -2279,7 +2364,7 @@ output:
|
|||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
|
|
||||||
baseArgs := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
|
baseArgs := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
args []string
|
args []string
|
||||||
@@ -2288,7 +2373,7 @@ output:
|
|||||||
}{
|
}{
|
||||||
{name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"},
|
{name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"},
|
||||||
{name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"},
|
{name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"},
|
||||||
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "duplicate prompt definition id"},
|
{name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "version is ambiguous"},
|
||||||
{name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"},
|
{name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"},
|
||||||
{name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"},
|
{name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"},
|
||||||
{name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"},
|
{name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"},
|
||||||
@@ -2317,6 +2402,7 @@ func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
@@ -2348,6 +2434,7 @@ func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
|
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
@@ -2388,6 +2475,7 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
|
"--config", lib.configPath,
|
||||||
"--promptkit-source", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--promptkit-source", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
@@ -2509,6 +2597,7 @@ type cliTestLibrary struct {
|
|||||||
rootDir string
|
rootDir string
|
||||||
promptDir string
|
promptDir string
|
||||||
profileDir string
|
profileDir string
|
||||||
|
configPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newCLITestLibrary(t *testing.T) *cliTestLibrary {
|
func newCLITestLibrary(t *testing.T) *cliTestLibrary {
|
||||||
@@ -2518,6 +2607,7 @@ func newCLITestLibrary(t *testing.T) *cliTestLibrary {
|
|||||||
rootDir: root,
|
rootDir: root,
|
||||||
promptDir: filepath.Join(root, "prompts"),
|
promptDir: filepath.Join(root, "prompts"),
|
||||||
profileDir: filepath.Join(root, "profiles"),
|
profileDir: filepath.Join(root, "profiles"),
|
||||||
|
configPath: writeAppConfigFile(t, "{}\n"),
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(lib.promptDir, 0o755); err != nil {
|
if err := os.MkdirAll(lib.promptDir, 0o755); err != nil {
|
||||||
t.Fatalf("failed to create prompt fixture directory: %v", err)
|
t.Fatalf("failed to create prompt fixture directory: %v", err)
|
||||||
@@ -2560,22 +2650,19 @@ func definitionFixtureRoot(t *testing.T) string {
|
|||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string {
|
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml")
|
frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml")
|
||||||
if _, err := os.Stat(frameworkPath); err != nil {
|
if _, err := os.Stat(frameworkPath); err != nil {
|
||||||
t.Fatalf("stat framework backend fixture: %v", err)
|
t.Fatalf("stat framework backend fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sources := "{}\n"
|
sources := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n",
|
||||||
if includeSources {
|
|
||||||
sources = fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n",
|
|
||||||
filepath.Join(fixtureRoot, "prompts"),
|
filepath.Join(fixtureRoot, "prompts"),
|
||||||
filepath.Join(fixtureRoot, "profiles"),
|
filepath.Join(fixtureRoot, "profiles"),
|
||||||
frameworkPath,
|
frameworkPath,
|
||||||
filepath.Join(fixtureRoot, "schemas"),
|
filepath.Join(fixtureRoot, "schemas"),
|
||||||
)
|
)
|
||||||
}
|
|
||||||
return writeAppConfigFile(t, sources)
|
return writeAppConfigFile(t, sources)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2639,3 +2726,30 @@ func writeAppConfigFile(t *testing.T, content string) string {
|
|||||||
}
|
}
|
||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeFrameworkFile(t *testing.T, root, name, definition string) string {
|
||||||
|
t.Helper()
|
||||||
|
dir := filepath.Join(root, "framework")
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatalf("create framework fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(dir, name)
|
||||||
|
if err := os.WriteFile(path, []byte(definition), 0o644); err != nil {
|
||||||
|
t.Fatalf("write framework fixture: %v", err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func sourceSelectionsYAML(promptDir, profileDir, frameworkFile string) string {
|
||||||
|
selectors := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n", promptDir, profileDir)
|
||||||
|
if frameworkFile != "" {
|
||||||
|
selectors += fmt.Sprintf(" - file: %q\n", frameworkFile)
|
||||||
|
}
|
||||||
|
return selectors
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeCLIFrameworkConfig(t *testing.T, lib *cliTestLibrary, frameworkDefinition, applicationFields string) string {
|
||||||
|
t.Helper()
|
||||||
|
frameworkPath := writeFrameworkFile(t, lib.rootDir, "definitions.yml", frameworkDefinition)
|
||||||
|
return writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, frameworkPath)+applicationFields)
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user