Migrate maintained PromptKit definitions and fixtures

This commit is contained in:
2026-11-18 19:19:24 +00:00
parent f858d57322
commit 27db6b6f47
22 changed files with 453 additions and 470 deletions

View File

@@ -200,7 +200,7 @@ func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t
provider.option: dotted-visible-value
profiles:
- id: summary
backend: local
backend: support/local
model: lower-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
@@ -232,7 +232,7 @@ func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t
provider.option: dotted-visible-value
profiles:
- id: summary
backend: local
backend: support/local
model: effective-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:

View File

@@ -18,11 +18,11 @@ import (
func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) {
conflicting := t.TempDir()
writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles:
selected:
- id: selected
model: first
`)
writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles:
selected:
- id: selected
model: second
`)
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil {
@@ -33,13 +33,13 @@ func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.
upper := t.TempDir()
empty := t.TempDir()
writeFrameworkSourceFile(t, lower, "lower.yml", `profiles:
selected:
- id: selected
endpoint: not-a-url
model: lower-model
temperature: 10
`)
writeFrameworkSourceFile(t, upper, "upper.yml", `profiles:
selected:
- id: selected
endpoint: http://127.0.0.1:1/v1
model: upper-model
temperature: 0.5
@@ -60,26 +60,25 @@ func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources
root := t.TempDir()
combined := filepath.Join(root, "combined.yml")
writeFrameworkSourceFile(t, root, "combined.yml", `backends:
local:
- id: local
endpoint: http://127.0.0.1:1/v1
profiles:
selected:
- id: selected
backend: local
model: combined-model
prompts:
combined.prompt:
"1":
default_profile: selected
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
- id: combined.prompt
default_profile: selected
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
client := &frameworkSourceCaptureClient{}
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client))
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt", PromptVersion: "1"}); err != nil {
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt"}); err != nil {
t.Fatalf("run definitions from one combined document: %v", err)
}
if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" {
@@ -94,17 +93,33 @@ prompts:
invalidUnused := filepath.Join(root, "invalid-unused.yml")
writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts:
unused.prompt:
- id: unused.prompt
messages:
- role: user
content: "{{"
output:
format: text
validation_mode: none
`)
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
}
}
func TestFrameworkRejectsObsoleteVersionKeyedPrompt(t *testing.T) {
root := t.TempDir()
path := writeFrameworkSourceFile(t, root, "obsolete.yml", `prompts:
old-prompt:
"1":
messages:
- role: user
content: "{{"
content: hello
output:
format: text
validation_mode: none
`)
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: path}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("obsolete version-keyed definition was not rejected by PromptKit: %v", err)
}
}
@@ -119,34 +134,30 @@ func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testin
t.Fatal(err)
}
mainFile := filepath.Join(root, "main.yml")
writeFrameworkSourceFile(t, root, "main.yml", `configkit:
writeFrameworkSourceFile(t, root, "main.yml", `config:
imports:
- path: catalog/base.yml
prompts:
" team/prompt ":
" derived/v2 ":
extends: " base/v1 "
required: true
`)
writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles:
" profile/primary ":
- id: profile-primary
endpoint: http://127.0.0.1:1/v1
model: imported-model
prompts:
" team/prompt ":
" base/v1 ":
default_profile: " profile/primary "
inputs:
- name: source
required: true
messages:
- role: user
content_file: ./body.md
- role: user
content: 'Artifact: {{input "source"}}'
output:
format: json
validation_mode: json_schema
schema_path: response.schema.json
- id: team-prompt
default_profile: profile-primary
inputs:
- name: source
required: true
messages:
- role: user
content_file: ./body.md
- role: user
content: 'Artifact: {{input "source"}}'
output:
format: json
validation_mode: json_schema
schema_path: response.schema.json
`)
writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}")
artifactPath := filepath.Join(root, "request-artifact.txt")
@@ -159,9 +170,9 @@ prompts:
writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact")
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
Vars: map[string]string{"audience": "reader"},
PromptID: "team-prompt",
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
Vars: map[string]string{"audience": "reader"},
})
if err != nil {
t.Fatalf("prepare inherited imported prompt: %v", err)
@@ -169,8 +180,8 @@ prompts:
if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" {
t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages)
}
if prepared.SelectedProfileID != " profile/primary " || prepared.PromptVersion != " derived/v2 " {
t.Fatalf("literal profile/version keys were not preserved: %+v", prepared)
if prepared.SelectedProfileID != "profile-primary" {
t.Fatalf("profile ID was not preserved: %+v", prepared)
}
if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" {
t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput)
@@ -178,9 +189,9 @@ prompts:
newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir)
newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
Vars: map[string]string{"audience": "reader"},
PromptID: "team-prompt",
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
Vars: map[string]string{"audience": "reader"},
})
if err != nil {
t.Fatalf("prepare with newly captured message: %v", err)
@@ -192,8 +203,7 @@ prompts:
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir))
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
"--prompt", " team/prompt ",
"--prompt-version", " derived/v2 ",
"--prompt", "team-prompt",
"--input", "source=" + artifactPath,
"--var", "audience=reader",
"--format", "json",
@@ -205,8 +215,8 @@ prompts:
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
t.Fatalf("decode rendered literal-ID fixture: %v", err)
}
if rendered.PromptID != " team/prompt " || rendered.PromptVersion != " derived/v2 " || rendered.SelectedProfileID != " profile/primary " {
t.Fatalf("CLI changed exact prompt/profile/version keys: %+v", rendered)
if rendered.PromptID != "team-prompt" || rendered.SelectedProfileID != "profile-primary" {
t.Fatalf("CLI changed exact prompt/profile IDs: %+v", rendered)
}
}
@@ -214,14 +224,14 @@ func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T)
root := t.TempDir()
frameworkFile := filepath.Join(root, "framework.yml")
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
first:
- id: first
endpoint: https://first.example/v1
api_key_env: FIRST_BACKEND_KEY
extra_params:
source: first
nested:
first_backend: true
second:
- id: second
endpoint: https://second.example/v1
api_key_env: SECOND_BACKEND_KEY
extra_params:
@@ -229,7 +239,7 @@ func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T)
nested:
second_backend: true
profiles:
parent:
- id: parent
backend: first
model: parent-model
temperature: 0.7
@@ -241,12 +251,12 @@ profiles:
nested:
parent_profile: true
retained: "yes"
backend-child:
extends: parent
- id: backend-child
extends: {id: parent}
backend: second
model: child-model
presence-child:
extends: parent
- id: presence-child
extends: {id: parent}
backend: second
api_key_env: null
temperature: null
@@ -255,29 +265,28 @@ profiles:
extra_params:
nested:
explicit_false: false
map-null:
- id: map-null
backend: second
model: map-model
extra_params: null
map-empty:
- id: map-empty
backend: second
model: map-model
extra_params: {}
prompts:
execution.prompt:
"1":
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
- id: execution.prompt
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
client := &frameworkSourceCaptureClient{}
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client))
for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} {
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", PromptVersion: "1", ProfileID: profileID}); err != nil {
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", ProfileID: profileID}); err != nil {
t.Fatalf("run profile %q: %v", profileID, err)
}
}
@@ -296,7 +305,7 @@ prompts:
presenceRequest := client.requests[1]
presenceTarget := presenceRequest.Target
if presenceTarget.APIKeyEnv != "" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
if presenceTarget.APIKeyEnv != "SECOND_BACKEND_KEY" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
!presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP {
t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence)
}
@@ -306,8 +315,8 @@ prompts:
if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) {
t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams)
}
if client.requests[2].Target.ExtraParams != nil {
t.Fatalf("whole-map null did not suppress backend defaults: %#v", client.requests[2].Target.ExtraParams)
if !reflect.DeepEqual(client.requests[2].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
t.Fatalf("whole-map null did not fall back to backend defaults: %#v", client.requests[2].Target.ExtraParams)
}
if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams)
@@ -317,17 +326,16 @@ prompts:
func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) {
root := t.TempDir()
frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles:
partial: {}
- id: partial
prompts:
partial.prompt:
"1":
default_profile: partial
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
- id: partial.prompt
default_profile: partial
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
provider := newTestLLMServer("completed partial profile", nil)
defer provider.Close()

View File

@@ -1089,7 +1089,7 @@ func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
profile := `profiles:
local-default:
- id: local-default
endpoint: http://127.0.0.1:1/v1
model: profile-model
temperature: 0.7
@@ -1147,7 +1147,7 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
t.Fatal(err)
}
base := filepath.Join(lib.rootDir, "config.yml")
framework := writeFrameworkSourceFile(t, lib.rootDir, "framework.yml", "configkit:\n imports:\n - path: prompts/prompt.render.yaml\n")
framework := writeFrameworkSourceFile(t, lib.rootDir, "framework.yml", "config:\n imports:\n - path: prompts/prompt.render.yaml\n required: true\n")
for path, body := range map[string]string{
base: "config:\n imports:\n - path: fragments/sources.yml\n required: true\n",
filepath.Join(lib.rootDir, "fragments", "sources.yml"): fmt.Sprintf("promptkit_sources:\n - file: %q\n - directory: %q\n", framework, lib.profileDir),
@@ -1174,25 +1174,24 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t
func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
custom:
"1":
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: custom
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
local-gpu:
- id: local-gpu
backend: local-gpu
model: local-model
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
configPath := writeCLIFrameworkConfig(t, lib, `backends:
local-gpu:
- id: local-gpu
endpoint: http://localhost:11434/v1
extra_params:
provider_option: enabled
@@ -1215,30 +1214,29 @@ func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) {
func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
custom:
"1":
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: custom
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles:
local-gpu:
- id: local-gpu
backend: local-gpu
model: local-model
`), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
configPath := writeCLIFrameworkConfig(t, lib, `backends:
local-base:
- id: local-base
endpoint: http://localhost:11434/v1
api_key_env: LOCAL_GPU_API_KEY
concurrency_limit: 2
local-gpu:
extends: local-base
- id: local-gpu
extends: {id: local-base}
queue_capacity: 0
extra_params:
provider_option: false
@@ -1267,17 +1265,16 @@ func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) {
func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts:
custom:
"1":
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: custom
default_profile: local-gpu
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n local-gpu:\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil {
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n - id: local-gpu\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil {
t.Fatal(err)
}
const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY"
@@ -1301,7 +1298,7 @@ func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *test
}))
defer provider.Close()
configPath := writeCLIFrameworkConfig(t, lib, fmt.Sprintf(`backends:
local-gpu:
- id: local-gpu
endpoint: %q
api_key_env: %s
extra_params:
@@ -1324,19 +1321,18 @@ func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *test
func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
session:
"1":
default_profile: local
session_id: definition-session
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: session
default_profile: local
session_id: definition-session
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`profiles:
local:
- id: local
endpoint: http://127.0.0.1:1/v1
model: local-model
reasoning_effort: low
@@ -1383,15 +1379,14 @@ func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) {
func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "plain.yaml", `prompts:
plain:
"1":
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: plain
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
@@ -1412,15 +1407,14 @@ func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) {
func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
lib := newCLITestLibrary(t)
writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts:
session:
"1":
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: session
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
@@ -1883,7 +1877,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
}{
{name: "inherited custom backend", profileID: "custom-derived", wantBackend: "fixture-custom", wantModel: "fixture-derived-model", wantAPIKeyEnv: "FIXTURE_PROFILE_API_KEY"},
{name: "endpoint only", profileID: "endpoint-only", wantModel: "fixture-endpoint-model"},
{name: "built in", profileID: "promptkit-openrouter-auto", wantBackend: "openrouter", wantModel: "openrouter/auto", wantAPIKeyEnv: "OPENROUTER_API_KEY"},
{name: "built in", profileID: "auto", wantBackend: "openrouter", wantModel: "openrouter/auto"},
}
for _, tc := range tests {
@@ -1929,21 +1923,20 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
func TestProfileInspectionUsesCombinedSourcesAndValidatesUnusedPrompts(t *testing.T) {
lib := newCLITestLibrary(t)
combined := `backends:
fixture:
- id: fixture
endpoint: http://127.0.0.1:1/v1
profiles:
inspectable:
- id: inspectable
backend: fixture
model: fixture-model
prompts:
unused:
"1.0.0":
messages:
- role: user
content: "valid unused prompt"
output:
format: text
validation_mode: none
- id: unused
messages:
- role: user
content: "valid unused prompt"
output:
format: text
validation_mode: none
`
frameworkPath := writeFrameworkFile(t, lib.rootDir, "combined.yml", combined)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath))
@@ -1976,19 +1969,18 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
promptDir := t.TempDir()
profileDir := t.TempDir()
writePromptDefinition(t, promptDir, "maintained.yaml", `prompts:
maintained:
"1.0.0":
default_profile: promptkit-openrouter-standard
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
- id: maintained
default_profile: gpt-sol-latest
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
`)
configPath := writeAppConfigFile(t, sourceSelectionsYAML(promptDir, profileDir, ""))
for _, profileID := range []string{"promptkit-openrouter-standard", "promptkit-openrouter-auto"} {
for _, profileID := range []string{"gpt-sol-latest", "auto"} {
t.Run(profileID+" inspection", func(t *testing.T) {
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", profileID, "--format", "json"})
if code != ExitOK {
@@ -2001,7 +1993,7 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
if inspection.ProfileID != profileID || pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) == "" {
t.Fatalf("maintained profile did not resolve: %+v", inspection)
}
if profileID == "promptkit-openrouter-auto" && pointerValue(inspection.ProfileModelParams.Model) != "openrouter/auto" {
if profileID == "auto" && pointerValue(inspection.ProfileModelParams.Model) != "openrouter/auto" {
t.Fatalf("unexpected auto target: %+v", inspection.ProfileModelParams)
}
})
@@ -2012,8 +2004,8 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
selection []string
wantID string
}{
{name: "prompt default", wantID: "promptkit-openrouter-standard"},
{name: "explicit selection", selection: []string{"--profile", "promptkit-openrouter-auto"}, wantID: "promptkit-openrouter-auto"},
{name: "prompt default", wantID: "gpt-sol-latest"},
{name: "explicit selection", selection: []string{"--profile", "auto"}, wantID: "auto"},
} {
t.Run(tc.name, func(t *testing.T) {
args := append([]string{"--config", configPath, "--prompt", "maintained", "--format", "json"}, tc.selection...)
@@ -2034,11 +2026,11 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
})
}
writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n derived-maintained:\n extends: promptkit-openrouter-standard\n model: fixture-inherited-model\n")
writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n promptkit-openrouter-auto:\n backend: openrouter\n model: fixture-override-model\n")
writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n - id: derived-maintained\n extends: {id: gpt-sol-latest}\n model: fixture-inherited-model\n")
writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n - id: auto\n backend: openrouter\n model: fixture-override-model\n")
for _, tc := range []struct{ id, model string }{
{id: "derived-maintained", model: "fixture-inherited-model"},
{id: "promptkit-openrouter-auto", model: "fixture-override-model"},
{id: "auto", model: "fixture-override-model"},
} {
t.Run(tc.id+" source", func(t *testing.T) {
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"profile", "--config", configPath, "--profile", tc.id, "--format", "json"})
@@ -2089,7 +2081,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
args := []string{"profile", "--config", configPath, "--profile", profileID}
if profileID == "invalid" {
profileDir := t.TempDir()
writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n invalid:\n endpoint: not-a-url\n model: fixture\n")
writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n - id: invalid\n endpoint: not-a-url\n model: fixture\n")
args = append(args,
"--promptkit-source", filepath.Join(fixtureRoot, "prompts"),
"--promptkit-source", profileDir,
@@ -2114,14 +2106,14 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
{
name: "invalid endpoint",
backend: `backends:
local-gpu:
- id: local-gpu
endpoint: not-a-url
`,
},
{
name: "invalid capacity relationship",
backend: `backends:
local-gpu:
- id: local-gpu
endpoint: http://localhost:11434/v1
queue_capacity: 0
`,
@@ -2129,7 +2121,7 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
{
name: "reserved extra parameter",
backend: `backends:
local-gpu:
- id: local-gpu
endpoint: http://localhost:11434/v1
extra_params:
model: forbidden
@@ -2154,12 +2146,12 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
t.Run("built-in backend overlay", func(t *testing.T) {
configPath := writeCLIFrameworkConfig(t, lib, `backends:
openrouter:
- id: openrouter
endpoint: http://localhost:11434/v1
`, "")
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile", "--config", configPath,
"--profile", "promptkit-openrouter-auto", "--format", "json",
"--profile", "auto", "--format", "json",
})
if code != ExitOK {
t.Fatalf("built-in backend overlay failed: code=%d out=%q err=%q", code, stdout, stderr)
@@ -2177,10 +2169,10 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) {
lib := newCLITestLibrary(t)
configPath := writeCLIFrameworkConfig(t, lib, `backends:
base:
- id: base
endpoint: not-a-url
child:
extends: base
- id: child
extends: {id: base}
endpoint: http://localhost:11434/v1
`, "")
cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"})
@@ -2332,7 +2324,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutCustomProfileSource(t *testing.T)
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
writePromptFile(t, lib.promptDir, "prompt.builtin", "auto")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", lib.configPath,
@@ -2343,10 +2335,10 @@ func TestRenderCommandUsesBuiltInProfileWithoutCustomProfileSource(t *testing.T)
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
if !strings.Contains(stdout, "selected_profile_id: auto") {
t.Fatalf("expected built-in selected profile, got %q", stdout)
}
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
if !strings.Contains(stdout, "model: openrouter/auto") {
t.Fatalf("expected built-in model, got %q", stdout)
}
}
@@ -2385,54 +2377,50 @@ func TestRenderCommandUsesDefinitionInputRules(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model")
writePromptDefinition(t, lib.promptDir, "sole.yaml", `prompts:
sole:
"1":
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: sole
default_profile: local
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts:
optional:
"1":
default_profile: local
inputs:
- name: note
required: false
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: optional
default_profile: local
inputs:
- name: note
required: false
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "required.yaml", `prompts:
required:
"1":
default_profile: local
inputs:
- name: note
required: true
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: required
default_profile: local
inputs:
- name: note
required: true
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`)
writePromptDefinition(t, lib.promptDir, "template.yaml", `prompts:
template:
"1":
default_profile: local
messages:
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: none
- id: template
default_profile: local
messages:
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: none
`)
baseArgs := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
@@ -2746,19 +2734,18 @@ func writePromptDefinition(t *testing.T, dir, name, definition string) {
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
t.Helper()
data := fmt.Sprintf(`prompts:
%q:
"1.0.0":
default_profile: %q
inputs:
- name: transcript
required: true
messages:
- role: user
content: %q
output:
format: text
validation_mode: none
repair_attempts: 0
- id: %q
default_profile: %q
inputs:
- name: transcript
required: true
messages:
- role: user
content: %q
output:
format: text
validation_mode: none
repair_attempts: 0
`, id, defaultProfile, templateContent)
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
t.Fatalf("failed to write prompt fixture: %v", err)
@@ -2767,7 +2754,7 @@ func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, template
func writeProfileFile(t *testing.T, dir, id, endpoint, model string) {
t.Helper()
data := fmt.Sprintf("profiles:\n %q:\n endpoint: %q\n model: %q\n", id, endpoint, model)
data := fmt.Sprintf("profiles:\n - id: %q\n endpoint: %q\n model: %q\n", id, endpoint, model)
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
t.Fatalf("failed to write profile fixture: %v", err)
}

View File

@@ -629,51 +629,48 @@ func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
{
name: "no declared inputs",
definition: `prompts:
p:
"1":
default_profile: exec
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: p
default_profile: exec
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`,
wantStatus: http.StatusOK,
},
{
name: "optional input omitted",
definition: `prompts:
p:
"1":
default_profile: exec
inputs:
- name: note
required: false
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: p
default_profile: exec
inputs:
- name: note
required: false
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`,
wantStatus: http.StatusOK,
},
{
name: "required input omitted",
definition: `prompts:
p:
"1":
default_profile: exec
inputs:
- name: note
required: true
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
- id: p
default_profile: exec
inputs:
- name: note
required: true
messages:
- role: user
content: "hello"
output:
format: text
validation_mode: none
`,
wantStatus: http.StatusBadRequest,
wantCode: "prompt_render_failed",
@@ -681,15 +678,14 @@ func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
{
name: "template input omitted",
definition: `prompts:
p:
"1":
default_profile: exec
messages:
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: none
- id: p
default_profile: exec
messages:
- role: user
content: '{{input "note"}}'
output:
format: text
validation_mode: none
`,
wantStatus: http.StatusBadRequest,
wantCode: "prompt_render_failed",
@@ -1020,18 +1016,18 @@ func TestHandlerSessionIDUsesPromptkitResolution(t *testing.T) {
profileDir := t.TempDir()
sessionLine := ""
if tc.definitionSession != "" {
sessionLine = " session_id: " + tc.definitionSession + "\n"
sessionLine = " session_id: " + tc.definitionSession + "\n"
}
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("prompts:\n p:\n \"1\":\n default_profile: exec\n"+sessionLine+` messages:
- role: user
content: "hi"
output:
format: text
validation_mode: none
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("prompts:\n - id: p\n default_profile: exec\n"+sessionLine+` messages:
- role: user
content: "hi"
output:
format: text
validation_mode: none
`), 0o644); err != nil {
t.Fatalf("write prompt fixture: %v", err)
}
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n exec:\n endpoint: http://127.0.0.1:1/v1\n model: test\n"), 0o644); err != nil {
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n - id: exec\n endpoint: http://127.0.0.1:1/v1\n model: test\n"), 0o644); err != nil {
t.Fatalf("write profile fixture: %v", err)
}
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, nil, promptkit.WithLLMClient(handlerLLMClient{}))
@@ -1067,20 +1063,19 @@ func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) {
promptDir := t.TempDir()
profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
p:
"1":
default_profile: limited
messages:
- role: user
content: "hi"
output:
format: text
validation_mode: none
- id: p
default_profile: limited
messages:
- role: user
content: "hi"
output:
format: text
validation_mode: none
`), 0o644); err != nil {
t.Fatalf("write prompt fixture: %v", err)
}
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
limited:
- id: limited
backend: limited
model: test
`), 0o644); err != nil {
@@ -1088,7 +1083,7 @@ func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) {
}
queueCapacity := 0
client := &blockingLLMClient{started: make(chan struct{}, 1), release: make(chan struct{})}
backend := promptkit.BackendConfig{Endpoint: stringPointer("http://127.0.0.1:1/v1"), ConcurrencyLimit: intPointer(1), QueueCapacity: &queueCapacity}
backend := promptkit.BackendConfig{ID: "limited", Endpoint: stringPointer("http://127.0.0.1:1/v1"), ConcurrencyLimit: intPointer(1), QueueCapacity: &queueCapacity}
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, &backend, promptkit.WithLLMClient(client))
h := NewHandler(engine)
first := httptest.NewRecorder()
@@ -1822,21 +1817,20 @@ func newHandlerEngineWithOptions(t *testing.T, options ...promptkit.Option) *pro
promptDir := t.TempDir()
profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
p:
"1":
default_profile: exec
messages:
- role: user
content: "hi"
output:
format: text
validation_mode: none
repair_attempts: 0
- id: p
default_profile: exec
messages:
- role: user
content: "hi"
output:
format: text
validation_mode: none
repair_attempts: 0
`), 0o644); err != nil {
t.Fatalf("write prompt fixture: %v", err)
}
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
exec:
- id: exec
endpoint: http://example.invalid/v1
model: model
`), 0o644); err != nil {
@@ -1855,7 +1849,7 @@ func newDefinitionHandler(t *testing.T, definition string) *Handler {
t.Fatalf("write prompt fixture: %v", err)
}
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
exec:
- id: exec
endpoint: http://example.invalid/v1
model: model
`), 0o644); err != nil {

View File

@@ -107,19 +107,18 @@ func TestCollectorReceivesRealEngineTerminalEvent(t *testing.T) {
promptDir := t.TempDir()
profileDir := t.TempDir()
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
p:
"1":
default_profile: local
messages:
- role: user
content: hi
output:
format: text
validation_mode: none
- id: p
default_profile: local
messages:
- role: user
content: hi
output:
format: text
validation_mode: none
`), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n local:\n endpoint: http://127.0.0.1:1/v1\n model: local-model\n"), 0o644); err != nil {
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n - id: local\n endpoint: http://127.0.0.1:1/v1\n model: local-model\n"), 0o644); err != nil {
t.Fatal(err)
}
promptInputs, err := promptkit.DirectoryInputs(promptDir)