diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6751265..affa4ef 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -3,8 +3,8 @@ ## Status, Baseline, And Authority Planned on 2026-10-04 against Scriptorium -`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–5 are complete; stages -6–13 are pending. +`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–6 are complete; stages +7–13 are pending. This plan is for a Luna coding agent: implement the stages in sequential order, normally one stage per implementation prompt. The [feature roadmap](promptkit.md) owns the target state, scope, and confirmed @@ -381,7 +381,14 @@ are protected without imposing filesystem validation on configured YAML paths. ## Stage 6: Migrate Existing CLI Execution And Definition-Inspection Coverage -Status: Pending. Depends on Stage 5. +Status: Complete. CLI run, render, report, deadline, and definition-inspection +fixtures now use native keyed definitions and ordered framework sources. Backend +definitions live in framework files, built-in backend overlays are accepted, +and tests retain source precedence, retries, version selection, required and +optional inputs, profile inheritance, provider data, and inspection boundaries. +`go test ./internal/adapter/cli ./internal/format` passes, as does the native +build. Repository-wide test, race, and vet checks stop only on removed +PromptKit constructors in HTTP/report fixtures assigned to Stage 7. Files: remaining execution/inspection sections in `internal/adapter/cli/run_test.go`; formatting fixtures only if affected. diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index 6604bbd..7dcb6c7 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -886,14 +886,20 @@ func TestDetermineExitCode(t *testing.T) { } func TestRunCommandVarsOptional(t *testing.T) { + lib := newCLITestLibrary(t) + configPath := writeAppConfigFile(t, "{}\n") + writePromptFile(t, lib.promptDir, "p", "local-default") + writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "m") + inputPath := lib.writeInputFile(t, "t.md", "hello") var stdout bytes.Buffer var stderr bytes.Buffer code := runCommand([]string{ - "--promptkit-source", "./prompts", - "--promptkit-source", "./profiles", + "--config", configPath, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "p", - "--input", "transcript=./t.md", + "--input", "transcript=" + inputPath, "--llm-base-url", "http://[::1", "--model", "m", }, &stdout, &stderr) @@ -921,10 +927,7 @@ func TestRunCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { writePromptFile(t, lib.promptDir, "prompt.default", "local-default") writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model") - configPath := writeAppConfigFile(t, fmt.Sprintf(` -prompt_dir: %s -profile_dir: %s -`, lib.promptDir, lib.profileDir)) + configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")) code, stdout, stderr := runCLICommand(t, runCommand, []string{ "--config", configPath, @@ -951,6 +954,7 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", writeAppConfigFile(t, "{}\n"), "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", @@ -1003,16 +1007,18 @@ func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript") writePromptFile(t, lib.promptDir, "prompt.render", "local-default") - profile := `id: local-default -endpoint: http://127.0.0.1:1/v1 -model: profile-model -temperature: 0.7 + profile := `profiles: + local-default: + endpoint: http://127.0.0.1:1/v1 + model: profile-model + temperature: 0.7 ` if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil { t.Fatalf("failed to write profile fixture: %v", err) } code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", @@ -1033,10 +1039,7 @@ func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) { writePromptFile(t, lib.promptDir, "prompt.render", "local-default") writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") - configPath := writeAppConfigFile(t, fmt.Sprintf(` -prompt_dir: %s -profile_dir: %s -`, lib.promptDir, lib.profileDir)) + configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")) code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--config", configPath, @@ -1065,7 +1068,7 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t base := filepath.Join(lib.rootDir, "config.yml") for path, body := range map[string]string{ base: "configkit:\n imports:\n - path: fragments/sources.yml\n", - filepath.Join(lib.rootDir, "fragments", "sources.yml"): fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\n", lib.promptDir, lib.profileDir), + filepath.Join(lib.rootDir, "fragments", "sources.yml"): sourceSelectionsYAML(lib.promptDir, lib.profileDir, ""), filepath.Join(lib.rootDir, "overlays", "local.yml"): "defaults:\n render_format: json\n", } { if err := os.WriteFile(path, []byte(body), 0o644); err != nil { @@ -1088,33 +1091,32 @@ func TestRenderCommandLoadsImportsAndOverlayWithoutChangingExecutionProfile(t *t func TestRenderCommandUsesConfiguredCustomBackend(t *testing.T) { lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom -version: "1" -default_profile: local-gpu -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts: + custom: + "1": + default_profile: local-gpu + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu -backend: local-gpu -model: local-model + if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles: + local-gpu: + backend: local-gpu + model: local-model `), 0o644); err != nil { t.Fatalf("write profile fixture: %v", err) } - configPath := writeAppConfigFile(t, fmt.Sprintf(` -prompt_dir: %s -profile_dir: %s -backends: + configPath := writeCLIFrameworkConfig(t, lib, `backends: local-gpu: endpoint: http://localhost:11434/v1 extra_params: provider_option: enabled concurrency_limit: 2 queue_capacity: 0 -`, lib.promptDir, lib.profileDir)) +`, "") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--config", configPath, @@ -1130,26 +1132,25 @@ backends: func TestRenderCommandUsesInheritedCustomBackend(t *testing.T) { lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom -version: "1" -default_profile: local-gpu -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts: + custom: + "1": + default_profile: local-gpu + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`id: local-gpu -backend: local-gpu -model: local-model + if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte(`profiles: + local-gpu: + backend: local-gpu + model: local-model `), 0o644); err != nil { t.Fatalf("write profile fixture: %v", err) } - configPath := writeAppConfigFile(t, fmt.Sprintf(` -prompt_dir: %s -profile_dir: %s -backends: + configPath := writeCLIFrameworkConfig(t, lib, `backends: local-base: endpoint: http://localhost:11434/v1 api_key_env: LOCAL_GPU_API_KEY @@ -1159,7 +1160,7 @@ backends: queue_capacity: 0 extra_params: provider_option: false -`, lib.promptDir, lib.profileDir)) +`, "") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--config", configPath, @@ -1183,17 +1184,18 @@ backends: func TestConfiguredBackendPreservesIntegerParameterAndOptionalCredential(t *testing.T) { lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "custom.yaml", `id: custom -version: "1" -default_profile: local-gpu -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "custom.yaml", `prompts: + custom: + "1": + default_profile: local-gpu + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("id: local-gpu\nbackend: local-gpu\nmodel: local-model\n"), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(lib.profileDir, "local-gpu.yaml"), []byte("profiles:\n local-gpu:\n backend: local-gpu\n model: local-model\n"), 0o644); err != nil { t.Fatal(err) } const credentialName = "SCRIPTORIUM_OPTIONAL_BACKEND_KEY" @@ -1216,15 +1218,13 @@ output: _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"generated"}}]}`)) })) defer provider.Close() - configPath := writeAppConfigFile(t, fmt.Sprintf(`prompt_dir: %q -profile_dir: %q -backends: + configPath := writeCLIFrameworkConfig(t, lib, fmt.Sprintf(`backends: local-gpu: endpoint: %q api_key_env: %s extra_params: synthetic_count: 7 -`, lib.promptDir, lib.profileDir, provider.URL+"/v1", credentialName)) +`, provider.URL+"/v1", credentialName), "") cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"}) if err != nil { t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err) @@ -1241,21 +1241,23 @@ backends: func TestRenderCommandMapsReasoningEffortAndSessionID(t *testing.T) { lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session -version: "1" -default_profile: local -session_id: definition-session -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts: + session: + "1": + default_profile: local + session_id: definition-session + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`id: local -endpoint: http://127.0.0.1:1/v1 -model: local-model -reasoning_effort: low + if err := os.WriteFile(filepath.Join(lib.profileDir, "local.yaml"), []byte(`profiles: + local: + endpoint: http://127.0.0.1:1/v1 + model: local-model + reasoning_effort: low `), 0o644); err != nil { t.Fatalf("write profile fixture: %v", err) } @@ -1273,7 +1275,7 @@ reasoning_effort: low {name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"}, } { t.Run(tc.name, func(t *testing.T) { - args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"} + args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"} args = append(args, tc.args...) code, stdout, stderr := runCLICommand(t, renderCommand, args) if code != ExitOK { @@ -1298,19 +1300,21 @@ reasoning_effort: low func TestRenderCommandOmitsEmptyEffectiveSessionID(t *testing.T) { lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "plain.yaml", `id: plain -version: "1" -default_profile: local -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "plain.yaml", `prompts: + plain: + "1": + default_profile: local + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "plain", @@ -1325,19 +1329,21 @@ output: func TestRenderCommandRejectsOverlongSessionID(t *testing.T) { lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "session.yaml", `id: session -version: "1" -default_profile: local -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "session.yaml", `prompts: + session: + "1": + default_profile: local + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model") code, _, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session", @@ -1353,23 +1359,24 @@ output: func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) { lib := newCLITestLibrary(t) - writePromptDefinition(t, lib.promptDir, "inspect.yaml", `id: inspect -version: "1" -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "inspect.yaml", `prompts: + inspect: + "1": + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"}) + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"}) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) { t.Fatalf("unexpected inspection output: %s", stdout) } - code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "missing"}) + code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "missing"}) if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") { t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) } @@ -1395,7 +1402,7 @@ func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) { func TestPromptkitDefinitionsRenderThroughCLI(t *testing.T) { fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true) + configPath := writeDefinitionFixtureConfig(t, fixtureRoot) inputPath := filepath.Join(fixtureRoot, "inputs", "source.md") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ @@ -1465,7 +1472,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) { })) defer provider.Close() writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") - configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\ntransient_retries: 1\n", lib.promptDir, lib.profileDir)) + configPath := writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, "")+"transient_retries: 1\n") args := []string{"--config", configPath, "--prompt", "retry.prompt", "--input", "transcript=" + inputPath} serve, err := parseServeArgs([]string{"--config", configPath}) @@ -1526,7 +1533,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) { func TestRunTimeoutParsingAndCancellation(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") - args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath} + args := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath} for _, tc := range []struct { name string flag []string @@ -1581,6 +1588,7 @@ func TestRunCommandWritesSeparateExecutionReport(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") path := filepath.Join(t.TempDir(), "execution.json") code, stdout, stderr := runCLICommand(t, runCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "reported", "--input", "transcript=" + inputPath, "--execution-report", path, @@ -1692,7 +1700,8 @@ func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) { func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) { root := t.TempDir() - args := []string{"--promptkit-source", root, "--prompt", "p"} + configPath := writeAppConfigFile(t, "{}\n") + args := []string{"--config", configPath, "--promptkit-source", root, "--prompt", "p"} path := filepath.Join(root, "execution.json") for _, tail := range [][]string{ {"--execution-report", ""}, @@ -1733,7 +1742,7 @@ func intPointer(value int) *int { return &value } func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) { fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true) + configPath := writeDefinitionFixtureConfig(t, fixtureRoot) tests := []struct { promptID string version string @@ -1774,7 +1783,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) { t.Setenv("FIXTURE_PROFILE_API_KEY", secret) fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true) + configPath := writeDefinitionFixtureConfig(t, fixtureRoot) tests := []struct { name string profileID string @@ -1813,11 +1822,13 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) { }) } - configWithoutPromptDir := writeDefinitionFixtureConfig(t, fixtureRoot, false) + configWithoutPromptDir := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - directory: %q\n - file: %q\n", + filepath.Join(fixtureRoot, "profiles"), + filepath.Join(fixtureRoot, "framework", "backends.yml"), + )) code, _, stderr := runCLICommand(t, inspectCommand, []string{ "profile", "--config", configWithoutPromptDir, - "--promptkit-source", filepath.Join(fixtureRoot, "profiles"), "--profile", "custom-derived", }) if code != ExitOK { @@ -1825,20 +1836,67 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) { } } +func TestProfileInspectionUsesCombinedSourcesAndValidatesUnusedPrompts(t *testing.T) { + lib := newCLITestLibrary(t) + combined := `backends: + fixture: + endpoint: http://127.0.0.1:1/v1 +profiles: + inspectable: + backend: fixture + model: fixture-model +prompts: + unused: + "1.0.0": + messages: + - role: user + content: "valid unused prompt" + output: + format: text + validation_mode: none +` + frameworkPath := writeFrameworkFile(t, lib.rootDir, "combined.yml", combined) + configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkPath)) + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ + "profile", "--config", configPath, "--profile", "inspectable", "--format", "json", + }) + if code != ExitOK { + t.Fatalf("combined-source profile inspection failed: code=%d out=%q err=%q", code, stdout, stderr) + } + var inspection renderformat.ProfileInspection + if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { + t.Fatalf("decode combined-source profile inspection: %v", err) + } + if inspection.EffectiveModelParams.BackendID != "fixture" || inspection.EffectiveModelParams.Model != "fixture-model" { + t.Fatalf("combined source did not provide the selected profile: %+v", inspection.EffectiveModelParams) + } + + invalidCombined := strings.Replace(combined, `content: "valid unused prompt"`, `content: "{{"`, 1) + invalidFrameworkPath := writeFrameworkFile(t, lib.rootDir, "combined-invalid.yml", invalidCombined) + invalidConfigPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", invalidFrameworkPath)) + code, stdout, stderr = runCLICommand(t, inspectCommand, []string{ + "profile", "--config", invalidConfigPath, "--profile", "inspectable", + }) + if code != ExitRuntimeError || stdout != "" || stderr == "" { + t.Fatalf("invalid unused prompt did not fail construction: code=%d out=%q err=%q", code, stdout, stderr) + } +} + func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.T) { promptDir := t.TempDir() profileDir := t.TempDir() - writePromptDefinition(t, promptDir, "maintained.yaml", `id: maintained -version: "1.0.0" -default_profile: promptkit-openrouter-standard -messages: - - role: user - content: Hello -output: - format: text - validation_mode: none + writePromptDefinition(t, promptDir, "maintained.yaml", `prompts: + maintained: + "1.0.0": + default_profile: promptkit-openrouter-standard + messages: + - role: user + content: Hello + output: + format: text + validation_mode: none `) - configPath := writeAppConfigFile(t, fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\n", promptDir, profileDir)) + configPath := writeAppConfigFile(t, sourceSelectionsYAML(promptDir, profileDir, "")) for _, profileID := range []string{"promptkit-openrouter-standard", "promptkit-openrouter-auto"} { t.Run(profileID+" inspection", func(t *testing.T) { @@ -1886,8 +1944,8 @@ output: }) } - writePromptDefinition(t, profileDir, "derived.yaml", "id: derived-maintained\nbase_profile: promptkit-openrouter-standard\nmodel: fixture-inherited-model\n") - writePromptDefinition(t, profileDir, "replacement.yaml", "id: promptkit-openrouter-auto\nbackend: openrouter\nmodel: fixture-override-model\n") + writePromptDefinition(t, profileDir, "derived.yaml", "profiles:\n derived-maintained:\n extends: promptkit-openrouter-standard\n model: fixture-inherited-model\n") + writePromptDefinition(t, profileDir, "replacement.yaml", "profiles:\n promptkit-openrouter-auto:\n backend: openrouter\n model: fixture-override-model\n") for _, tc := range []struct{ id, model string }{ {id: "derived-maintained", model: "fixture-inherited-model"}, {id: "promptkit-openrouter-auto", model: "fixture-override-model"}, @@ -1910,7 +1968,7 @@ output: func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.T) { fixtureRoot := definitionFixtureRoot(t) - configPath := writeDefinitionFixtureConfig(t, fixtureRoot, true) + configPath := writeDefinitionFixtureConfig(t, fixtureRoot) overrideDir := t.TempDir() writeProfileFile(t, overrideDir, "custom-derived", "http://127.0.0.1:9000/v1", "override-model") outPath := filepath.Join(t.TempDir(), "inspection.json") @@ -1918,7 +1976,9 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing. code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ "profile", "--config", configPath, + "--promptkit-source", filepath.Join(fixtureRoot, "prompts"), "--promptkit-source", overrideDir, + "--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"), "--profile", "custom-derived", "--format", "json", "--out", outPath, @@ -1936,18 +1996,18 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing. for _, profileID := range []string{"missing", "invalid"} { t.Run(profileID, func(t *testing.T) { - profileDir := filepath.Join(fixtureRoot, "profiles") + args := []string{"profile", "--config", configPath, "--profile", profileID} if profileID == "invalid" { - profileDir = t.TempDir() - writePromptDefinition(t, profileDir, "invalid.yaml", "id: invalid\nendpoint: not-a-url\nmodel: fixture\n") + profileDir := t.TempDir() + writePromptDefinition(t, profileDir, "invalid.yaml", "profiles:\n invalid:\n endpoint: not-a-url\n model: fixture\n") + args = append(args, + "--promptkit-source", filepath.Join(fixtureRoot, "prompts"), + "--promptkit-source", profileDir, + "--promptkit-source", filepath.Join(fixtureRoot, "framework", "backends.yml"), + ) } - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ - "profile", - "--config", configPath, - "--promptkit-source", profileDir, - "--profile", profileID, - }) - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") { + code, stdout, stderr := runCLICommand(t, inspectCommand, args) + if code != ExitRuntimeError || stdout != "" || (profileID == "invalid" && !strings.Contains(stderr, "endpoint must use http or https")) { t.Fatalf("expected inspection failure without partial output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) } }) @@ -1961,40 +2021,36 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) { name string backend string }{ - { - name: "reserved ID", - backend: `openrouter: - endpoint: http://localhost:11434/v1`, - }, { name: "invalid endpoint", - backend: `local-gpu: - endpoint: not-a-url`, + backend: `backends: + local-gpu: + endpoint: not-a-url +`, }, { name: "invalid capacity relationship", - backend: `local-gpu: + backend: `backends: + local-gpu: endpoint: http://localhost:11434/v1 - queue_capacity: 0`, + queue_capacity: 0 +`, }, { name: "reserved extra parameter", - backend: `local-gpu: + backend: `backends: + local-gpu: endpoint: http://localhost:11434/v1 extra_params: - model: forbidden`, + model: forbidden +`, }, } { t.Run(tc.name, func(t *testing.T) { - configPath := writeAppConfigFile(t, fmt.Sprintf(` -prompt_dir: %s -profile_dir: %s -backends: - %s -`, lib.promptDir, lib.profileDir, tc.backend)) + configPath := writeCLIFrameworkConfig(t, lib, tc.backend, "") cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"}) if err != nil { - t.Fatalf("expected config decoding to succeed, got %v", err) + t.Fatalf("expected application config decoding to succeed, got %v", err) } _, err = newEngine(cfg.runConfig.engineSettings()) if !errors.Is(err, promptkit.ErrInvalidConfig) { @@ -2005,20 +2061,38 @@ backends: } }) } + + t.Run("built-in backend overlay", func(t *testing.T) { + configPath := writeCLIFrameworkConfig(t, lib, `backends: + openrouter: + endpoint: http://localhost:11434/v1 +`, "") + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ + "profile", "--config", configPath, + "--profile", "promptkit-openrouter-auto", "--format", "json", + }) + if code != ExitOK { + t.Fatalf("built-in backend overlay failed: code=%d out=%q err=%q", code, stdout, stderr) + } + var inspection renderformat.ProfileInspection + if err := json.Unmarshal([]byte(stdout), &inspection); err != nil { + t.Fatalf("decode built-in profile inspection: %v", err) + } + if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" { + t.Fatalf("caller overlay did not update the built-in backend: %+v", inspection.EffectiveModelParams) + } + }) } func TestInheritedBackendParentStillReceivesPromptkitValidation(t *testing.T) { lib := newCLITestLibrary(t) - configPath := writeAppConfigFile(t, fmt.Sprintf(` -prompt_dir: %s -profile_dir: %s -backends: + configPath := writeCLIFrameworkConfig(t, lib, `backends: base: endpoint: not-a-url child: extends: base endpoint: http://localhost:11434/v1 -`, lib.promptDir, lib.profileDir)) +`, "") cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "custom"}) if err != nil { t.Fatalf("application loading rejected the configured parent before Promptkit: %v", err) @@ -2036,6 +2110,7 @@ func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", @@ -2058,6 +2133,7 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", @@ -2084,11 +2160,13 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) { } func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) { + lib := newCLITestLibrary(t) var stdout bytes.Buffer var stderr bytes.Buffer code := renderCommand([]string{ - "--promptkit-source", "./prompts", - "--promptkit-source", "./profiles", + "--config", lib.configPath, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "p", "--input", "transcript=./x.md", "--format", "yaml", @@ -2110,6 +2188,7 @@ func TestRenderCommandOutWritesToFile(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", @@ -2139,6 +2218,7 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", @@ -2165,6 +2245,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) { writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "prompt.builtin", "--input", "transcript=" + inputPath, @@ -2189,6 +2270,7 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", @@ -2212,74 +2294,77 @@ func TestRenderCommandUsesDefinitionInputRulesAndPromptVersions(t *testing.T) { lib := newCLITestLibrary(t) writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "model") - writePromptDefinition(t, lib.promptDir, "sole.yaml", `id: sole -version: "1" -default_profile: local -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "sole.yaml", `prompts: + sole: + "1": + default_profile: local + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - writePromptDefinition(t, lib.promptDir, "versioned-one.yaml", `id: versioned -version: "1" -default_profile: local -messages: - - role: user - content: "one" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "versioned.yaml", `prompts: + versioned: + "1": + default_profile: local + messages: + - role: user + content: "one" + output: + format: text + validation_mode: none + "2": + default_profile: local + messages: + - role: user + content: "two" + output: + format: text + validation_mode: none `) - writePromptDefinition(t, lib.promptDir, "versioned-two.yaml", `id: versioned -version: "2" -default_profile: local -messages: - - role: user - content: "two" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "optional.yaml", `prompts: + optional: + "1": + default_profile: local + inputs: + - name: note + required: false + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - writePromptDefinition(t, lib.promptDir, "optional.yaml", `id: optional -version: "1" -default_profile: local -inputs: - - name: note - required: false -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "required.yaml", `prompts: + required: + "1": + default_profile: local + inputs: + - name: note + required: true + messages: + - role: user + content: "hello" + output: + format: text + validation_mode: none `) - writePromptDefinition(t, lib.promptDir, "required.yaml", `id: required -version: "1" -default_profile: local -inputs: - - name: note - required: true -messages: - - role: user - content: "hello" -output: - format: text - validation_mode: none -`) - writePromptDefinition(t, lib.promptDir, "template.yaml", `id: template -version: "1" -default_profile: local -messages: - - role: user - content: '{{input "note"}}' -output: - format: text - validation_mode: none + writePromptDefinition(t, lib.promptDir, "template.yaml", `prompts: + template: + "1": + default_profile: local + messages: + - role: user + content: '{{input "note"}}' + output: + format: text + validation_mode: none `) - baseArgs := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir} + baseArgs := []string{"--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir} for _, tc := range []struct { name string args []string @@ -2288,7 +2373,7 @@ output: }{ {name: "sole version selected when omitted", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt_version: 1"}, {name: "explicit version selected", args: []string{"--prompt", "versioned", "--prompt-version", "2"}, wantCode: ExitOK, wantText: "prompt_version: 2"}, - {name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "duplicate prompt definition id"}, + {name: "multiple versions require selection", args: []string{"--prompt", "versioned"}, wantCode: ExitRuntimeError, wantText: "version is ambiguous"}, {name: "no declared inputs", args: []string{"--prompt", "sole"}, wantCode: ExitOK, wantText: "prompt: sole"}, {name: "optional input omitted", args: []string{"--prompt", "optional"}, wantCode: ExitOK, wantText: "prompt: optional"}, {name: "required input omitted", args: []string{"--prompt", "required"}, wantCode: ExitRuntimeError, wantText: "required"}, @@ -2317,6 +2402,7 @@ func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", @@ -2348,6 +2434,7 @@ func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) { writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", @@ -2388,6 +2475,7 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ + "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", @@ -2509,6 +2597,7 @@ type cliTestLibrary struct { rootDir string promptDir string profileDir string + configPath string } func newCLITestLibrary(t *testing.T) *cliTestLibrary { @@ -2518,6 +2607,7 @@ func newCLITestLibrary(t *testing.T) *cliTestLibrary { rootDir: root, promptDir: filepath.Join(root, "prompts"), profileDir: filepath.Join(root, "profiles"), + configPath: writeAppConfigFile(t, "{}\n"), } if err := os.MkdirAll(lib.promptDir, 0o755); err != nil { t.Fatalf("failed to create prompt fixture directory: %v", err) @@ -2560,22 +2650,19 @@ func definitionFixtureRoot(t *testing.T) string { return root } -func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string { +func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string) string { t.Helper() frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml") if _, err := os.Stat(frameworkPath); err != nil { t.Fatalf("stat framework backend fixture: %v", err) } - sources := "{}\n" - if includeSources { - sources = fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n", - filepath.Join(fixtureRoot, "prompts"), - filepath.Join(fixtureRoot, "profiles"), - frameworkPath, - filepath.Join(fixtureRoot, "schemas"), - ) - } + sources := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n", + filepath.Join(fixtureRoot, "prompts"), + filepath.Join(fixtureRoot, "profiles"), + frameworkPath, + filepath.Join(fixtureRoot, "schemas"), + ) return writeAppConfigFile(t, sources) } @@ -2639,3 +2726,30 @@ func writeAppConfigFile(t *testing.T, content string) string { } return path } + +func writeFrameworkFile(t *testing.T, root, name, definition string) string { + t.Helper() + dir := filepath.Join(root, "framework") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create framework fixture directory: %v", err) + } + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(definition), 0o644); err != nil { + t.Fatalf("write framework fixture: %v", err) + } + return path +} + +func sourceSelectionsYAML(promptDir, profileDir, frameworkFile string) string { + selectors := fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n", promptDir, profileDir) + if frameworkFile != "" { + selectors += fmt.Sprintf(" - file: %q\n", frameworkFile) + } + return selectors +} + +func writeCLIFrameworkConfig(t *testing.T, lib *cliTestLibrary, frameworkDefinition, applicationFields string) string { + t.Helper() + frameworkPath := writeFrameworkFile(t, lib.rootDir, "definitions.yml", frameworkDefinition) + return writeAppConfigFile(t, sourceSelectionsYAML(lib.promptDir, lib.profileDir, frameworkPath)+applicationFields) +}