Migrate CLI tests to native PromptKit sources

This commit is contained in:
2026-10-04 12:04:14 +00:00
parent 2027b80311
commit 54c8e9011e
3 changed files with 286 additions and 396 deletions

View File

@@ -3,7 +3,8 @@
## Status, Baseline, And Authority
Planned on 2026-10-04 against Scriptorium
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. All thirteen stages are pending.
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–5 are complete; stages
6–13 are pending.
This plan is for a Luna coding agent: implement the stages in sequential order,
normally one stage per implementation prompt. The
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
@@ -333,7 +334,14 @@ all effective definitions can be constructed offline with their resource origins
## Stage 5: Migrate CLI Parsing And Application-Inspection Tests
Status: Pending. Depends on Stage 4.
Status: Complete. CLI parser/config tests now protect ordered repeatable source
selection and whole-list overrides, native shared definition helpers, retired
flags and application fields, and application-only inspection with missing or
invalid framework paths. Focused CLI parser/config tests and `internal/config`
pass, as does the native definition-fixture render. The production build passes.
Full CLI tests retain only the execution/inspection fixture migrations assigned
to Stage 6; repository-wide test, race, and vet checks also reach the HTTP/report
constructor migrations assigned to Stage 7.
Files: `internal/adapter/cli/config_test.go` and parser/configuration sections
and shared helpers of `run_test.go`.

View File

@@ -9,8 +9,6 @@ import (
"path/filepath"
"strings"
"testing"
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
)
func TestConfigCommandsInspectApplicationWithoutEngine(t *testing.T) {
@@ -45,17 +43,17 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
}
base := filepath.Join(dir, "config.yml")
for path, body := range map[string]string{
base: "configkit:\n imports:\n - path: fragment.yml\nprompt_dir: ' ./before/../prompts '\nprofile_dir: ./profiles\n",
base: "configkit:\n imports:\n - path: fragment.yml\npromptkit_sources:\n - directory: ' ./before/../prompts '\nschema_dir: ./schemas\n",
filepath.Join(dir, "fragment.yml"): "server:\n addr: :8081\n",
filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n",
filepath.Join(dir, "overlays", "local.yml"): "promptkit_sources:\n - directory: ./overlay-prompts\nserver:\n addr: :9090\n",
} {
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
args := []string{"--config", base, "--config-overlay", "local", "--profile-dir=", "--addr", ":7070"}
args := []string{"--config", base, "--config-overlay", "local", "--schema-dir=", "--addr", ":7070"}
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"show"}, args...))
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "prompt_dir: prompts") || !strings.Contains(stdout, "profile_dir: ''") || !strings.Contains(stdout, "addr: :7070") {
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "directory: overlay-prompts") || !strings.Contains(stdout, "schema_dir: ''") || !strings.Contains(stdout, "addr: :7070") || strings.Contains(stdout, "before/../prompts") {
t.Fatalf("resolved effective output changed: code=%d out=%q err=%q", code, stdout, stderr)
}
code, stdout, stderr = runCLICommand(t, configCommand, append([]string{"sources"}, args...))
@@ -73,121 +71,45 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
}
}
func TestConfigCommandsInspectInheritedBackends(t *testing.T) {
const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY"
const keyValue = "synthetic-inheritance-key-bytes"
path := writeAppConfigFile(t, `backends:
base:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY
concurrency_limit: 2
queue_capacity: 4
extra_params:
enabled: true
shared: visible-value
child:
extends: base
queue_capacity: 0
extra_params:
enabled: false
`)
t.Setenv(keyName, keyValue)
if err := os.Unsetenv(keyName); err != nil {
func TestApplicationInspectionDoesNotAcquireFrameworkSources(t *testing.T) {
dir := t.TempDir()
frameworkPath := filepath.Join(dir, "invalid-framework.yml")
if err := os.WriteFile(frameworkPath, []byte("not: [valid\n"), 0o600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
if code != ExitOK || stdout != "" || stderr != "" {
t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr)
}
if err := os.Setenv(keyName, keyValue); err != nil {
t.Fatal(err)
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path})
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
}
for _, fragment := range []string{
"endpoint: http://127.0.0.1:1/v1",
"api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY",
"shared: visible-value",
"enabled: false",
"queue_capacity: 0",
for _, frameworkSource := range []struct {
name string
path string
}{
{name: "invalid definition", path: frameworkPath},
{name: "missing path", path: filepath.Join(dir, "missing-framework.yml")},
} {
if !strings.Contains(stdout, fragment) {
t.Fatalf("config show omitted %q: %s", fragment, stdout)
}
}
if strings.Contains(stdout, "extends:") {
t.Fatalf("config show exposed the structural directive: %s", stdout)
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path})
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
}
for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} {
if !strings.Contains(stdout, fragment) {
t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout)
}
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path})
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) {
t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
}
code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path})
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr)
}
var sources []configurationSource
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" {
t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err)
}
}
func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) {
path := writeAppConfigFile(t, `backends:
worker:
endpoint: http://127.0.0.1:1/v1
extra_params:
extends: missing
`)
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") {
t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr)
}
}
func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) {
const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY"
const secret = "synthetic-secret-value-should-not-appear"
path := writeAppConfigFile(t, `backends:
synthetic:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_CONFIG_TEST_API_KEY
extra_params:
visible_parameter: synthetic-ordinary-value
`)
t.Setenv(name, secret)
for _, present := range []bool{true, false} {
if !present {
if err := os.Unsetenv(name); err != nil {
t.Fatal(err)
t.Run(frameworkSource.name, func(t *testing.T) {
path := writeAppConfigFile(t, "promptkit_sources:\n - file: "+frameworkSource.path+"\n")
for _, args := range [][]string{
{"validate"},
{"show"},
{"sources"},
{"explain", "--path", "/promptkit_sources/0/file"},
} {
code, stdout, stderr := runCLICommand(t, configCommand, append(args, "--config", path))
if code != ExitOK || stderr != "" {
t.Fatalf("application inspection acquired configured framework source: args=%q code=%d out=%q err=%q", args, code, stdout, stderr)
}
}
}
code, stdout, stderr := runCLICommand(t, configCommand, []string{"show", "--config", path})
if code != ExitOK || stderr != "" || !strings.Contains(stdout, name) || !strings.Contains(stdout, "synthetic-ordinary-value") || strings.Contains(stdout, secret) {
t.Fatalf("credential boundary changed with present=%t: code=%d out=%q err=%q", present, code, stdout, stderr)
}
})
}
result, err := appconfig.Load(appconfig.LoadOptions{Path: path, Explicit: true})
if err != nil {
t.Fatal(err)
}
settings := appconfig.Settings(result.Config)
if len(settings.Backends) != 1 || settings.Backends[0].ExtraParams["visible_parameter"] != "synthetic-ordinary-value" {
t.Fatalf("runtime backend parameters changed: %+v", settings.Backends)
}
func TestConfigCommandsRejectRetiredFrameworkFields(t *testing.T) {
for _, field := range []string{"prompt_dir", "profile_dir", "backends"} {
t.Run(field, func(t *testing.T) {
path := writeAppConfigFile(t, field+": {}\n")
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("retired application field %q was accepted: code=%d out=%q err=%q", field, code, stdout, stderr)
}
})
}
}
@@ -213,51 +135,36 @@ func TestConfigCommandFailuresDoNotProduceSuccessfulOutput(t *testing.T) {
}
}
func TestConfigExplainUsesResolvedHistoryAndKeepsParametersVisible(t *testing.T) {
func TestConfigExplainUsesResolvedHistory(t *testing.T) {
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil {
t.Fatal(err)
}
base := filepath.Join(dir, "config.yml")
for path, body := range map[string]string{
base: `prompt_dir: " ./prompts/../prompts "
base: `promptkit_sources:
- directory: " ./prompts/../prompts "
server:
addr: :8080
backends:
"team/blue~1":
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_KEY
extra_params:
"0": base-parameter
`,
filepath.Join(dir, "overlays", "local.yml"): `server:
addr: :9090
backends:
"team/blue~1":
extra_params:
"0": overlay-parameter
`,
filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n",
} {
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
const keyValue = "synthetic-key-bytes-must-not-appear"
t.Setenv("SCRIPTORIUM_EXPLAIN_KEY", keyValue)
args := []string{"--config", base, "--config-overlay", "local", "--addr", ":9090"}
for _, tc := range []struct {
pointer string
want []string
}{
{pointer: "/server/addr", want: []string{":8080", ":9090", "--addr"}},
{pointer: "/prompt_dir", want: []string{"normalized application path", "prompts"}},
{pointer: "/backends/team~1blue~01/extra_params/0", want: []string{"base-parameter", "overlay-parameter"}},
{pointer: "/backends/team~1blue~01/api_key_env", want: []string{"SCRIPTORIUM_EXPLAIN_KEY"}},
{pointer: "", want: []string{"base-parameter", "overlay-parameter", "SCRIPTORIUM_EXPLAIN_KEY"}},
{pointer: "/promptkit_sources/0/directory", want: []string{"normalized application path", "prompts"}},
{pointer: "", want: []string{"promptkit_sources", "server", "addr"}},
} {
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", tc.pointer}, args...))
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
t.Fatalf("explain %q failed or disclosed key: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr)
if code != ExitOK || stderr != "" {
t.Fatalf("explain %q failed: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr)
}
for _, want := range tc.want {
if !strings.Contains(stdout, want) {
@@ -265,13 +172,6 @@ backends:
}
}
}
if err := os.Unsetenv("SCRIPTORIUM_EXPLAIN_KEY"); err != nil {
t.Fatal(err)
}
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", "/backends/team~1blue~01/api_key_env"}, args...))
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "SCRIPTORIUM_EXPLAIN_KEY") || strings.Contains(stdout, keyValue) {
t.Fatalf("absent credential changed reference explanation: code=%d out=%q err=%q", code, stdout, stderr)
}
}
func TestConfigExplainRejectsUnknownOrMalformedPaths(t *testing.T) {

View File

@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
@@ -19,6 +20,7 @@ import (
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
)
@@ -66,30 +68,31 @@ func TestParseMappingsMalformed(t *testing.T) {
func TestParseRunArgsRequiredFlags(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
_, err := parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b"})
if err == nil {
t.Fatal("expected missing --prompt-dir error")
}
if !strings.Contains(err.Error(), "prompt directory is required") {
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
}
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--input", "a=b"})
if err != nil {
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
t.Fatalf("expected omitted source list to be accepted, got %v", err)
}
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
if len(cfg.promptkitSources) != 0 {
t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources)
}
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
_, err = parseRunArgs([]string{"--config", configPath})
if err == nil {
t.Fatal("expected missing --prompt error")
}
cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"})
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p"})
if err == nil {
t.Fatal("expected retired --prompt-dir flag to be rejected")
}
_, err = parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p"})
if err == nil {
t.Fatal("expected retired --profile-dir flag to be rejected")
}
cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt", "p"})
if err != nil {
t.Fatalf("expected omitted --input to be accepted, got %v", err)
t.Fatalf("expected omitted --input and source list to be accepted, got %v", err)
}
if len(cfg.inputRaw) != 0 {
t.Fatalf("expected no input mappings, got %#v", cfg.inputRaw)
@@ -97,9 +100,16 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
}
func TestParseRunArgsFlagMapping(t *testing.T) {
lib := newCLITestLibrary(t)
configPath := writeAppConfigFile(t, "{}\n")
sourceFile := filepath.Join(lib.rootDir, "promptkit,extra.yml")
if err := os.WriteFile(sourceFile, []byte("not parsed by application loading\n"), 0o600); err != nil {
t.Fatal(err)
}
cfg, err := parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--promptkit-source", lib.promptDir,
"--promptkit-source", sourceFile,
"--prompt", "prompt.a",
"--prompt-version", "2",
"--profile", "profile.a",
@@ -117,8 +127,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
if err != nil {
t.Fatalf("expected valid args, got %v", err)
}
if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") {
t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir)
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) {
t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources)
}
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
@@ -131,10 +141,60 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
}
}
func TestPromptKitSourceFlagsReplaceConfiguredSequenceAsOneOverride(t *testing.T) {
lib := newCLITestLibrary(t)
sourceFile := filepath.Join(lib.rootDir, "fixture,part.yml")
if err := os.WriteFile(sourceFile, []byte("contents are not read by application parsing\n"), 0o600); err != nil {
t.Fatal(err)
}
configPath := writeAppConfigFile(t, "promptkit_sources:\n - directory: ./configured-but-not-loaded\n")
omitted, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(omitted.promptkitSources, []appconfig.SourceSelection{{Directory: "configured-but-not-loaded"}}) {
t.Fatalf("omitting source flags changed configured selections: %#v", omitted.promptkitSources)
}
args := []string{"--config", configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile, "--prompt", "p"}
selected, err := parseRunArgs(args)
if err != nil {
t.Fatal(err)
}
want := []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}
if !reflect.DeepEqual(selected.promptkitSources, want) {
t.Fatalf("explicit source flags did not replace config in argument order: got %#v, want %#v", selected.promptkitSources, want)
}
flags := flag.NewFlagSet("source override", flag.ContinueOnError)
flags.SetOutput(io.Discard)
var sources promptKitSourceFlagValue
flags.Var(&sources, "promptkit-source", "")
if err := flags.Parse([]string{"--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile}); err != nil {
t.Fatal(err)
}
overrides := applicationOverrides(flags)
if len(overrides) != 1 || overrides[0].Path.String() != "/promptkit_sources" || overrides[0].Flag != "--promptkit-source" {
t.Fatalf("expected one whole-list source override, got %#v", overrides)
}
if !reflect.DeepEqual(overrides[0].Value, []any{
map[string]any{"directory": lib.promptDir},
map[string]any{"file": sourceFile},
}) {
t.Fatalf("source override is not a plain ordered sequence of maps: %#v", overrides[0].Value)
}
_, err = parseRunArgs([]string{"--promptkit-source", filepath.Join(lib.rootDir, "missing.yml"), "--prompt", "p"})
if err == nil {
t.Fatal("expected unavailable explicit source path to be rejected")
}
}
func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
cfg, err := parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
@@ -148,8 +208,6 @@ func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) {
_, err := parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--prompt", "p",
"--input", "a=b",
"--llm-api-key", "secret",
@@ -162,20 +220,12 @@ func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) {
func TestParseServeArgsRequiredFlags(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
_, err := parseServeArgs([]string{"--config", configPath, "--profile-dir", "./profiles"})
if err == nil {
t.Fatal("expected missing --prompt-dir error")
}
if !strings.Contains(err.Error(), "prompt directory is required") {
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
}
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
cfg, err := parseServeArgs([]string{"--config", configPath})
if err != nil {
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
t.Fatalf("expected source list to be optional, got %v", err)
}
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
if len(cfg.promptkitSources) != 0 {
t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources)
}
if cfg.addr != defaults.HTTPAddrDefault {
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
@@ -186,7 +236,7 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
}
func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
base := []string{"--prompt-dir", "./prompts", "--profile-dir", "./profiles"}
base := []string{}
tests := [][]string{
{"--llm-base-url", "http://localhost:8000/v1"},
{"--model", "gpt-4o-mini"},
@@ -219,6 +269,7 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
"--max-response-bytes",
"inspect prompt",
"inspect profile",
"--promptkit-source PATH",
} {
if !strings.Contains(usage, want) {
t.Fatalf("expected usage to include %q, got %q", want, usage)
@@ -227,9 +278,9 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
}
func TestParseRunArgsTimeout(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
cfg, err := parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
@@ -241,8 +292,7 @@ func TestParseRunArgsTimeout(t *testing.T) {
}
cfg, err = parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
"--timeout", "2m30s",
@@ -256,9 +306,9 @@ func TestParseRunArgsTimeout(t *testing.T) {
}
func TestParseRenderArgsDefaultsAndFormat(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
cfg, err := parseRenderArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
@@ -271,9 +321,9 @@ func TestParseRenderArgsDefaultsAndFormat(t *testing.T) {
}
func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
cfg, err := parseRenderArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
"--format", "text",
@@ -286,8 +336,7 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
}
cfg, err = parseRenderArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
"--format", "json",
@@ -300,8 +349,7 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
}
_, err = parseRenderArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
"--format", "yaml",
@@ -314,10 +362,11 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
}
}
func TestParseRunArgsWithExplicitConfigLoadsDirectories(t *testing.T) {
func TestParseRunArgsWithExplicitConfigLoadsSourceSelections(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
promptkit_sources:
- directory: ./from-config/prompts
- directory: ./from-config/profiles
schema_dir: ./from-config/schemas
`)
@@ -330,11 +379,8 @@ schema_dir: ./from-config/schemas
t.Fatalf("expected valid args, got %v", err)
}
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
}
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: "from-config/prompts"}, {Directory: "from-config/profiles"}}) {
t.Fatalf("unexpected source selections from config: %#v", cfg.promptkitSources)
}
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
@@ -369,17 +415,23 @@ func TestParseRunArgsInvalidExplicitConfigReturnsError(t *testing.T) {
}
}
func TestParseRunArgsCLIOverridesConfigDirectories(t *testing.T) {
func TestParseRunArgsCLIOverridesConfiguredSourceList(t *testing.T) {
lib := newCLITestLibrary(t)
file := filepath.Join(lib.rootDir, "explicit.yml")
if err := os.WriteFile(file, []byte("not read by application loading\n"), 0o600); err != nil {
t.Fatal(err)
}
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
promptkit_sources:
- directory: ./from-config/prompts
- directory: ./from-config/profiles
schema_dir: ./from-config/schemas
`)
cfg, err := parseRunArgs([]string{
"--config", configPath,
"--prompt-dir", "./from-cli/prompts",
"--profile-dir", "./from-cli/profiles",
"--promptkit-source", lib.promptDir,
"--promptkit-source", file,
"--schema-dir", "./from-cli/schemas",
"--prompt", "p",
"--input", "a=b",
@@ -388,11 +440,8 @@ schema_dir: ./from-config/schemas
t.Fatalf("expected valid args, got %v", err)
}
if cfg.promptDir != filepath.Clean("./from-cli/prompts") {
t.Fatalf("expected CLI prompt dir override, got %q", cfg.promptDir)
}
if cfg.profileDir != filepath.Clean("./from-cli/profiles") {
t.Fatalf("expected CLI profile dir override, got %q", cfg.profileDir)
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: file}}) {
t.Fatalf("expected explicit CLI list to replace config, got %#v", cfg.promptkitSources)
}
if cfg.schemaDir != filepath.Clean("./from-cli/schemas") {
t.Fatalf("expected CLI schema dir override, got %q", cfg.schemaDir)
@@ -400,39 +449,40 @@ schema_dir: ./from-config/schemas
}
func TestApplicationFlagsPreserveExplicitEmptyValues(t *testing.T) {
configPath := writeAppConfigFile(t, `prompt_dir: ./configured/prompts
profile_dir: ./configured/profiles
configPath := writeAppConfigFile(t, `promptkit_sources:
- directory: ./configured/prompts
- directory: ./configured/profiles
schema_dir: ./configured/schemas
server:
addr: ":9090"
artifact_root: ./configured/artifacts
`)
if _, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--prompt-dir="}); err == nil || !strings.Contains(err.Error(), "prompt directory is required") {
t.Fatalf("explicit empty prompt directory restored lower value: %v", err)
if _, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--promptkit-source="}); err == nil {
t.Fatal("blank explicit source path should be rejected")
}
render, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p", "--profile-dir=", "--schema-dir="})
if err != nil || render.profileDir != "" || render.schemaDir != "" {
t.Fatalf("render did not preserve explicit empty sources: cfg=%+v err=%v", render, err)
render, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p", "--schema-dir="})
if err != nil || len(render.promptkitSources) != 2 || render.schemaDir != "" {
t.Fatalf("render did not preserve configured sources and explicit empty schema dir: cfg=%+v err=%v", render, err)
}
serve, err := parseServeArgs([]string{"--config", configPath, "--addr=", "--artifact-root="})
if err != nil || serve.addr != "" || serve.artifactRoot != "" {
t.Fatalf("serve did not preserve explicit empty settings: cfg=%+v err=%v", serve, err)
}
prompt, err := parsePromptInspectionArgs([]string{"--config", configPath, "--prompt", "p", "--prompt-dir="})
prompt, err := parsePromptInspectionArgs([]string{"--config", configPath, "--prompt", "p"})
if err != nil {
t.Fatal(err)
}
if _, err := resolveAppSettingsForPromptInspection(prompt); err == nil || !strings.Contains(err.Error(), "prompt directory is required") {
t.Fatalf("prompt inspection restored lower directory: %v", err)
if settings, err := resolveAppSettingsForPromptInspection(prompt); err != nil || len(settings.promptkitSources) != 2 {
t.Fatalf("prompt inspection should accept configured source selectors: settings=%+v err=%v", settings, err)
}
profile, err := parseProfileInspectionArgs([]string{"--config", configPath, "--profile", "p", "--profile-dir="})
profile, err := parseProfileInspectionArgs([]string{"--config", configPath, "--profile", "p"})
if err != nil {
t.Fatal(err)
}
settings, err := resolveAppSettingsForProfileInspection(profile)
if err != nil || settings.profileDir != "" {
t.Fatalf("profile inspection required a source or restored lower value: settings=%+v err=%v", settings, err)
if err != nil || len(settings.promptkitSources) != 2 {
t.Fatalf("profile inspection should retain configured source selectors: settings=%+v err=%v", settings, err)
}
}
@@ -442,11 +492,11 @@ func TestApplicationLoadUsesNativeYAMLAndSafeDiagnostics(t *testing.T) {
"server: null\n",
"server:\n addr: null\n",
"server:\n unknown: 1\n",
"prompt_dir: &alias ./prompts\n",
"prompt_dir: ./prompts\n---\nprofile_dir: ./profiles\n",
"promptkit_sources: &alias []\n",
"promptkit_sources: []\n---\nschema_dir: ./schemas\n",
} {
path := writeAppConfigFile(t, body)
_, err := parseRunArgs([]string{"--config", path, "--prompt-dir", "./prompts", "--prompt", "p"})
_, err := parseRunArgs([]string{"--config", path, "--prompt", "p"})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("native YAML failure %q lost structured diagnostics: %v", body, err)
@@ -455,7 +505,7 @@ func TestApplicationLoadUsesNativeYAMLAndSafeDiagnostics(t *testing.T) {
const rejected = "sensitive-synthetic-marker"
path := writeAppConfigFile(t, "defaults:\n render_format: "+rejected+"\n")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt-dir", "./prompts", "--prompt", "p"})
code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt", "p"})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "validate.error") || strings.Contains(stderr, rejected) {
t.Fatalf("unsafe or misclassified load diagnostic: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -480,8 +530,9 @@ func TestApplicationDiagnosticWriterFailure(t *testing.T) {
func TestParseRenderArgsWithExplicitConfigLoadsDirectoriesAndFormat(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
promptkit_sources:
- directory: ./from-config/prompts
- directory: ./from-config/profiles
defaults:
render_format: json
`)
@@ -495,11 +546,8 @@ defaults:
t.Fatalf("expected valid args, got %v", err)
}
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
}
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
if len(cfg.promptkitSources) != 2 {
t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources)
}
if cfg.outputFormat != renderformat.PreparedRunFormatJSON {
t.Fatalf("expected render format from config, got %q", cfg.outputFormat)
@@ -508,8 +556,9 @@ defaults:
func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
promptkit_sources:
- directory: ./from-config/prompts
- directory: ./from-config/profiles
defaults:
render_format: json
`)
@@ -530,8 +579,9 @@ defaults:
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
promptkit_sources:
- directory: ./from-config/prompts
- directory: ./from-config/profiles
schema_dir: ./from-config/schemas
server:
addr: 127.0.0.1:9000
@@ -553,11 +603,8 @@ server:
t.Fatalf("expected valid args, got %v", err)
}
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
}
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
if len(cfg.promptkitSources) != 2 {
t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources)
}
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
@@ -581,8 +628,9 @@ server:
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./from-config/prompts
profile_dir: ./from-config/profiles
promptkit_sources:
- directory: ./from-config/prompts
- directory: ./from-config/profiles
schema_dir: ./from-config/schemas
server:
addr: 127.0.0.1:9000
@@ -599,11 +647,8 @@ server:
t.Fatalf("expected valid args, got %v", err)
}
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
}
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
if len(cfg.promptkitSources) != 2 {
t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources)
}
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
@@ -638,7 +683,6 @@ func TestParseServeArgsRejectsNegativeSizeLimits(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := parseServeArgs([]string{
"--prompt-dir", "./prompts",
tc.flag, "-1",
})
if err == nil {
@@ -670,7 +714,6 @@ func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
err := tc.parse([]string{
"--prompt-dir", "./prompts",
"--prompt", "p",
"--input", "a=b",
"--max-request-bytes", "1024",
@@ -683,9 +726,9 @@ func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) {
}
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
runCfg, err := parseRunArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "prompt-1",
"--profile", "profile-1",
"--input", "transcript=./transcript.md",
@@ -703,8 +746,7 @@ func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *tes
}
renderCfg, err := parseRenderArgs([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--config", configPath,
"--prompt", "prompt-1",
"--profile", "profile-1",
"--input", "transcript=./transcript.md",
@@ -818,75 +860,13 @@ func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
}
}
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
configPath := writeAppConfigFile(t, `
profile_dir: ./profiles
`)
_, err := parseRunArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
if err == nil {
t.Fatal("expected missing prompt_dir error")
func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 {
t.Fatalf("run should parse without caller sources: cfg=%+v err=%v", cfg, err)
}
if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") {
t.Fatalf("expected clear prompt_dir guidance, got %v", err)
}
}
func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./prompts
`)
cfg, err := parseRunArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
if err != nil {
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
}
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
}
}
func TestParseRenderArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
configPath := writeAppConfigFile(t, `
profile_dir: ./profiles
`)
_, err := parseRenderArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
if err == nil {
t.Fatal("expected missing prompt_dir error")
}
if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") {
t.Fatalf("expected clear prompt_dir guidance, got %v", err)
}
}
func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
configPath := writeAppConfigFile(t, `
prompt_dir: ./prompts
`)
cfg, err := parseRenderArgs([]string{
"--config", configPath,
"--prompt", "p",
"--input", "a=b",
})
if err != nil {
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
}
if cfg.profileDir != "" {
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
if cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 {
t.Fatalf("render should parse without caller sources: cfg=%+v err=%v", cfg, err)
}
}
@@ -910,8 +890,8 @@ func TestRunCommandVarsOptional(t *testing.T) {
var stderr bytes.Buffer
code := runCommand([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--promptkit-source", "./prompts",
"--promptkit-source", "./profiles",
"--prompt", "p",
"--input", "transcript=./t.md",
"--llm-base-url", "http://[::1",
@@ -971,8 +951,8 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
"--profile", "local-default",
"--input", "transcript=" + inputPath,
@@ -1033,8 +1013,8 @@ temperature: 0.7
}
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
"--input", "transcript=" + inputPath,
"--temperature", "0",
@@ -1246,8 +1226,8 @@ backends:
synthetic_count: 7
`, lib.promptDir, lib.profileDir, provider.URL+"/v1", credentialName))
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"})
if err != nil || len(cfg.backends) != 1 || cfg.backends[0].ExtraParams["synthetic_count"] != int64(7) {
t.Fatalf("ConfigKit integer did not reach runtime settings: cfg=%+v err=%v", cfg, err)
if err != nil {
t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err)
}
code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", configPath, "--prompt", "custom"})
if code != ExitOK {
@@ -1293,7 +1273,7 @@ reasoning_effort: low
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
} {
t.Run(tc.name, func(t *testing.T) {
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "session"}
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"}
args = append(args, tc.args...)
code, stdout, stderr := runCLICommand(t, renderCommand, args)
if code != ExitOK {
@@ -1331,8 +1311,8 @@ output:
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "plain",
})
if code != ExitOK {
@@ -1358,8 +1338,8 @@ output:
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
code, _, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "session",
"--session-id", strings.Repeat("x", 257),
})
@@ -1382,14 +1362,14 @@ output:
format: text
validation_mode: none
`)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"})
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
if code != ExitOK {
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
}
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
t.Fatalf("unexpected inspection output: %s", stdout)
}
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"})
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "missing"})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -1546,7 +1526,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
lib := newCLITestLibrary(t)
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
for _, tc := range []struct {
name string
flag []string
@@ -1601,7 +1581,7 @@ func TestRunCommandWritesSeparateExecutionReport(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
path := filepath.Join(t.TempDir(), "execution.json")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir,
"--prompt", "reported", "--input", "transcript=" + inputPath,
"--execution-report", path,
})
@@ -1712,7 +1692,7 @@ func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) {
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
root := t.TempDir()
args := []string{"--prompt-dir", root, "--prompt", "p"}
args := []string{"--promptkit-source", root, "--prompt", "p"}
path := filepath.Join(root, "execution.json")
for _, tail := range [][]string{
{"--execution-report", ""},
@@ -1837,7 +1817,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
code, _, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configWithoutPromptDir,
"--profile-dir", filepath.Join(fixtureRoot, "profiles"),
"--promptkit-source", filepath.Join(fixtureRoot, "profiles"),
"--profile", "custom-derived",
})
if code != ExitOK {
@@ -1938,7 +1918,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--profile-dir", overrideDir,
"--promptkit-source", overrideDir,
"--profile", "custom-derived",
"--format", "json",
"--out", outPath,
@@ -1964,7 +1944,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"profile",
"--config", configPath,
"--profile-dir", profileDir,
"--promptkit-source", profileDir,
"--profile", profileID,
})
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
@@ -2056,8 +2036,8 @@ func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
"--input", "transcript=" + inputPath,
"--format", "text",
@@ -2078,8 +2058,8 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
"--input", "transcript=" + inputPath,
"--format", "json",
@@ -2107,8 +2087,8 @@ func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
code := renderCommand([]string{
"--prompt-dir", "./prompts",
"--profile-dir", "./profiles",
"--promptkit-source", "./prompts",
"--promptkit-source", "./profiles",
"--prompt", "p",
"--input", "transcript=./x.md",
"--format", "yaml",
@@ -2130,8 +2110,8 @@ func TestRenderCommandOutWritesToFile(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.render",
"--input", "transcript=" + inputPath,
"--out", outPath,
@@ -2159,8 +2139,8 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
"--input", "transcript=" + inputPath,
})
@@ -2185,7 +2165,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--promptkit-source", lib.promptDir,
"--prompt", "prompt.builtin",
"--input", "transcript=" + inputPath,
})
@@ -2209,8 +2189,8 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
"--profile", "quality",
"--input", "transcript=" + inputPath,
@@ -2299,7 +2279,7 @@ output:
validation_mode: none
`)
baseArgs := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir}
baseArgs := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
for _, tc := range []struct {
name string
args []string
@@ -2337,8 +2317,8 @@ func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
"--input", "transcript=" + inputPath,
})
@@ -2368,8 +2348,8 @@ func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
"--profile", "quality",
"--input", "transcript=" + inputPath,
@@ -2408,8 +2388,8 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--prompt-dir", lib.promptDir,
"--profile-dir", lib.profileDir,
"--promptkit-source", lib.promptDir,
"--promptkit-source", lib.profileDir,
"--prompt", "prompt.default",
"--input", "transcript=" + inputPath,
"--llm-base-url", overrideServer.URL + "/v1",
@@ -2582,20 +2562,21 @@ func definitionFixtureRoot(t *testing.T) string {
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string {
t.Helper()
sources := ""
if includeSources {
sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas"))
frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml")
if _, err := os.Stat(frameworkPath); err != nil {
t.Fatalf("stat framework backend fixture: %v", err)
}
return writeAppConfigFile(t, sources+`backends:
fixture-custom:
endpoint: http://127.0.0.1:11434/v1
api_key_env: FIXTURE_BACKEND_API_KEY
extra_params:
backend_option:
enabled: true
concurrency_limit: 2
queue_capacity: 0
`)
sources := "{}\n"
if includeSources {
sources = fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n",
filepath.Join(fixtureRoot, "prompts"),
filepath.Join(fixtureRoot, "profiles"),
frameworkPath,
filepath.Join(fixtureRoot, "schemas"),
)
}
return writeAppConfigFile(t, sources)
}
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
@@ -2612,19 +2593,20 @@ func writePromptDefinition(t *testing.T, dir, name, definition string) {
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
t.Helper()
data := fmt.Sprintf(`id: %s
version: "1.0.0"
default_profile: %s
inputs:
- name: transcript
required: true
messages:
- role: user
content: %q
output:
format: text
validation_mode: none
repair_attempts: 0
data := fmt.Sprintf(`prompts:
%q:
"1.0.0":
default_profile: %q
inputs:
- name: transcript
required: true
messages:
- role: user
content: %q
output:
format: text
validation_mode: none
repair_attempts: 0
`, id, defaultProfile, templateContent)
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
t.Fatalf("failed to write prompt fixture: %v", err)
@@ -2633,7 +2615,7 @@ output:
func writeProfileFile(t *testing.T, dir, id, endpoint, model string) {
t.Helper()
data := fmt.Sprintf("id: %s\nendpoint: %s\nmodel: %s\n", id, endpoint, model)
data := fmt.Sprintf("profiles:\n %q:\n endpoint: %q\n model: %q\n", id, endpoint, model)
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
t.Fatalf("failed to write profile fixture: %v", err)
}