diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5ae4ea3..6751265 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -3,7 +3,8 @@ ## Status, Baseline, And Authority Planned on 2026-10-04 against Scriptorium -`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. All thirteen stages are pending. +`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–5 are complete; stages +6–13 are pending. This plan is for a Luna coding agent: implement the stages in sequential order, normally one stage per implementation prompt. The [feature roadmap](promptkit.md) owns the target state, scope, and confirmed @@ -333,7 +334,14 @@ all effective definitions can be constructed offline with their resource origins ## Stage 5: Migrate CLI Parsing And Application-Inspection Tests -Status: Pending. Depends on Stage 4. +Status: Complete. CLI parser/config tests now protect ordered repeatable source +selection and whole-list overrides, native shared definition helpers, retired +flags and application fields, and application-only inspection with missing or +invalid framework paths. Focused CLI parser/config tests and `internal/config` +pass, as does the native definition-fixture render. The production build passes. +Full CLI tests retain only the execution/inspection fixture migrations assigned +to Stage 6; repository-wide test, race, and vet checks also reach the HTTP/report +constructor migrations assigned to Stage 7. Files: `internal/adapter/cli/config_test.go` and parser/configuration sections and shared helpers of `run_test.go`. diff --git a/internal/adapter/cli/config_test.go b/internal/adapter/cli/config_test.go index 452a308..2f753c1 100644 --- a/internal/adapter/cli/config_test.go +++ b/internal/adapter/cli/config_test.go @@ -9,8 +9,6 @@ import ( "path/filepath" "strings" "testing" - - appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" ) func TestConfigCommandsInspectApplicationWithoutEngine(t *testing.T) { @@ -45,17 +43,17 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) { } base := filepath.Join(dir, "config.yml") for path, body := range map[string]string{ - base: "configkit:\n imports:\n - path: fragment.yml\nprompt_dir: ' ./before/../prompts '\nprofile_dir: ./profiles\n", + base: "configkit:\n imports:\n - path: fragment.yml\npromptkit_sources:\n - directory: ' ./before/../prompts '\nschema_dir: ./schemas\n", filepath.Join(dir, "fragment.yml"): "server:\n addr: :8081\n", - filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n", + filepath.Join(dir, "overlays", "local.yml"): "promptkit_sources:\n - directory: ./overlay-prompts\nserver:\n addr: :9090\n", } { if err := os.WriteFile(path, []byte(body), 0o600); err != nil { t.Fatal(err) } } - args := []string{"--config", base, "--config-overlay", "local", "--profile-dir=", "--addr", ":7070"} + args := []string{"--config", base, "--config-overlay", "local", "--schema-dir=", "--addr", ":7070"} code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"show"}, args...)) - if code != ExitOK || stderr != "" || !strings.Contains(stdout, "prompt_dir: prompts") || !strings.Contains(stdout, "profile_dir: ''") || !strings.Contains(stdout, "addr: :7070") { + if code != ExitOK || stderr != "" || !strings.Contains(stdout, "directory: overlay-prompts") || !strings.Contains(stdout, "schema_dir: ''") || !strings.Contains(stdout, "addr: :7070") || strings.Contains(stdout, "before/../prompts") { t.Fatalf("resolved effective output changed: code=%d out=%q err=%q", code, stdout, stderr) } code, stdout, stderr = runCLICommand(t, configCommand, append([]string{"sources"}, args...)) @@ -73,121 +71,45 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) { } } -func TestConfigCommandsInspectInheritedBackends(t *testing.T) { - const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY" - const keyValue = "synthetic-inheritance-key-bytes" - path := writeAppConfigFile(t, `backends: - base: - endpoint: http://127.0.0.1:1/v1 - api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY - concurrency_limit: 2 - queue_capacity: 4 - extra_params: - enabled: true - shared: visible-value - child: - extends: base - queue_capacity: 0 - extra_params: - enabled: false -`) - t.Setenv(keyName, keyValue) - if err := os.Unsetenv(keyName); err != nil { +func TestApplicationInspectionDoesNotAcquireFrameworkSources(t *testing.T) { + dir := t.TempDir() + frameworkPath := filepath.Join(dir, "invalid-framework.yml") + if err := os.WriteFile(frameworkPath, []byte("not: [valid\n"), 0o600); err != nil { t.Fatal(err) } - code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path}) - if code != ExitOK || stdout != "" || stderr != "" { - t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr) - } - if err := os.Setenv(keyName, keyValue); err != nil { - t.Fatal(err) - } - - code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path}) - if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) { - t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr) - } - for _, fragment := range []string{ - "endpoint: http://127.0.0.1:1/v1", - "api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY", - "shared: visible-value", - "enabled: false", - "queue_capacity: 0", + for _, frameworkSource := range []struct { + name string + path string + }{ + {name: "invalid definition", path: frameworkPath}, + {name: "missing path", path: filepath.Join(dir, "missing-framework.yml")}, } { - if !strings.Contains(stdout, fragment) { - t.Fatalf("config show omitted %q: %s", fragment, stdout) - } - } - if strings.Contains(stdout, "extends:") { - t.Fatalf("config show exposed the structural directive: %s", stdout) - } - - code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path}) - if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) { - t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr) - } - for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} { - if !strings.Contains(stdout, fragment) { - t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout) - } - } - code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path}) - if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) { - t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr) - } - - code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path}) - if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) { - t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr) - } - var sources []configurationSource - if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" { - t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err) - } -} - -func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) { - path := writeAppConfigFile(t, `backends: - worker: - endpoint: http://127.0.0.1:1/v1 - extra_params: - extends: missing -`) - code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path}) - if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") { - t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr) - } -} - -func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) { - const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY" - const secret = "synthetic-secret-value-should-not-appear" - path := writeAppConfigFile(t, `backends: - synthetic: - endpoint: http://127.0.0.1:1/v1 - api_key_env: SCRIPTORIUM_CONFIG_TEST_API_KEY - extra_params: - visible_parameter: synthetic-ordinary-value -`) - t.Setenv(name, secret) - for _, present := range []bool{true, false} { - if !present { - if err := os.Unsetenv(name); err != nil { - t.Fatal(err) + t.Run(frameworkSource.name, func(t *testing.T) { + path := writeAppConfigFile(t, "promptkit_sources:\n - file: "+frameworkSource.path+"\n") + for _, args := range [][]string{ + {"validate"}, + {"show"}, + {"sources"}, + {"explain", "--path", "/promptkit_sources/0/file"}, + } { + code, stdout, stderr := runCLICommand(t, configCommand, append(args, "--config", path)) + if code != ExitOK || stderr != "" { + t.Fatalf("application inspection acquired configured framework source: args=%q code=%d out=%q err=%q", args, code, stdout, stderr) + } } - } - code, stdout, stderr := runCLICommand(t, configCommand, []string{"show", "--config", path}) - if code != ExitOK || stderr != "" || !strings.Contains(stdout, name) || !strings.Contains(stdout, "synthetic-ordinary-value") || strings.Contains(stdout, secret) { - t.Fatalf("credential boundary changed with present=%t: code=%d out=%q err=%q", present, code, stdout, stderr) - } + }) } - result, err := appconfig.Load(appconfig.LoadOptions{Path: path, Explicit: true}) - if err != nil { - t.Fatal(err) - } - settings := appconfig.Settings(result.Config) - if len(settings.Backends) != 1 || settings.Backends[0].ExtraParams["visible_parameter"] != "synthetic-ordinary-value" { - t.Fatalf("runtime backend parameters changed: %+v", settings.Backends) +} + +func TestConfigCommandsRejectRetiredFrameworkFields(t *testing.T) { + for _, field := range []string{"prompt_dir", "profile_dir", "backends"} { + t.Run(field, func(t *testing.T) { + path := writeAppConfigFile(t, field+": {}\n") + code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path}) + if code != ExitRuntimeError || stdout != "" || stderr == "" { + t.Fatalf("retired application field %q was accepted: code=%d out=%q err=%q", field, code, stdout, stderr) + } + }) } } @@ -213,51 +135,36 @@ func TestConfigCommandFailuresDoNotProduceSuccessfulOutput(t *testing.T) { } } -func TestConfigExplainUsesResolvedHistoryAndKeepsParametersVisible(t *testing.T) { +func TestConfigExplainUsesResolvedHistory(t *testing.T) { dir := t.TempDir() if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil { t.Fatal(err) } base := filepath.Join(dir, "config.yml") for path, body := range map[string]string{ - base: `prompt_dir: " ./prompts/../prompts " + base: `promptkit_sources: + - directory: " ./prompts/../prompts " server: addr: :8080 -backends: - "team/blue~1": - endpoint: http://127.0.0.1:1/v1 - api_key_env: SCRIPTORIUM_EXPLAIN_KEY - extra_params: - "0": base-parameter -`, - filepath.Join(dir, "overlays", "local.yml"): `server: - addr: :9090 -backends: - "team/blue~1": - extra_params: - "0": overlay-parameter `, + filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n", } { if err := os.WriteFile(path, []byte(body), 0o600); err != nil { t.Fatal(err) } } - const keyValue = "synthetic-key-bytes-must-not-appear" - t.Setenv("SCRIPTORIUM_EXPLAIN_KEY", keyValue) args := []string{"--config", base, "--config-overlay", "local", "--addr", ":9090"} for _, tc := range []struct { pointer string want []string }{ {pointer: "/server/addr", want: []string{":8080", ":9090", "--addr"}}, - {pointer: "/prompt_dir", want: []string{"normalized application path", "prompts"}}, - {pointer: "/backends/team~1blue~01/extra_params/0", want: []string{"base-parameter", "overlay-parameter"}}, - {pointer: "/backends/team~1blue~01/api_key_env", want: []string{"SCRIPTORIUM_EXPLAIN_KEY"}}, - {pointer: "", want: []string{"base-parameter", "overlay-parameter", "SCRIPTORIUM_EXPLAIN_KEY"}}, + {pointer: "/promptkit_sources/0/directory", want: []string{"normalized application path", "prompts"}}, + {pointer: "", want: []string{"promptkit_sources", "server", "addr"}}, } { code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", tc.pointer}, args...)) - if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) { - t.Fatalf("explain %q failed or disclosed key: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr) + if code != ExitOK || stderr != "" { + t.Fatalf("explain %q failed: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr) } for _, want := range tc.want { if !strings.Contains(stdout, want) { @@ -265,13 +172,6 @@ backends: } } } - if err := os.Unsetenv("SCRIPTORIUM_EXPLAIN_KEY"); err != nil { - t.Fatal(err) - } - code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", "/backends/team~1blue~01/api_key_env"}, args...)) - if code != ExitOK || stderr != "" || !strings.Contains(stdout, "SCRIPTORIUM_EXPLAIN_KEY") || strings.Contains(stdout, keyValue) { - t.Fatalf("absent credential changed reference explanation: code=%d out=%q err=%q", code, stdout, stderr) - } } func TestConfigExplainRejectsUnknownOrMalformedPaths(t *testing.T) { diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index b3a3586..6604bbd 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "flag" "fmt" "io" "net/http" @@ -19,6 +20,7 @@ import ( configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" + appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" "gitea.maximumdirect.net/eric/scriptorium/internal/defaults" renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" ) @@ -66,30 +68,31 @@ func TestParseMappingsMalformed(t *testing.T) { func TestParseRunArgsRequiredFlags(t *testing.T) { configPath := writeAppConfigFile(t, "{}\n") - _, err := parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b"}) - if err == nil { - t.Fatal("expected missing --prompt-dir error") - } - if !strings.Contains(err.Error(), "prompt directory is required") { - t.Fatalf("expected clear prompt-dir guidance, got %v", err) - } - - cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"}) + cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--input", "a=b"}) if err != nil { - t.Fatalf("expected missing --profile-dir to be accepted, got %v", err) + t.Fatalf("expected omitted source list to be accepted, got %v", err) } - if cfg.profileDir != "" { - t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) + if len(cfg.promptkitSources) != 0 { + t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources) } - _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"}) + _, err = parseRunArgs([]string{"--config", configPath}) if err == nil { t.Fatal("expected missing --prompt error") } - cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"}) + _, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p"}) + if err == nil { + t.Fatal("expected retired --prompt-dir flag to be rejected") + } + _, err = parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p"}) + if err == nil { + t.Fatal("expected retired --profile-dir flag to be rejected") + } + + cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt", "p"}) if err != nil { - t.Fatalf("expected omitted --input to be accepted, got %v", err) + t.Fatalf("expected omitted --input and source list to be accepted, got %v", err) } if len(cfg.inputRaw) != 0 { t.Fatalf("expected no input mappings, got %#v", cfg.inputRaw) @@ -97,9 +100,16 @@ func TestParseRunArgsRequiredFlags(t *testing.T) { } func TestParseRunArgsFlagMapping(t *testing.T) { + lib := newCLITestLibrary(t) + configPath := writeAppConfigFile(t, "{}\n") + sourceFile := filepath.Join(lib.rootDir, "promptkit,extra.yml") + if err := os.WriteFile(sourceFile, []byte("not parsed by application loading\n"), 0o600); err != nil { + t.Fatal(err) + } cfg, err := parseRunArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, + "--promptkit-source", lib.promptDir, + "--promptkit-source", sourceFile, "--prompt", "prompt.a", "--prompt-version", "2", "--profile", "profile.a", @@ -117,8 +127,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) { if err != nil { t.Fatalf("expected valid args, got %v", err) } - if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") { - t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir) + if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) { + t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources) } if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" { t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID) @@ -131,10 +141,60 @@ func TestParseRunArgsFlagMapping(t *testing.T) { } } +func TestPromptKitSourceFlagsReplaceConfiguredSequenceAsOneOverride(t *testing.T) { + lib := newCLITestLibrary(t) + sourceFile := filepath.Join(lib.rootDir, "fixture,part.yml") + if err := os.WriteFile(sourceFile, []byte("contents are not read by application parsing\n"), 0o600); err != nil { + t.Fatal(err) + } + configPath := writeAppConfigFile(t, "promptkit_sources:\n - directory: ./configured-but-not-loaded\n") + + omitted, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(omitted.promptkitSources, []appconfig.SourceSelection{{Directory: "configured-but-not-loaded"}}) { + t.Fatalf("omitting source flags changed configured selections: %#v", omitted.promptkitSources) + } + + args := []string{"--config", configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile, "--prompt", "p"} + selected, err := parseRunArgs(args) + if err != nil { + t.Fatal(err) + } + want := []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}} + if !reflect.DeepEqual(selected.promptkitSources, want) { + t.Fatalf("explicit source flags did not replace config in argument order: got %#v, want %#v", selected.promptkitSources, want) + } + + flags := flag.NewFlagSet("source override", flag.ContinueOnError) + flags.SetOutput(io.Discard) + var sources promptKitSourceFlagValue + flags.Var(&sources, "promptkit-source", "") + if err := flags.Parse([]string{"--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile}); err != nil { + t.Fatal(err) + } + overrides := applicationOverrides(flags) + if len(overrides) != 1 || overrides[0].Path.String() != "/promptkit_sources" || overrides[0].Flag != "--promptkit-source" { + t.Fatalf("expected one whole-list source override, got %#v", overrides) + } + if !reflect.DeepEqual(overrides[0].Value, []any{ + map[string]any{"directory": lib.promptDir}, + map[string]any{"file": sourceFile}, + }) { + t.Fatalf("source override is not a plain ordered sequence of maps: %#v", overrides[0].Value) + } + + _, err = parseRunArgs([]string{"--promptkit-source", filepath.Join(lib.rootDir, "missing.yml"), "--prompt", "p"}) + if err == nil { + t.Fatal("expected unavailable explicit source path to be rejected") + } +} + func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) { + configPath := writeAppConfigFile(t, "{}\n") cfg, err := parseRunArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "p", "--input", "a=b", }) @@ -148,8 +208,6 @@ func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) { func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) { _, err := parseRunArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b", "--llm-api-key", "secret", @@ -162,20 +220,12 @@ func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) { func TestParseServeArgsRequiredFlags(t *testing.T) { configPath := writeAppConfigFile(t, "{}\n") - _, err := parseServeArgs([]string{"--config", configPath, "--profile-dir", "./profiles"}) - if err == nil { - t.Fatal("expected missing --prompt-dir error") - } - if !strings.Contains(err.Error(), "prompt directory is required") { - t.Fatalf("expected clear prompt-dir guidance, got %v", err) - } - - cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"}) + cfg, err := parseServeArgs([]string{"--config", configPath}) if err != nil { - t.Fatalf("expected missing --profile-dir to be accepted, got %v", err) + t.Fatalf("expected source list to be optional, got %v", err) } - if cfg.profileDir != "" { - t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) + if len(cfg.promptkitSources) != 0 { + t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources) } if cfg.addr != defaults.HTTPAddrDefault { t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr) @@ -186,7 +236,7 @@ func TestParseServeArgsRequiredFlags(t *testing.T) { } func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) { - base := []string{"--prompt-dir", "./prompts", "--profile-dir", "./profiles"} + base := []string{} tests := [][]string{ {"--llm-base-url", "http://localhost:8000/v1"}, {"--model", "gpt-4o-mini"}, @@ -219,6 +269,7 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) { "--max-response-bytes", "inspect prompt", "inspect profile", + "--promptkit-source PATH", } { if !strings.Contains(usage, want) { t.Fatalf("expected usage to include %q, got %q", want, usage) @@ -227,9 +278,9 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) { } func TestParseRunArgsTimeout(t *testing.T) { + configPath := writeAppConfigFile(t, "{}\n") cfg, err := parseRunArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "p", "--input", "a=b", }) @@ -241,8 +292,7 @@ func TestParseRunArgsTimeout(t *testing.T) { } cfg, err = parseRunArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "p", "--input", "a=b", "--timeout", "2m30s", @@ -256,9 +306,9 @@ func TestParseRunArgsTimeout(t *testing.T) { } func TestParseRenderArgsDefaultsAndFormat(t *testing.T) { + configPath := writeAppConfigFile(t, "{}\n") cfg, err := parseRenderArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "p", "--input", "a=b", }) @@ -271,9 +321,9 @@ func TestParseRenderArgsDefaultsAndFormat(t *testing.T) { } func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) { + configPath := writeAppConfigFile(t, "{}\n") cfg, err := parseRenderArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "p", "--input", "a=b", "--format", "text", @@ -286,8 +336,7 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) { } cfg, err = parseRenderArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "p", "--input", "a=b", "--format", "json", @@ -300,8 +349,7 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) { } _, err = parseRenderArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "p", "--input", "a=b", "--format", "yaml", @@ -314,10 +362,11 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) { } } -func TestParseRunArgsWithExplicitConfigLoadsDirectories(t *testing.T) { +func TestParseRunArgsWithExplicitConfigLoadsSourceSelections(t *testing.T) { configPath := writeAppConfigFile(t, ` -prompt_dir: ./from-config/prompts -profile_dir: ./from-config/profiles +promptkit_sources: + - directory: ./from-config/prompts + - directory: ./from-config/profiles schema_dir: ./from-config/schemas `) @@ -330,11 +379,8 @@ schema_dir: ./from-config/schemas t.Fatalf("expected valid args, got %v", err) } - if cfg.promptDir != filepath.Clean("./from-config/prompts") { - t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) - } - if cfg.profileDir != filepath.Clean("./from-config/profiles") { - t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) + if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: "from-config/prompts"}, {Directory: "from-config/profiles"}}) { + t.Fatalf("unexpected source selections from config: %#v", cfg.promptkitSources) } if cfg.schemaDir != filepath.Clean("./from-config/schemas") { t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) @@ -369,17 +415,23 @@ func TestParseRunArgsInvalidExplicitConfigReturnsError(t *testing.T) { } } -func TestParseRunArgsCLIOverridesConfigDirectories(t *testing.T) { +func TestParseRunArgsCLIOverridesConfiguredSourceList(t *testing.T) { + lib := newCLITestLibrary(t) + file := filepath.Join(lib.rootDir, "explicit.yml") + if err := os.WriteFile(file, []byte("not read by application loading\n"), 0o600); err != nil { + t.Fatal(err) + } configPath := writeAppConfigFile(t, ` -prompt_dir: ./from-config/prompts -profile_dir: ./from-config/profiles +promptkit_sources: + - directory: ./from-config/prompts + - directory: ./from-config/profiles schema_dir: ./from-config/schemas `) cfg, err := parseRunArgs([]string{ "--config", configPath, - "--prompt-dir", "./from-cli/prompts", - "--profile-dir", "./from-cli/profiles", + "--promptkit-source", lib.promptDir, + "--promptkit-source", file, "--schema-dir", "./from-cli/schemas", "--prompt", "p", "--input", "a=b", @@ -388,11 +440,8 @@ schema_dir: ./from-config/schemas t.Fatalf("expected valid args, got %v", err) } - if cfg.promptDir != filepath.Clean("./from-cli/prompts") { - t.Fatalf("expected CLI prompt dir override, got %q", cfg.promptDir) - } - if cfg.profileDir != filepath.Clean("./from-cli/profiles") { - t.Fatalf("expected CLI profile dir override, got %q", cfg.profileDir) + if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: file}}) { + t.Fatalf("expected explicit CLI list to replace config, got %#v", cfg.promptkitSources) } if cfg.schemaDir != filepath.Clean("./from-cli/schemas") { t.Fatalf("expected CLI schema dir override, got %q", cfg.schemaDir) @@ -400,39 +449,40 @@ schema_dir: ./from-config/schemas } func TestApplicationFlagsPreserveExplicitEmptyValues(t *testing.T) { - configPath := writeAppConfigFile(t, `prompt_dir: ./configured/prompts -profile_dir: ./configured/profiles + configPath := writeAppConfigFile(t, `promptkit_sources: + - directory: ./configured/prompts + - directory: ./configured/profiles schema_dir: ./configured/schemas server: addr: ":9090" artifact_root: ./configured/artifacts `) - if _, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--prompt-dir="}); err == nil || !strings.Contains(err.Error(), "prompt directory is required") { - t.Fatalf("explicit empty prompt directory restored lower value: %v", err) + if _, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--promptkit-source="}); err == nil { + t.Fatal("blank explicit source path should be rejected") } - render, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p", "--profile-dir=", "--schema-dir="}) - if err != nil || render.profileDir != "" || render.schemaDir != "" { - t.Fatalf("render did not preserve explicit empty sources: cfg=%+v err=%v", render, err) + render, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p", "--schema-dir="}) + if err != nil || len(render.promptkitSources) != 2 || render.schemaDir != "" { + t.Fatalf("render did not preserve configured sources and explicit empty schema dir: cfg=%+v err=%v", render, err) } serve, err := parseServeArgs([]string{"--config", configPath, "--addr=", "--artifact-root="}) if err != nil || serve.addr != "" || serve.artifactRoot != "" { t.Fatalf("serve did not preserve explicit empty settings: cfg=%+v err=%v", serve, err) } - prompt, err := parsePromptInspectionArgs([]string{"--config", configPath, "--prompt", "p", "--prompt-dir="}) + prompt, err := parsePromptInspectionArgs([]string{"--config", configPath, "--prompt", "p"}) if err != nil { t.Fatal(err) } - if _, err := resolveAppSettingsForPromptInspection(prompt); err == nil || !strings.Contains(err.Error(), "prompt directory is required") { - t.Fatalf("prompt inspection restored lower directory: %v", err) + if settings, err := resolveAppSettingsForPromptInspection(prompt); err != nil || len(settings.promptkitSources) != 2 { + t.Fatalf("prompt inspection should accept configured source selectors: settings=%+v err=%v", settings, err) } - profile, err := parseProfileInspectionArgs([]string{"--config", configPath, "--profile", "p", "--profile-dir="}) + profile, err := parseProfileInspectionArgs([]string{"--config", configPath, "--profile", "p"}) if err != nil { t.Fatal(err) } settings, err := resolveAppSettingsForProfileInspection(profile) - if err != nil || settings.profileDir != "" { - t.Fatalf("profile inspection required a source or restored lower value: settings=%+v err=%v", settings, err) + if err != nil || len(settings.promptkitSources) != 2 { + t.Fatalf("profile inspection should retain configured source selectors: settings=%+v err=%v", settings, err) } } @@ -442,11 +492,11 @@ func TestApplicationLoadUsesNativeYAMLAndSafeDiagnostics(t *testing.T) { "server: null\n", "server:\n addr: null\n", "server:\n unknown: 1\n", - "prompt_dir: &alias ./prompts\n", - "prompt_dir: ./prompts\n---\nprofile_dir: ./profiles\n", + "promptkit_sources: &alias []\n", + "promptkit_sources: []\n---\nschema_dir: ./schemas\n", } { path := writeAppConfigFile(t, body) - _, err := parseRunArgs([]string{"--config", path, "--prompt-dir", "./prompts", "--prompt", "p"}) + _, err := parseRunArgs([]string{"--config", path, "--prompt", "p"}) var loadError *configkit.LoadError if !errors.As(err, &loadError) { t.Fatalf("native YAML failure %q lost structured diagnostics: %v", body, err) @@ -455,7 +505,7 @@ func TestApplicationLoadUsesNativeYAMLAndSafeDiagnostics(t *testing.T) { const rejected = "sensitive-synthetic-marker" path := writeAppConfigFile(t, "defaults:\n render_format: "+rejected+"\n") - code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt-dir", "./prompts", "--prompt", "p"}) + code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt", "p"}) if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "validate.error") || strings.Contains(stderr, rejected) { t.Fatalf("unsafe or misclassified load diagnostic: code=%d stdout=%q stderr=%q", code, stdout, stderr) } @@ -480,8 +530,9 @@ func TestApplicationDiagnosticWriterFailure(t *testing.T) { func TestParseRenderArgsWithExplicitConfigLoadsDirectoriesAndFormat(t *testing.T) { configPath := writeAppConfigFile(t, ` -prompt_dir: ./from-config/prompts -profile_dir: ./from-config/profiles +promptkit_sources: + - directory: ./from-config/prompts + - directory: ./from-config/profiles defaults: render_format: json `) @@ -495,11 +546,8 @@ defaults: t.Fatalf("expected valid args, got %v", err) } - if cfg.promptDir != filepath.Clean("./from-config/prompts") { - t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) - } - if cfg.profileDir != filepath.Clean("./from-config/profiles") { - t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) + if len(cfg.promptkitSources) != 2 { + t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources) } if cfg.outputFormat != renderformat.PreparedRunFormatJSON { t.Fatalf("expected render format from config, got %q", cfg.outputFormat) @@ -508,8 +556,9 @@ defaults: func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) { configPath := writeAppConfigFile(t, ` -prompt_dir: ./from-config/prompts -profile_dir: ./from-config/profiles +promptkit_sources: + - directory: ./from-config/prompts + - directory: ./from-config/profiles defaults: render_format: json `) @@ -530,8 +579,9 @@ defaults: func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) { configPath := writeAppConfigFile(t, ` -prompt_dir: ./from-config/prompts -profile_dir: ./from-config/profiles +promptkit_sources: + - directory: ./from-config/prompts + - directory: ./from-config/profiles schema_dir: ./from-config/schemas server: addr: 127.0.0.1:9000 @@ -553,11 +603,8 @@ server: t.Fatalf("expected valid args, got %v", err) } - if cfg.promptDir != filepath.Clean("./from-config/prompts") { - t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) - } - if cfg.profileDir != filepath.Clean("./from-config/profiles") { - t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) + if len(cfg.promptkitSources) != 2 { + t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources) } if cfg.schemaDir != filepath.Clean("./from-config/schemas") { t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) @@ -581,8 +628,9 @@ server: func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) { configPath := writeAppConfigFile(t, ` -prompt_dir: ./from-config/prompts -profile_dir: ./from-config/profiles +promptkit_sources: + - directory: ./from-config/prompts + - directory: ./from-config/profiles schema_dir: ./from-config/schemas server: addr: 127.0.0.1:9000 @@ -599,11 +647,8 @@ server: t.Fatalf("expected valid args, got %v", err) } - if cfg.promptDir != filepath.Clean("./from-config/prompts") { - t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir) - } - if cfg.profileDir != filepath.Clean("./from-config/profiles") { - t.Fatalf("expected profile dir from config, got %q", cfg.profileDir) + if len(cfg.promptkitSources) != 2 { + t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources) } if cfg.schemaDir != filepath.Clean("./from-config/schemas") { t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir) @@ -638,7 +683,6 @@ func TestParseServeArgsRejectsNegativeSizeLimits(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { _, err := parseServeArgs([]string{ - "--prompt-dir", "./prompts", tc.flag, "-1", }) if err == nil { @@ -670,7 +714,6 @@ func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { err := tc.parse([]string{ - "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b", "--max-request-bytes", "1024", @@ -683,9 +726,9 @@ func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) { } func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) { + configPath := writeAppConfigFile(t, "{}\n") runCfg, err := parseRunArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "prompt-1", "--profile", "profile-1", "--input", "transcript=./transcript.md", @@ -703,8 +746,7 @@ func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *tes } renderCfg, err := parseRenderArgs([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--config", configPath, "--prompt", "prompt-1", "--profile", "profile-1", "--input", "transcript=./transcript.md", @@ -818,75 +860,13 @@ func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) { } } -func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) { - configPath := writeAppConfigFile(t, ` -profile_dir: ./profiles -`) - - _, err := parseRunArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err == nil { - t.Fatal("expected missing prompt_dir error") +func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) { + configPath := writeAppConfigFile(t, "{}\n") + if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 { + t.Fatalf("run should parse without caller sources: cfg=%+v err=%v", cfg, err) } - if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") { - t.Fatalf("expected clear prompt_dir guidance, got %v", err) - } -} - -func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) { - configPath := writeAppConfigFile(t, ` -prompt_dir: ./prompts -`) - - cfg, err := parseRunArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected missing profile_dir to be accepted, got %v", err) - } - if cfg.profileDir != "" { - t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) - } -} - -func TestParseRenderArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) { - configPath := writeAppConfigFile(t, ` -profile_dir: ./profiles -`) - - _, err := parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err == nil { - t.Fatal("expected missing prompt_dir error") - } - if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") { - t.Fatalf("expected clear prompt_dir guidance, got %v", err) - } -} - -func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) { - configPath := writeAppConfigFile(t, ` -prompt_dir: ./prompts -`) - - cfg, err := parseRenderArgs([]string{ - "--config", configPath, - "--prompt", "p", - "--input", "a=b", - }) - if err != nil { - t.Fatalf("expected missing profile_dir to be accepted, got %v", err) - } - if cfg.profileDir != "" { - t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir) + if cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 { + t.Fatalf("render should parse without caller sources: cfg=%+v err=%v", cfg, err) } } @@ -910,8 +890,8 @@ func TestRunCommandVarsOptional(t *testing.T) { var stderr bytes.Buffer code := runCommand([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--promptkit-source", "./prompts", + "--promptkit-source", "./profiles", "--prompt", "p", "--input", "transcript=./t.md", "--llm-base-url", "http://[::1", @@ -971,8 +951,8 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", "--profile", "local-default", "--input", "transcript=" + inputPath, @@ -1033,8 +1013,8 @@ temperature: 0.7 } code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--temperature", "0", @@ -1246,8 +1226,8 @@ backends: synthetic_count: 7 `, lib.promptDir, lib.profileDir, provider.URL+"/v1", credentialName)) cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"}) - if err != nil || len(cfg.backends) != 1 || cfg.backends[0].ExtraParams["synthetic_count"] != int64(7) { - t.Fatalf("ConfigKit integer did not reach runtime settings: cfg=%+v err=%v", cfg, err) + if err != nil { + t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err) } code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", configPath, "--prompt", "custom"}) if code != ExitOK { @@ -1293,7 +1273,7 @@ reasoning_effort: low {name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"}, } { t.Run(tc.name, func(t *testing.T) { - args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "session"} + args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"} args = append(args, tc.args...) code, stdout, stderr := runCLICommand(t, renderCommand, args) if code != ExitOK { @@ -1331,8 +1311,8 @@ output: writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "plain", }) if code != ExitOK { @@ -1358,8 +1338,8 @@ output: writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model") code, _, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "session", "--session-id", strings.Repeat("x", 257), }) @@ -1382,14 +1362,14 @@ output: format: text validation_mode: none `) - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"}) + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"}) if code != ExitOK { t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr) } if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) { t.Fatalf("unexpected inspection output: %s", stdout) } - code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"}) + code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "missing"}) if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") { t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr) } @@ -1546,7 +1526,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) { func TestRunTimeoutParsingAndCancellation(t *testing.T) { lib := newCLITestLibrary(t) inputPath := lib.writeInputFile(t, "transcript.md", "hello") - args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath} + args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath} for _, tc := range []struct { name string flag []string @@ -1601,7 +1581,7 @@ func TestRunCommandWritesSeparateExecutionReport(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model") path := filepath.Join(t.TempDir(), "execution.json") code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "reported", "--input", "transcript=" + inputPath, "--execution-report", path, }) @@ -1712,7 +1692,7 @@ func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) { func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) { root := t.TempDir() - args := []string{"--prompt-dir", root, "--prompt", "p"} + args := []string{"--promptkit-source", root, "--prompt", "p"} path := filepath.Join(root, "execution.json") for _, tail := range [][]string{ {"--execution-report", ""}, @@ -1837,7 +1817,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) { code, _, stderr := runCLICommand(t, inspectCommand, []string{ "profile", "--config", configWithoutPromptDir, - "--profile-dir", filepath.Join(fixtureRoot, "profiles"), + "--promptkit-source", filepath.Join(fixtureRoot, "profiles"), "--profile", "custom-derived", }) if code != ExitOK { @@ -1938,7 +1918,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing. code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ "profile", "--config", configPath, - "--profile-dir", overrideDir, + "--promptkit-source", overrideDir, "--profile", "custom-derived", "--format", "json", "--out", outPath, @@ -1964,7 +1944,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing. code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ "profile", "--config", configPath, - "--profile-dir", profileDir, + "--promptkit-source", profileDir, "--profile", profileID, }) if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") { @@ -2056,8 +2036,8 @@ func TestRenderCommandExplicitTextFormatWorks(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--format", "text", @@ -2078,8 +2058,8 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--format", "json", @@ -2107,8 +2087,8 @@ func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer code := renderCommand([]string{ - "--prompt-dir", "./prompts", - "--profile-dir", "./profiles", + "--promptkit-source", "./prompts", + "--promptkit-source", "./profiles", "--prompt", "p", "--input", "transcript=./x.md", "--format", "yaml", @@ -2130,8 +2110,8 @@ func TestRenderCommandOutWritesToFile(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.render", "--input", "transcript=" + inputPath, "--out", outPath, @@ -2159,8 +2139,8 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", "--input", "transcript=" + inputPath, }) @@ -2185,7 +2165,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) { writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, + "--promptkit-source", lib.promptDir, "--prompt", "prompt.builtin", "--input", "transcript=" + inputPath, }) @@ -2209,8 +2189,8 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) { writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model") code, stdout, stderr := runCLICommand(t, renderCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", "--profile", "quality", "--input", "transcript=" + inputPath, @@ -2299,7 +2279,7 @@ output: validation_mode: none `) - baseArgs := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir} + baseArgs := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir} for _, tc := range []struct { name string args []string @@ -2337,8 +2317,8 @@ func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", "--input", "transcript=" + inputPath, }) @@ -2368,8 +2348,8 @@ func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) { writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", "--profile", "quality", "--input", "transcript=" + inputPath, @@ -2408,8 +2388,8 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) { writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model") code, stdout, stderr := runCLICommand(t, runCommand, []string{ - "--prompt-dir", lib.promptDir, - "--profile-dir", lib.profileDir, + "--promptkit-source", lib.promptDir, + "--promptkit-source", lib.profileDir, "--prompt", "prompt.default", "--input", "transcript=" + inputPath, "--llm-base-url", overrideServer.URL + "/v1", @@ -2582,20 +2562,21 @@ func definitionFixtureRoot(t *testing.T) string { func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string { t.Helper() - sources := "" - if includeSources { - sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas")) + frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml") + if _, err := os.Stat(frameworkPath); err != nil { + t.Fatalf("stat framework backend fixture: %v", err) } - return writeAppConfigFile(t, sources+`backends: - fixture-custom: - endpoint: http://127.0.0.1:11434/v1 - api_key_env: FIXTURE_BACKEND_API_KEY - extra_params: - backend_option: - enabled: true - concurrency_limit: 2 - queue_capacity: 0 -`) + + sources := "{}\n" + if includeSources { + sources = fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n", + filepath.Join(fixtureRoot, "prompts"), + filepath.Join(fixtureRoot, "profiles"), + frameworkPath, + filepath.Join(fixtureRoot, "schemas"), + ) + } + return writeAppConfigFile(t, sources) } func writePromptFile(t *testing.T, dir, id, defaultProfile string) { @@ -2612,19 +2593,20 @@ func writePromptDefinition(t *testing.T, dir, name, definition string) { func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) { t.Helper() - data := fmt.Sprintf(`id: %s -version: "1.0.0" -default_profile: %s -inputs: - - name: transcript - required: true -messages: - - role: user - content: %q -output: - format: text - validation_mode: none - repair_attempts: 0 + data := fmt.Sprintf(`prompts: + %q: + "1.0.0": + default_profile: %q + inputs: + - name: transcript + required: true + messages: + - role: user + content: %q + output: + format: text + validation_mode: none + repair_attempts: 0 `, id, defaultProfile, templateContent) if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { t.Fatalf("failed to write prompt fixture: %v", err) @@ -2633,7 +2615,7 @@ output: func writeProfileFile(t *testing.T, dir, id, endpoint, model string) { t.Helper() - data := fmt.Sprintf("id: %s\nendpoint: %s\nmodel: %s\n", id, endpoint, model) + data := fmt.Sprintf("profiles:\n %q:\n endpoint: %q\n model: %q\n", id, endpoint, model) if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { t.Fatalf("failed to write profile fixture: %v", err) }