Migrate CLI tests to native PromptKit sources
This commit is contained in:
@@ -3,7 +3,8 @@
|
|||||||
## Status, Baseline, And Authority
|
## Status, Baseline, And Authority
|
||||||
|
|
||||||
Planned on 2026-10-04 against Scriptorium
|
Planned on 2026-10-04 against Scriptorium
|
||||||
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. All thirteen stages are pending.
|
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–5 are complete; stages
|
||||||
|
6–13 are pending.
|
||||||
This plan is for a Luna coding agent: implement the stages in sequential order,
|
This plan is for a Luna coding agent: implement the stages in sequential order,
|
||||||
normally one stage per implementation prompt. The
|
normally one stage per implementation prompt. The
|
||||||
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
||||||
@@ -333,7 +334,14 @@ all effective definitions can be constructed offline with their resource origins
|
|||||||
|
|
||||||
## Stage 5: Migrate CLI Parsing And Application-Inspection Tests
|
## Stage 5: Migrate CLI Parsing And Application-Inspection Tests
|
||||||
|
|
||||||
Status: Pending. Depends on Stage 4.
|
Status: Complete. CLI parser/config tests now protect ordered repeatable source
|
||||||
|
selection and whole-list overrides, native shared definition helpers, retired
|
||||||
|
flags and application fields, and application-only inspection with missing or
|
||||||
|
invalid framework paths. Focused CLI parser/config tests and `internal/config`
|
||||||
|
pass, as does the native definition-fixture render. The production build passes.
|
||||||
|
Full CLI tests retain only the execution/inspection fixture migrations assigned
|
||||||
|
to Stage 6; repository-wide test, race, and vet checks also reach the HTTP/report
|
||||||
|
constructor migrations assigned to Stage 7.
|
||||||
|
|
||||||
Files: `internal/adapter/cli/config_test.go` and parser/configuration sections
|
Files: `internal/adapter/cli/config_test.go` and parser/configuration sections
|
||||||
and shared helpers of `run_test.go`.
|
and shared helpers of `run_test.go`.
|
||||||
|
|||||||
@@ -9,8 +9,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestConfigCommandsInspectApplicationWithoutEngine(t *testing.T) {
|
func TestConfigCommandsInspectApplicationWithoutEngine(t *testing.T) {
|
||||||
@@ -45,17 +43,17 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
base := filepath.Join(dir, "config.yml")
|
base := filepath.Join(dir, "config.yml")
|
||||||
for path, body := range map[string]string{
|
for path, body := range map[string]string{
|
||||||
base: "configkit:\n imports:\n - path: fragment.yml\nprompt_dir: ' ./before/../prompts '\nprofile_dir: ./profiles\n",
|
base: "configkit:\n imports:\n - path: fragment.yml\npromptkit_sources:\n - directory: ' ./before/../prompts '\nschema_dir: ./schemas\n",
|
||||||
filepath.Join(dir, "fragment.yml"): "server:\n addr: :8081\n",
|
filepath.Join(dir, "fragment.yml"): "server:\n addr: :8081\n",
|
||||||
filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n",
|
filepath.Join(dir, "overlays", "local.yml"): "promptkit_sources:\n - directory: ./overlay-prompts\nserver:\n addr: :9090\n",
|
||||||
} {
|
} {
|
||||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
args := []string{"--config", base, "--config-overlay", "local", "--profile-dir=", "--addr", ":7070"}
|
args := []string{"--config", base, "--config-overlay", "local", "--schema-dir=", "--addr", ":7070"}
|
||||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"show"}, args...))
|
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"show"}, args...))
|
||||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "prompt_dir: prompts") || !strings.Contains(stdout, "profile_dir: ''") || !strings.Contains(stdout, "addr: :7070") {
|
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "directory: overlay-prompts") || !strings.Contains(stdout, "schema_dir: ''") || !strings.Contains(stdout, "addr: :7070") || strings.Contains(stdout, "before/../prompts") {
|
||||||
t.Fatalf("resolved effective output changed: code=%d out=%q err=%q", code, stdout, stderr)
|
t.Fatalf("resolved effective output changed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
code, stdout, stderr = runCLICommand(t, configCommand, append([]string{"sources"}, args...))
|
code, stdout, stderr = runCLICommand(t, configCommand, append([]string{"sources"}, args...))
|
||||||
@@ -73,121 +71,45 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigCommandsInspectInheritedBackends(t *testing.T) {
|
func TestApplicationInspectionDoesNotAcquireFrameworkSources(t *testing.T) {
|
||||||
const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY"
|
dir := t.TempDir()
|
||||||
const keyValue = "synthetic-inheritance-key-bytes"
|
frameworkPath := filepath.Join(dir, "invalid-framework.yml")
|
||||||
path := writeAppConfigFile(t, `backends:
|
if err := os.WriteFile(frameworkPath, []byte("not: [valid\n"), 0o600); err != nil {
|
||||||
base:
|
|
||||||
endpoint: http://127.0.0.1:1/v1
|
|
||||||
api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY
|
|
||||||
concurrency_limit: 2
|
|
||||||
queue_capacity: 4
|
|
||||||
extra_params:
|
|
||||||
enabled: true
|
|
||||||
shared: visible-value
|
|
||||||
child:
|
|
||||||
extends: base
|
|
||||||
queue_capacity: 0
|
|
||||||
extra_params:
|
|
||||||
enabled: false
|
|
||||||
`)
|
|
||||||
t.Setenv(keyName, keyValue)
|
|
||||||
if err := os.Unsetenv(keyName); err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
for _, frameworkSource := range []struct {
|
||||||
if code != ExitOK || stdout != "" || stderr != "" {
|
name string
|
||||||
t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr)
|
path string
|
||||||
}
|
}{
|
||||||
if err := os.Setenv(keyName, keyValue); err != nil {
|
{name: "invalid definition", path: frameworkPath},
|
||||||
t.Fatal(err)
|
{name: "missing path", path: filepath.Join(dir, "missing-framework.yml")},
|
||||||
}
|
|
||||||
|
|
||||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path})
|
|
||||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
|
||||||
t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
|
||||||
}
|
|
||||||
for _, fragment := range []string{
|
|
||||||
"endpoint: http://127.0.0.1:1/v1",
|
|
||||||
"api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY",
|
|
||||||
"shared: visible-value",
|
|
||||||
"enabled: false",
|
|
||||||
"queue_capacity: 0",
|
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(stdout, fragment) {
|
t.Run(frameworkSource.name, func(t *testing.T) {
|
||||||
t.Fatalf("config show omitted %q: %s", fragment, stdout)
|
path := writeAppConfigFile(t, "promptkit_sources:\n - file: "+frameworkSource.path+"\n")
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"validate"},
|
||||||
|
{"show"},
|
||||||
|
{"sources"},
|
||||||
|
{"explain", "--path", "/promptkit_sources/0/file"},
|
||||||
|
} {
|
||||||
|
code, stdout, stderr := runCLICommand(t, configCommand, append(args, "--config", path))
|
||||||
|
if code != ExitOK || stderr != "" {
|
||||||
|
t.Fatalf("application inspection acquired configured framework source: args=%q code=%d out=%q err=%q", args, code, stdout, stderr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if strings.Contains(stdout, "extends:") {
|
})
|
||||||
t.Fatalf("config show exposed the structural directive: %s", stdout)
|
|
||||||
}
|
|
||||||
|
|
||||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path})
|
|
||||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
|
||||||
t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
|
||||||
}
|
|
||||||
for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} {
|
|
||||||
if !strings.Contains(stdout, fragment) {
|
|
||||||
t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path})
|
|
||||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) {
|
|
||||||
t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
|
||||||
}
|
|
||||||
|
|
||||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path})
|
|
||||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
|
||||||
t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr)
|
|
||||||
}
|
|
||||||
var sources []configurationSource
|
|
||||||
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" {
|
|
||||||
t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) {
|
func TestConfigCommandsRejectRetiredFrameworkFields(t *testing.T) {
|
||||||
path := writeAppConfigFile(t, `backends:
|
for _, field := range []string{"prompt_dir", "profile_dir", "backends"} {
|
||||||
worker:
|
t.Run(field, func(t *testing.T) {
|
||||||
endpoint: http://127.0.0.1:1/v1
|
path := writeAppConfigFile(t, field+": {}\n")
|
||||||
extra_params:
|
|
||||||
extends: missing
|
|
||||||
`)
|
|
||||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") {
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||||
t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr)
|
t.Fatalf("retired application field %q was accepted: code=%d out=%q err=%q", field, code, stdout, stderr)
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
|
|
||||||
func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) {
|
|
||||||
const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY"
|
|
||||||
const secret = "synthetic-secret-value-should-not-appear"
|
|
||||||
path := writeAppConfigFile(t, `backends:
|
|
||||||
synthetic:
|
|
||||||
endpoint: http://127.0.0.1:1/v1
|
|
||||||
api_key_env: SCRIPTORIUM_CONFIG_TEST_API_KEY
|
|
||||||
extra_params:
|
|
||||||
visible_parameter: synthetic-ordinary-value
|
|
||||||
`)
|
|
||||||
t.Setenv(name, secret)
|
|
||||||
for _, present := range []bool{true, false} {
|
|
||||||
if !present {
|
|
||||||
if err := os.Unsetenv(name); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"show", "--config", path})
|
|
||||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, name) || !strings.Contains(stdout, "synthetic-ordinary-value") || strings.Contains(stdout, secret) {
|
|
||||||
t.Fatalf("credential boundary changed with present=%t: code=%d out=%q err=%q", present, code, stdout, stderr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
result, err := appconfig.Load(appconfig.LoadOptions{Path: path, Explicit: true})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
settings := appconfig.Settings(result.Config)
|
|
||||||
if len(settings.Backends) != 1 || settings.Backends[0].ExtraParams["visible_parameter"] != "synthetic-ordinary-value" {
|
|
||||||
t.Fatalf("runtime backend parameters changed: %+v", settings.Backends)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -213,51 +135,36 @@ func TestConfigCommandFailuresDoNotProduceSuccessfulOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigExplainUsesResolvedHistoryAndKeepsParametersVisible(t *testing.T) {
|
func TestConfigExplainUsesResolvedHistory(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
base := filepath.Join(dir, "config.yml")
|
base := filepath.Join(dir, "config.yml")
|
||||||
for path, body := range map[string]string{
|
for path, body := range map[string]string{
|
||||||
base: `prompt_dir: " ./prompts/../prompts "
|
base: `promptkit_sources:
|
||||||
|
- directory: " ./prompts/../prompts "
|
||||||
server:
|
server:
|
||||||
addr: :8080
|
addr: :8080
|
||||||
backends:
|
|
||||||
"team/blue~1":
|
|
||||||
endpoint: http://127.0.0.1:1/v1
|
|
||||||
api_key_env: SCRIPTORIUM_EXPLAIN_KEY
|
|
||||||
extra_params:
|
|
||||||
"0": base-parameter
|
|
||||||
`,
|
|
||||||
filepath.Join(dir, "overlays", "local.yml"): `server:
|
|
||||||
addr: :9090
|
|
||||||
backends:
|
|
||||||
"team/blue~1":
|
|
||||||
extra_params:
|
|
||||||
"0": overlay-parameter
|
|
||||||
`,
|
`,
|
||||||
|
filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n",
|
||||||
} {
|
} {
|
||||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const keyValue = "synthetic-key-bytes-must-not-appear"
|
|
||||||
t.Setenv("SCRIPTORIUM_EXPLAIN_KEY", keyValue)
|
|
||||||
args := []string{"--config", base, "--config-overlay", "local", "--addr", ":9090"}
|
args := []string{"--config", base, "--config-overlay", "local", "--addr", ":9090"}
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
pointer string
|
pointer string
|
||||||
want []string
|
want []string
|
||||||
}{
|
}{
|
||||||
{pointer: "/server/addr", want: []string{":8080", ":9090", "--addr"}},
|
{pointer: "/server/addr", want: []string{":8080", ":9090", "--addr"}},
|
||||||
{pointer: "/prompt_dir", want: []string{"normalized application path", "prompts"}},
|
{pointer: "/promptkit_sources/0/directory", want: []string{"normalized application path", "prompts"}},
|
||||||
{pointer: "/backends/team~1blue~01/extra_params/0", want: []string{"base-parameter", "overlay-parameter"}},
|
{pointer: "", want: []string{"promptkit_sources", "server", "addr"}},
|
||||||
{pointer: "/backends/team~1blue~01/api_key_env", want: []string{"SCRIPTORIUM_EXPLAIN_KEY"}},
|
|
||||||
{pointer: "", want: []string{"base-parameter", "overlay-parameter", "SCRIPTORIUM_EXPLAIN_KEY"}},
|
|
||||||
} {
|
} {
|
||||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", tc.pointer}, args...))
|
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", tc.pointer}, args...))
|
||||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
if code != ExitOK || stderr != "" {
|
||||||
t.Fatalf("explain %q failed or disclosed key: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr)
|
t.Fatalf("explain %q failed: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr)
|
||||||
}
|
}
|
||||||
for _, want := range tc.want {
|
for _, want := range tc.want {
|
||||||
if !strings.Contains(stdout, want) {
|
if !strings.Contains(stdout, want) {
|
||||||
@@ -265,13 +172,6 @@ backends:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := os.Unsetenv("SCRIPTORIUM_EXPLAIN_KEY"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", "/backends/team~1blue~01/api_key_env"}, args...))
|
|
||||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "SCRIPTORIUM_EXPLAIN_KEY") || strings.Contains(stdout, keyValue) {
|
|
||||||
t.Fatalf("absent credential changed reference explanation: code=%d out=%q err=%q", code, stdout, stderr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigExplainRejectsUnknownOrMalformedPaths(t *testing.T) {
|
func TestConfigExplainRejectsUnknownOrMalformedPaths(t *testing.T) {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -19,6 +20,7 @@ import (
|
|||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
)
|
)
|
||||||
@@ -66,30 +68,31 @@ func TestParseMappingsMalformed(t *testing.T) {
|
|||||||
func TestParseRunArgsRequiredFlags(t *testing.T) {
|
func TestParseRunArgsRequiredFlags(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, "{}\n")
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
|
||||||
_, err := parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p", "--input", "a=b"})
|
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--input", "a=b"})
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected missing --prompt-dir error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "prompt directory is required") {
|
|
||||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
t.Fatalf("expected omitted source list to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if cfg.profileDir != "" {
|
if len(cfg.promptkitSources) != 0 {
|
||||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
|
_, err = parseRunArgs([]string{"--config", configPath})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected missing --prompt error")
|
t.Fatal("expected missing --prompt error")
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--prompt", "p"})
|
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected retired --prompt-dir flag to be rejected")
|
||||||
|
}
|
||||||
|
_, err = parseRunArgs([]string{"--config", configPath, "--profile-dir", "./profiles", "--prompt", "p"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected retired --profile-dir flag to be rejected")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg, err = parseRunArgs([]string{"--config", configPath, "--prompt", "p"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected omitted --input to be accepted, got %v", err)
|
t.Fatalf("expected omitted --input and source list to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if len(cfg.inputRaw) != 0 {
|
if len(cfg.inputRaw) != 0 {
|
||||||
t.Fatalf("expected no input mappings, got %#v", cfg.inputRaw)
|
t.Fatalf("expected no input mappings, got %#v", cfg.inputRaw)
|
||||||
@@ -97,9 +100,16 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunArgsFlagMapping(t *testing.T) {
|
func TestParseRunArgsFlagMapping(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
sourceFile := filepath.Join(lib.rootDir, "promptkit,extra.yml")
|
||||||
|
if err := os.WriteFile(sourceFile, []byte("not parsed by application loading\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
cfg, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
"--promptkit-source", lib.promptDir,
|
||||||
|
"--promptkit-source", sourceFile,
|
||||||
"--prompt", "prompt.a",
|
"--prompt", "prompt.a",
|
||||||
"--prompt-version", "2",
|
"--prompt-version", "2",
|
||||||
"--profile", "profile.a",
|
"--profile", "profile.a",
|
||||||
@@ -117,8 +127,8 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
}
|
}
|
||||||
if cfg.promptDir != filepath.Clean("./prompts") || cfg.profileDir != filepath.Clean("./profiles") {
|
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}) {
|
||||||
t.Fatalf("unexpected dirs: prompt=%q profile=%q", cfg.promptDir, cfg.profileDir)
|
t.Fatalf("unexpected source selections: %#v", cfg.promptkitSources)
|
||||||
}
|
}
|
||||||
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
|
if cfg.promptID != "prompt.a" || cfg.promptVersion != "2" || cfg.profileID != "profile.a" {
|
||||||
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
|
t.Fatalf("unexpected prompt/version/profile ids: %q %q %q", cfg.promptID, cfg.promptVersion, cfg.profileID)
|
||||||
@@ -131,10 +141,60 @@ func TestParseRunArgsFlagMapping(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitSourceFlagsReplaceConfiguredSequenceAsOneOverride(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
sourceFile := filepath.Join(lib.rootDir, "fixture,part.yml")
|
||||||
|
if err := os.WriteFile(sourceFile, []byte("contents are not read by application parsing\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
configPath := writeAppConfigFile(t, "promptkit_sources:\n - directory: ./configured-but-not-loaded\n")
|
||||||
|
|
||||||
|
omitted, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(omitted.promptkitSources, []appconfig.SourceSelection{{Directory: "configured-but-not-loaded"}}) {
|
||||||
|
t.Fatalf("omitting source flags changed configured selections: %#v", omitted.promptkitSources)
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{"--config", configPath, "--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile, "--prompt", "p"}
|
||||||
|
selected, err := parseRunArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: sourceFile}}
|
||||||
|
if !reflect.DeepEqual(selected.promptkitSources, want) {
|
||||||
|
t.Fatalf("explicit source flags did not replace config in argument order: got %#v, want %#v", selected.promptkitSources, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := flag.NewFlagSet("source override", flag.ContinueOnError)
|
||||||
|
flags.SetOutput(io.Discard)
|
||||||
|
var sources promptKitSourceFlagValue
|
||||||
|
flags.Var(&sources, "promptkit-source", "")
|
||||||
|
if err := flags.Parse([]string{"--promptkit-source", lib.promptDir, "--promptkit-source", sourceFile}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overrides := applicationOverrides(flags)
|
||||||
|
if len(overrides) != 1 || overrides[0].Path.String() != "/promptkit_sources" || overrides[0].Flag != "--promptkit-source" {
|
||||||
|
t.Fatalf("expected one whole-list source override, got %#v", overrides)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(overrides[0].Value, []any{
|
||||||
|
map[string]any{"directory": lib.promptDir},
|
||||||
|
map[string]any{"file": sourceFile},
|
||||||
|
}) {
|
||||||
|
t.Fatalf("source override is not a plain ordered sequence of maps: %#v", overrides[0].Value)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = parseRunArgs([]string{"--promptkit-source", filepath.Join(lib.rootDir, "missing.yml"), "--prompt", "p"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected unavailable explicit source path to be rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
|
func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
cfg, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
})
|
})
|
||||||
@@ -148,8 +208,6 @@ func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) {
|
|||||||
|
|
||||||
func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) {
|
func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) {
|
||||||
_, err := parseRunArgs([]string{
|
_, err := parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--llm-api-key", "secret",
|
"--llm-api-key", "secret",
|
||||||
@@ -162,20 +220,12 @@ func TestParseRunArgsRejectsRawLLMAPIKeyFlag(t *testing.T) {
|
|||||||
func TestParseServeArgsRequiredFlags(t *testing.T) {
|
func TestParseServeArgsRequiredFlags(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, "{}\n")
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
|
||||||
_, err := parseServeArgs([]string{"--config", configPath, "--profile-dir", "./profiles"})
|
cfg, err := parseServeArgs([]string{"--config", configPath})
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected missing --prompt-dir error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "prompt directory is required") {
|
|
||||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
t.Fatalf("expected source list to be optional, got %v", err)
|
||||||
}
|
}
|
||||||
if cfg.profileDir != "" {
|
if len(cfg.promptkitSources) != 0 {
|
||||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
t.Fatalf("expected no caller sources, got %#v", cfg.promptkitSources)
|
||||||
}
|
}
|
||||||
if cfg.addr != defaults.HTTPAddrDefault {
|
if cfg.addr != defaults.HTTPAddrDefault {
|
||||||
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
||||||
@@ -186,7 +236,7 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
|
func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
|
||||||
base := []string{"--prompt-dir", "./prompts", "--profile-dir", "./profiles"}
|
base := []string{}
|
||||||
tests := [][]string{
|
tests := [][]string{
|
||||||
{"--llm-base-url", "http://localhost:8000/v1"},
|
{"--llm-base-url", "http://localhost:8000/v1"},
|
||||||
{"--model", "gpt-4o-mini"},
|
{"--model", "gpt-4o-mini"},
|
||||||
@@ -219,6 +269,7 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
|
|||||||
"--max-response-bytes",
|
"--max-response-bytes",
|
||||||
"inspect prompt",
|
"inspect prompt",
|
||||||
"inspect profile",
|
"inspect profile",
|
||||||
|
"--promptkit-source PATH",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(usage, want) {
|
if !strings.Contains(usage, want) {
|
||||||
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
||||||
@@ -227,9 +278,9 @@ func TestUsageIncludesExecutionAndServeFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunArgsTimeout(t *testing.T) {
|
func TestParseRunArgsTimeout(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
cfg, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
})
|
})
|
||||||
@@ -241,8 +292,7 @@ func TestParseRunArgsTimeout(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cfg, err = parseRunArgs([]string{
|
cfg, err = parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--timeout", "2m30s",
|
"--timeout", "2m30s",
|
||||||
@@ -256,9 +306,9 @@ func TestParseRunArgsTimeout(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRenderArgsDefaultsAndFormat(t *testing.T) {
|
func TestParseRenderArgsDefaultsAndFormat(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
cfg, err := parseRenderArgs([]string{
|
cfg, err := parseRenderArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
})
|
})
|
||||||
@@ -271,9 +321,9 @@ func TestParseRenderArgsDefaultsAndFormat(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
|
func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
cfg, err := parseRenderArgs([]string{
|
cfg, err := parseRenderArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--format", "text",
|
"--format", "text",
|
||||||
@@ -286,8 +336,7 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cfg, err = parseRenderArgs([]string{
|
cfg, err = parseRenderArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--format", "json",
|
"--format", "json",
|
||||||
@@ -300,8 +349,7 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseRenderArgs([]string{
|
_, err = parseRenderArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--format", "yaml",
|
"--format", "yaml",
|
||||||
@@ -314,10 +362,11 @@ func TestParseRenderArgsExplicitFormatsAndUnknown(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunArgsWithExplicitConfigLoadsDirectories(t *testing.T) {
|
func TestParseRunArgsWithExplicitConfigLoadsSourceSelections(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./from-config/prompts
|
promptkit_sources:
|
||||||
profile_dir: ./from-config/profiles
|
- directory: ./from-config/prompts
|
||||||
|
- directory: ./from-config/profiles
|
||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
`)
|
`)
|
||||||
|
|
||||||
@@ -330,11 +379,8 @@ schema_dir: ./from-config/schemas
|
|||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: "from-config/prompts"}, {Directory: "from-config/profiles"}}) {
|
||||||
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
t.Fatalf("unexpected source selections from config: %#v", cfg.promptkitSources)
|
||||||
}
|
|
||||||
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
||||||
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
||||||
}
|
}
|
||||||
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
||||||
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
||||||
@@ -369,17 +415,23 @@ func TestParseRunArgsInvalidExplicitConfigReturnsError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunArgsCLIOverridesConfigDirectories(t *testing.T) {
|
func TestParseRunArgsCLIOverridesConfiguredSourceList(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
file := filepath.Join(lib.rootDir, "explicit.yml")
|
||||||
|
if err := os.WriteFile(file, []byte("not read by application loading\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./from-config/prompts
|
promptkit_sources:
|
||||||
profile_dir: ./from-config/profiles
|
- directory: ./from-config/prompts
|
||||||
|
- directory: ./from-config/profiles
|
||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
`)
|
`)
|
||||||
|
|
||||||
cfg, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt-dir", "./from-cli/prompts",
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", "./from-cli/profiles",
|
"--promptkit-source", file,
|
||||||
"--schema-dir", "./from-cli/schemas",
|
"--schema-dir", "./from-cli/schemas",
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
@@ -388,11 +440,8 @@ schema_dir: ./from-config/schemas
|
|||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.promptDir != filepath.Clean("./from-cli/prompts") {
|
if !reflect.DeepEqual(cfg.promptkitSources, []appconfig.SourceSelection{{Directory: lib.promptDir}, {File: file}}) {
|
||||||
t.Fatalf("expected CLI prompt dir override, got %q", cfg.promptDir)
|
t.Fatalf("expected explicit CLI list to replace config, got %#v", cfg.promptkitSources)
|
||||||
}
|
|
||||||
if cfg.profileDir != filepath.Clean("./from-cli/profiles") {
|
|
||||||
t.Fatalf("expected CLI profile dir override, got %q", cfg.profileDir)
|
|
||||||
}
|
}
|
||||||
if cfg.schemaDir != filepath.Clean("./from-cli/schemas") {
|
if cfg.schemaDir != filepath.Clean("./from-cli/schemas") {
|
||||||
t.Fatalf("expected CLI schema dir override, got %q", cfg.schemaDir)
|
t.Fatalf("expected CLI schema dir override, got %q", cfg.schemaDir)
|
||||||
@@ -400,39 +449,40 @@ schema_dir: ./from-config/schemas
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestApplicationFlagsPreserveExplicitEmptyValues(t *testing.T) {
|
func TestApplicationFlagsPreserveExplicitEmptyValues(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `prompt_dir: ./configured/prompts
|
configPath := writeAppConfigFile(t, `promptkit_sources:
|
||||||
profile_dir: ./configured/profiles
|
- directory: ./configured/prompts
|
||||||
|
- directory: ./configured/profiles
|
||||||
schema_dir: ./configured/schemas
|
schema_dir: ./configured/schemas
|
||||||
server:
|
server:
|
||||||
addr: ":9090"
|
addr: ":9090"
|
||||||
artifact_root: ./configured/artifacts
|
artifact_root: ./configured/artifacts
|
||||||
`)
|
`)
|
||||||
if _, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--prompt-dir="}); err == nil || !strings.Contains(err.Error(), "prompt directory is required") {
|
if _, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p", "--promptkit-source="}); err == nil {
|
||||||
t.Fatalf("explicit empty prompt directory restored lower value: %v", err)
|
t.Fatal("blank explicit source path should be rejected")
|
||||||
}
|
}
|
||||||
render, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p", "--profile-dir=", "--schema-dir="})
|
render, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p", "--schema-dir="})
|
||||||
if err != nil || render.profileDir != "" || render.schemaDir != "" {
|
if err != nil || len(render.promptkitSources) != 2 || render.schemaDir != "" {
|
||||||
t.Fatalf("render did not preserve explicit empty sources: cfg=%+v err=%v", render, err)
|
t.Fatalf("render did not preserve configured sources and explicit empty schema dir: cfg=%+v err=%v", render, err)
|
||||||
}
|
}
|
||||||
serve, err := parseServeArgs([]string{"--config", configPath, "--addr=", "--artifact-root="})
|
serve, err := parseServeArgs([]string{"--config", configPath, "--addr=", "--artifact-root="})
|
||||||
if err != nil || serve.addr != "" || serve.artifactRoot != "" {
|
if err != nil || serve.addr != "" || serve.artifactRoot != "" {
|
||||||
t.Fatalf("serve did not preserve explicit empty settings: cfg=%+v err=%v", serve, err)
|
t.Fatalf("serve did not preserve explicit empty settings: cfg=%+v err=%v", serve, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
prompt, err := parsePromptInspectionArgs([]string{"--config", configPath, "--prompt", "p", "--prompt-dir="})
|
prompt, err := parsePromptInspectionArgs([]string{"--config", configPath, "--prompt", "p"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := resolveAppSettingsForPromptInspection(prompt); err == nil || !strings.Contains(err.Error(), "prompt directory is required") {
|
if settings, err := resolveAppSettingsForPromptInspection(prompt); err != nil || len(settings.promptkitSources) != 2 {
|
||||||
t.Fatalf("prompt inspection restored lower directory: %v", err)
|
t.Fatalf("prompt inspection should accept configured source selectors: settings=%+v err=%v", settings, err)
|
||||||
}
|
}
|
||||||
profile, err := parseProfileInspectionArgs([]string{"--config", configPath, "--profile", "p", "--profile-dir="})
|
profile, err := parseProfileInspectionArgs([]string{"--config", configPath, "--profile", "p"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
settings, err := resolveAppSettingsForProfileInspection(profile)
|
settings, err := resolveAppSettingsForProfileInspection(profile)
|
||||||
if err != nil || settings.profileDir != "" {
|
if err != nil || len(settings.promptkitSources) != 2 {
|
||||||
t.Fatalf("profile inspection required a source or restored lower value: settings=%+v err=%v", settings, err)
|
t.Fatalf("profile inspection should retain configured source selectors: settings=%+v err=%v", settings, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -442,11 +492,11 @@ func TestApplicationLoadUsesNativeYAMLAndSafeDiagnostics(t *testing.T) {
|
|||||||
"server: null\n",
|
"server: null\n",
|
||||||
"server:\n addr: null\n",
|
"server:\n addr: null\n",
|
||||||
"server:\n unknown: 1\n",
|
"server:\n unknown: 1\n",
|
||||||
"prompt_dir: &alias ./prompts\n",
|
"promptkit_sources: &alias []\n",
|
||||||
"prompt_dir: ./prompts\n---\nprofile_dir: ./profiles\n",
|
"promptkit_sources: []\n---\nschema_dir: ./schemas\n",
|
||||||
} {
|
} {
|
||||||
path := writeAppConfigFile(t, body)
|
path := writeAppConfigFile(t, body)
|
||||||
_, err := parseRunArgs([]string{"--config", path, "--prompt-dir", "./prompts", "--prompt", "p"})
|
_, err := parseRunArgs([]string{"--config", path, "--prompt", "p"})
|
||||||
var loadError *configkit.LoadError
|
var loadError *configkit.LoadError
|
||||||
if !errors.As(err, &loadError) {
|
if !errors.As(err, &loadError) {
|
||||||
t.Fatalf("native YAML failure %q lost structured diagnostics: %v", body, err)
|
t.Fatalf("native YAML failure %q lost structured diagnostics: %v", body, err)
|
||||||
@@ -455,7 +505,7 @@ func TestApplicationLoadUsesNativeYAMLAndSafeDiagnostics(t *testing.T) {
|
|||||||
|
|
||||||
const rejected = "sensitive-synthetic-marker"
|
const rejected = "sensitive-synthetic-marker"
|
||||||
path := writeAppConfigFile(t, "defaults:\n render_format: "+rejected+"\n")
|
path := writeAppConfigFile(t, "defaults:\n render_format: "+rejected+"\n")
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt-dir", "./prompts", "--prompt", "p"})
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{"--config", path, "--prompt", "p"})
|
||||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "validate.error") || strings.Contains(stderr, rejected) {
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "validate.error") || strings.Contains(stderr, rejected) {
|
||||||
t.Fatalf("unsafe or misclassified load diagnostic: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("unsafe or misclassified load diagnostic: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
@@ -480,8 +530,9 @@ func TestApplicationDiagnosticWriterFailure(t *testing.T) {
|
|||||||
|
|
||||||
func TestParseRenderArgsWithExplicitConfigLoadsDirectoriesAndFormat(t *testing.T) {
|
func TestParseRenderArgsWithExplicitConfigLoadsDirectoriesAndFormat(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./from-config/prompts
|
promptkit_sources:
|
||||||
profile_dir: ./from-config/profiles
|
- directory: ./from-config/prompts
|
||||||
|
- directory: ./from-config/profiles
|
||||||
defaults:
|
defaults:
|
||||||
render_format: json
|
render_format: json
|
||||||
`)
|
`)
|
||||||
@@ -495,11 +546,8 @@ defaults:
|
|||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
if len(cfg.promptkitSources) != 2 {
|
||||||
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources)
|
||||||
}
|
|
||||||
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
||||||
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
||||||
}
|
}
|
||||||
if cfg.outputFormat != renderformat.PreparedRunFormatJSON {
|
if cfg.outputFormat != renderformat.PreparedRunFormatJSON {
|
||||||
t.Fatalf("expected render format from config, got %q", cfg.outputFormat)
|
t.Fatalf("expected render format from config, got %q", cfg.outputFormat)
|
||||||
@@ -508,8 +556,9 @@ defaults:
|
|||||||
|
|
||||||
func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) {
|
func TestParseRenderArgsExplicitFormatOverridesConfigDefaultFormat(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./from-config/prompts
|
promptkit_sources:
|
||||||
profile_dir: ./from-config/profiles
|
- directory: ./from-config/prompts
|
||||||
|
- directory: ./from-config/profiles
|
||||||
defaults:
|
defaults:
|
||||||
render_format: json
|
render_format: json
|
||||||
`)
|
`)
|
||||||
@@ -530,8 +579,9 @@ defaults:
|
|||||||
|
|
||||||
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
|
func TestParseServeArgsWithExplicitConfigLoadsSettingsAndCLIAddrOverrides(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./from-config/prompts
|
promptkit_sources:
|
||||||
profile_dir: ./from-config/profiles
|
- directory: ./from-config/prompts
|
||||||
|
- directory: ./from-config/profiles
|
||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9000
|
addr: 127.0.0.1:9000
|
||||||
@@ -553,11 +603,8 @@ server:
|
|||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
if len(cfg.promptkitSources) != 2 {
|
||||||
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources)
|
||||||
}
|
|
||||||
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
||||||
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
||||||
}
|
}
|
||||||
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
||||||
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
||||||
@@ -581,8 +628,9 @@ server:
|
|||||||
|
|
||||||
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
|
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./from-config/prompts
|
promptkit_sources:
|
||||||
profile_dir: ./from-config/profiles
|
- directory: ./from-config/prompts
|
||||||
|
- directory: ./from-config/profiles
|
||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9000
|
addr: 127.0.0.1:9000
|
||||||
@@ -599,11 +647,8 @@ server:
|
|||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.promptDir != filepath.Clean("./from-config/prompts") {
|
if len(cfg.promptkitSources) != 2 {
|
||||||
t.Fatalf("expected prompt dir from config, got %q", cfg.promptDir)
|
t.Fatalf("expected configured source selectors, got %#v", cfg.promptkitSources)
|
||||||
}
|
|
||||||
if cfg.profileDir != filepath.Clean("./from-config/profiles") {
|
|
||||||
t.Fatalf("expected profile dir from config, got %q", cfg.profileDir)
|
|
||||||
}
|
}
|
||||||
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
if cfg.schemaDir != filepath.Clean("./from-config/schemas") {
|
||||||
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
t.Fatalf("expected schema dir from config, got %q", cfg.schemaDir)
|
||||||
@@ -638,7 +683,6 @@ func TestParseServeArgsRejectsNegativeSizeLimits(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
_, err := parseServeArgs([]string{
|
_, err := parseServeArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
|
||||||
tc.flag, "-1",
|
tc.flag, "-1",
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -670,7 +714,6 @@ func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) {
|
|||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
err := tc.parse([]string{
|
err := tc.parse([]string{
|
||||||
"--prompt-dir", "./prompts",
|
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
"--max-request-bytes", "1024",
|
"--max-request-bytes", "1024",
|
||||||
@@ -683,9 +726,9 @@ func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
runCfg, err := parseRunArgs([]string{
|
runCfg, err := parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "prompt-1",
|
"--prompt", "prompt-1",
|
||||||
"--profile", "profile-1",
|
"--profile", "profile-1",
|
||||||
"--input", "transcript=./transcript.md",
|
"--input", "transcript=./transcript.md",
|
||||||
@@ -703,8 +746,7 @@ func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *tes
|
|||||||
}
|
}
|
||||||
|
|
||||||
renderCfg, err := parseRenderArgs([]string{
|
renderCfg, err := parseRenderArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--config", configPath,
|
||||||
"--profile-dir", "./profiles",
|
|
||||||
"--prompt", "prompt-1",
|
"--prompt", "prompt-1",
|
||||||
"--profile", "profile-1",
|
"--profile", "profile-1",
|
||||||
"--input", "transcript=./transcript.md",
|
"--input", "transcript=./transcript.md",
|
||||||
@@ -818,75 +860,13 @@ func TestBuildRunRequestAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
func TestRunAndRenderParsingDoesNotRequireCallerSources(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
profile_dir: ./profiles
|
if cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 {
|
||||||
`)
|
t.Fatalf("run should parse without caller sources: cfg=%+v err=%v", cfg, err)
|
||||||
|
|
||||||
_, err := parseRunArgs([]string{
|
|
||||||
"--config", configPath,
|
|
||||||
"--prompt", "p",
|
|
||||||
"--input", "a=b",
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected missing prompt_dir error")
|
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") {
|
if cfg, err := parseRenderArgs([]string{"--config", configPath, "--prompt", "p"}); err != nil || len(cfg.promptkitSources) != 0 {
|
||||||
t.Fatalf("expected clear prompt_dir guidance, got %v", err)
|
t.Fatalf("render should parse without caller sources: cfg=%+v err=%v", cfg, err)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
|
||||||
configPath := writeAppConfigFile(t, `
|
|
||||||
prompt_dir: ./prompts
|
|
||||||
`)
|
|
||||||
|
|
||||||
cfg, err := parseRunArgs([]string{
|
|
||||||
"--config", configPath,
|
|
||||||
"--prompt", "p",
|
|
||||||
"--input", "a=b",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
|
||||||
}
|
|
||||||
if cfg.profileDir != "" {
|
|
||||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRenderArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
|
||||||
configPath := writeAppConfigFile(t, `
|
|
||||||
profile_dir: ./profiles
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := parseRenderArgs([]string{
|
|
||||||
"--config", configPath,
|
|
||||||
"--prompt", "p",
|
|
||||||
"--input", "a=b",
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected missing prompt_dir error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "prompt directory is required") || !strings.Contains(err.Error(), "config.yml prompt_dir") {
|
|
||||||
t.Fatalf("expected clear prompt_dir guidance, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
|
||||||
configPath := writeAppConfigFile(t, `
|
|
||||||
prompt_dir: ./prompts
|
|
||||||
`)
|
|
||||||
|
|
||||||
cfg, err := parseRenderArgs([]string{
|
|
||||||
"--config", configPath,
|
|
||||||
"--prompt", "p",
|
|
||||||
"--input", "a=b",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
|
||||||
}
|
|
||||||
if cfg.profileDir != "" {
|
|
||||||
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -910,8 +890,8 @@ func TestRunCommandVarsOptional(t *testing.T) {
|
|||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := runCommand([]string{
|
code := runCommand([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--promptkit-source", "./prompts",
|
||||||
"--profile-dir", "./profiles",
|
"--promptkit-source", "./profiles",
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "transcript=./t.md",
|
"--input", "transcript=./t.md",
|
||||||
"--llm-base-url", "http://[::1",
|
"--llm-base-url", "http://[::1",
|
||||||
@@ -971,8 +951,8 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--profile", "local-default",
|
"--profile", "local-default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
@@ -1033,8 +1013,8 @@ temperature: 0.7
|
|||||||
}
|
}
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--temperature", "0",
|
"--temperature", "0",
|
||||||
@@ -1246,8 +1226,8 @@ backends:
|
|||||||
synthetic_count: 7
|
synthetic_count: 7
|
||||||
`, lib.promptDir, lib.profileDir, provider.URL+"/v1", credentialName))
|
`, lib.promptDir, lib.profileDir, provider.URL+"/v1", credentialName))
|
||||||
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"})
|
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt", "custom"})
|
||||||
if err != nil || len(cfg.backends) != 1 || cfg.backends[0].ExtraParams["synthetic_count"] != int64(7) {
|
if err != nil {
|
||||||
t.Fatalf("ConfigKit integer did not reach runtime settings: cfg=%+v err=%v", cfg, err)
|
t.Fatalf("ConfigKit application config did not parse: cfg=%+v err=%v", cfg, err)
|
||||||
}
|
}
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", configPath, "--prompt", "custom"})
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{"--config", configPath, "--prompt", "custom"})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
@@ -1293,7 +1273,7 @@ reasoning_effort: low
|
|||||||
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
|
{name: "direct session replaces definition", args: []string{"--session-id", "direct-session"}, wantReasoning: "low", wantSessionID: "direct-session"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "session"}
|
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "session"}
|
||||||
args = append(args, tc.args...)
|
args = append(args, tc.args...)
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
code, stdout, stderr := runCLICommand(t, renderCommand, args)
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
@@ -1331,8 +1311,8 @@ output:
|
|||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "plain",
|
"--prompt", "plain",
|
||||||
})
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
@@ -1358,8 +1338,8 @@ output:
|
|||||||
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local", "http://127.0.0.1:1/v1", "local-model")
|
||||||
|
|
||||||
code, _, stderr := runCLICommand(t, renderCommand, []string{
|
code, _, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "session",
|
"--prompt", "session",
|
||||||
"--session-id", strings.Repeat("x", 257),
|
"--session-id", strings.Repeat("x", 257),
|
||||||
})
|
})
|
||||||
@@ -1382,14 +1362,14 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "inspect", "--format", "json"})
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
|
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
|
||||||
t.Fatalf("unexpected inspection output: %s", stdout)
|
t.Fatalf("unexpected inspection output: %s", stdout)
|
||||||
}
|
}
|
||||||
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--prompt-dir", lib.promptDir, "--prompt", "missing"})
|
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--promptkit-source", lib.promptDir, "--prompt", "missing"})
|
||||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||||
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
@@ -1546,7 +1526,7 @@ func TestTransientRetrySettingsAndCLIOverride(t *testing.T) {
|
|||||||
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
|
func TestRunTimeoutParsingAndCancellation(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
args := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
|
args := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir, "--prompt", "timed", "--input", "transcript=" + inputPath}
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
flag []string
|
flag []string
|
||||||
@@ -1601,7 +1581,7 @@ func TestRunCommandWritesSeparateExecutionReport(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
writeProfileFile(t, lib.profileDir, "local-default", provider.URL+"/v1", "local-model")
|
||||||
path := filepath.Join(t.TempDir(), "execution.json")
|
path := filepath.Join(t.TempDir(), "execution.json")
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "reported", "--input", "transcript=" + inputPath,
|
"--prompt", "reported", "--input", "transcript=" + inputPath,
|
||||||
"--execution-report", path,
|
"--execution-report", path,
|
||||||
})
|
})
|
||||||
@@ -1712,7 +1692,7 @@ func TestExecutionReportLeavesExistingFileWhenNoTerminalEvent(t *testing.T) {
|
|||||||
|
|
||||||
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
|
func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
args := []string{"--prompt-dir", root, "--prompt", "p"}
|
args := []string{"--promptkit-source", root, "--prompt", "p"}
|
||||||
path := filepath.Join(root, "execution.json")
|
path := filepath.Join(root, "execution.json")
|
||||||
for _, tail := range [][]string{
|
for _, tail := range [][]string{
|
||||||
{"--execution-report", ""},
|
{"--execution-report", ""},
|
||||||
@@ -1837,7 +1817,7 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
|||||||
code, _, stderr := runCLICommand(t, inspectCommand, []string{
|
code, _, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
"profile",
|
"profile",
|
||||||
"--config", configWithoutPromptDir,
|
"--config", configWithoutPromptDir,
|
||||||
"--profile-dir", filepath.Join(fixtureRoot, "profiles"),
|
"--promptkit-source", filepath.Join(fixtureRoot, "profiles"),
|
||||||
"--profile", "custom-derived",
|
"--profile", "custom-derived",
|
||||||
})
|
})
|
||||||
if code != ExitOK {
|
if code != ExitOK {
|
||||||
@@ -1938,7 +1918,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
|
|||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
"profile",
|
"profile",
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--profile-dir", overrideDir,
|
"--promptkit-source", overrideDir,
|
||||||
"--profile", "custom-derived",
|
"--profile", "custom-derived",
|
||||||
"--format", "json",
|
"--format", "json",
|
||||||
"--out", outPath,
|
"--out", outPath,
|
||||||
@@ -1964,7 +1944,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
|
|||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
"profile",
|
"profile",
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--profile-dir", profileDir,
|
"--promptkit-source", profileDir,
|
||||||
"--profile", profileID,
|
"--profile", profileID,
|
||||||
})
|
})
|
||||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||||
@@ -2056,8 +2036,8 @@ func TestRenderCommandExplicitTextFormatWorks(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--format", "text",
|
"--format", "text",
|
||||||
@@ -2078,8 +2058,8 @@ func TestRenderCommandExplicitJSONFormatOutputsValidJSON(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--format", "json",
|
"--format", "json",
|
||||||
@@ -2107,8 +2087,8 @@ func TestRenderCommandUnknownFormatFailsClearly(t *testing.T) {
|
|||||||
var stdout bytes.Buffer
|
var stdout bytes.Buffer
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
code := renderCommand([]string{
|
code := renderCommand([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--promptkit-source", "./prompts",
|
||||||
"--profile-dir", "./profiles",
|
"--promptkit-source", "./profiles",
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "transcript=./x.md",
|
"--input", "transcript=./x.md",
|
||||||
"--format", "yaml",
|
"--format", "yaml",
|
||||||
@@ -2130,8 +2110,8 @@ func TestRenderCommandOutWritesToFile(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.render",
|
"--prompt", "prompt.render",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--out", outPath,
|
"--out", outPath,
|
||||||
@@ -2159,8 +2139,8 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
writeProfileFile(t, lib.profileDir, "local-default", "http://127.0.0.1:1/v1", "default-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
})
|
})
|
||||||
@@ -2185,7 +2165,7 @@ func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
|||||||
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--prompt", "prompt.builtin",
|
"--prompt", "prompt.builtin",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
})
|
})
|
||||||
@@ -2209,8 +2189,8 @@ func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
|
writeProfileFile(t, lib.profileDir, "quality", "http://127.0.0.1:1/v1", "quality-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--profile", "quality",
|
"--profile", "quality",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
@@ -2299,7 +2279,7 @@ output:
|
|||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
|
|
||||||
baseArgs := []string{"--prompt-dir", lib.promptDir, "--profile-dir", lib.profileDir}
|
baseArgs := []string{"--promptkit-source", lib.promptDir, "--promptkit-source", lib.profileDir}
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
args []string
|
args []string
|
||||||
@@ -2337,8 +2317,8 @@ func TestRunCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", ts.URL+"/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
})
|
})
|
||||||
@@ -2368,8 +2348,8 @@ func TestRunCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
|
writeProfileFile(t, lib.profileDir, "quality", overrideServer.URL+"/v1", "quality-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--profile", "quality",
|
"--profile", "quality",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
@@ -2408,8 +2388,8 @@ func TestRunCommandRuntimeFlagsOverrideSelectedProfileValues(t *testing.T) {
|
|||||||
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
|
writeProfileFile(t, lib.profileDir, "local-default", baseServer.URL+"/v1", "profile-model")
|
||||||
|
|
||||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||||
"--prompt-dir", lib.promptDir,
|
"--promptkit-source", lib.promptDir,
|
||||||
"--profile-dir", lib.profileDir,
|
"--promptkit-source", lib.profileDir,
|
||||||
"--prompt", "prompt.default",
|
"--prompt", "prompt.default",
|
||||||
"--input", "transcript=" + inputPath,
|
"--input", "transcript=" + inputPath,
|
||||||
"--llm-base-url", overrideServer.URL + "/v1",
|
"--llm-base-url", overrideServer.URL + "/v1",
|
||||||
@@ -2582,20 +2562,21 @@ func definitionFixtureRoot(t *testing.T) string {
|
|||||||
|
|
||||||
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string {
|
func writeDefinitionFixtureConfig(t *testing.T, fixtureRoot string, includeSources bool) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
sources := ""
|
frameworkPath := filepath.Join(fixtureRoot, "framework", "backends.yml")
|
||||||
if includeSources {
|
if _, err := os.Stat(frameworkPath); err != nil {
|
||||||
sources = fmt.Sprintf("prompt_dir: %q\nprofile_dir: %q\nschema_dir: %q\n", filepath.Join(fixtureRoot, "prompts"), filepath.Join(fixtureRoot, "profiles"), filepath.Join(fixtureRoot, "schemas"))
|
t.Fatalf("stat framework backend fixture: %v", err)
|
||||||
}
|
}
|
||||||
return writeAppConfigFile(t, sources+`backends:
|
|
||||||
fixture-custom:
|
sources := "{}\n"
|
||||||
endpoint: http://127.0.0.1:11434/v1
|
if includeSources {
|
||||||
api_key_env: FIXTURE_BACKEND_API_KEY
|
sources = fmt.Sprintf("promptkit_sources:\n - directory: %q\n - directory: %q\n - file: %q\nschema_dir: %q\n",
|
||||||
extra_params:
|
filepath.Join(fixtureRoot, "prompts"),
|
||||||
backend_option:
|
filepath.Join(fixtureRoot, "profiles"),
|
||||||
enabled: true
|
frameworkPath,
|
||||||
concurrency_limit: 2
|
filepath.Join(fixtureRoot, "schemas"),
|
||||||
queue_capacity: 0
|
)
|
||||||
`)
|
}
|
||||||
|
return writeAppConfigFile(t, sources)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
func writePromptFile(t *testing.T, dir, id, defaultProfile string) {
|
||||||
@@ -2612,16 +2593,17 @@ func writePromptDefinition(t *testing.T, dir, name, definition string) {
|
|||||||
|
|
||||||
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
func writePromptFileWithTemplate(t *testing.T, dir, id, defaultProfile, templateContent string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data := fmt.Sprintf(`id: %s
|
data := fmt.Sprintf(`prompts:
|
||||||
version: "1.0.0"
|
%q:
|
||||||
default_profile: %s
|
"1.0.0":
|
||||||
inputs:
|
default_profile: %q
|
||||||
|
inputs:
|
||||||
- name: transcript
|
- name: transcript
|
||||||
required: true
|
required: true
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
content: %q
|
content: %q
|
||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
repair_attempts: 0
|
repair_attempts: 0
|
||||||
@@ -2633,7 +2615,7 @@ output:
|
|||||||
|
|
||||||
func writeProfileFile(t *testing.T, dir, id, endpoint, model string) {
|
func writeProfileFile(t *testing.T, dir, id, endpoint, model string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
data := fmt.Sprintf("id: %s\nendpoint: %s\nmodel: %s\n", id, endpoint, model)
|
data := fmt.Sprintf("profiles:\n %q:\n endpoint: %q\n model: %q\n", id, endpoint, model)
|
||||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil {
|
||||||
t.Fatalf("failed to write profile fixture: %v", err)
|
t.Fatalf("failed to write profile fixture: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user