Migrate CLI tests to native PromptKit sources
This commit is contained in:
@@ -9,8 +9,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
)
|
||||
|
||||
func TestConfigCommandsInspectApplicationWithoutEngine(t *testing.T) {
|
||||
@@ -45,17 +43,17 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
|
||||
}
|
||||
base := filepath.Join(dir, "config.yml")
|
||||
for path, body := range map[string]string{
|
||||
base: "configkit:\n imports:\n - path: fragment.yml\nprompt_dir: ' ./before/../prompts '\nprofile_dir: ./profiles\n",
|
||||
base: "configkit:\n imports:\n - path: fragment.yml\npromptkit_sources:\n - directory: ' ./before/../prompts '\nschema_dir: ./schemas\n",
|
||||
filepath.Join(dir, "fragment.yml"): "server:\n addr: :8081\n",
|
||||
filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n",
|
||||
filepath.Join(dir, "overlays", "local.yml"): "promptkit_sources:\n - directory: ./overlay-prompts\nserver:\n addr: :9090\n",
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
args := []string{"--config", base, "--config-overlay", "local", "--profile-dir=", "--addr", ":7070"}
|
||||
args := []string{"--config", base, "--config-overlay", "local", "--schema-dir=", "--addr", ":7070"}
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"show"}, args...))
|
||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "prompt_dir: prompts") || !strings.Contains(stdout, "profile_dir: ''") || !strings.Contains(stdout, "addr: :7070") {
|
||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "directory: overlay-prompts") || !strings.Contains(stdout, "schema_dir: ''") || !strings.Contains(stdout, "addr: :7070") || strings.Contains(stdout, "before/../prompts") {
|
||||
t.Fatalf("resolved effective output changed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, append([]string{"sources"}, args...))
|
||||
@@ -73,121 +71,45 @@ func TestConfigCommandsExposeComposedSourcesAndResolvedValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigCommandsInspectInheritedBackends(t *testing.T) {
|
||||
const keyName = "SCRIPTORIUM_INHERITANCE_TEST_API_KEY"
|
||||
const keyValue = "synthetic-inheritance-key-bytes"
|
||||
path := writeAppConfigFile(t, `backends:
|
||||
base:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY
|
||||
concurrency_limit: 2
|
||||
queue_capacity: 4
|
||||
extra_params:
|
||||
enabled: true
|
||||
shared: visible-value
|
||||
child:
|
||||
extends: base
|
||||
queue_capacity: 0
|
||||
extra_params:
|
||||
enabled: false
|
||||
`)
|
||||
t.Setenv(keyName, keyValue)
|
||||
if err := os.Unsetenv(keyName); err != nil {
|
||||
func TestApplicationInspectionDoesNotAcquireFrameworkSources(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
frameworkPath := filepath.Join(dir, "invalid-framework.yml")
|
||||
if err := os.WriteFile(frameworkPath, []byte("not: [valid\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||
if code != ExitOK || stdout != "" || stderr != "" {
|
||||
t.Fatalf("config validate required credentials or an engine: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
if err := os.Setenv(keyName, keyValue); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"show", "--config", path})
|
||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("config show failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
for _, fragment := range []string{
|
||||
"endpoint: http://127.0.0.1:1/v1",
|
||||
"api_key_env: SCRIPTORIUM_INHERITANCE_TEST_API_KEY",
|
||||
"shared: visible-value",
|
||||
"enabled: false",
|
||||
"queue_capacity: 0",
|
||||
for _, frameworkSource := range []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{name: "invalid definition", path: frameworkPath},
|
||||
{name: "missing path", path: filepath.Join(dir, "missing-framework.yml")},
|
||||
} {
|
||||
if !strings.Contains(stdout, fragment) {
|
||||
t.Fatalf("config show omitted %q: %s", fragment, stdout)
|
||||
}
|
||||
}
|
||||
if strings.Contains(stdout, "extends:") {
|
||||
t.Fatalf("config show exposed the structural directive: %s", stdout)
|
||||
}
|
||||
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/endpoint", "--config", path})
|
||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("endpoint explanation failed or disclosed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
for _, fragment := range []string{"/backends/base/endpoint", "/backends/child/endpoint", "/backends/child/extends", "http://127.0.0.1:1/v1"} {
|
||||
if !strings.Contains(stdout, fragment) {
|
||||
t.Fatalf("endpoint explanation omitted %q: %s", fragment, stdout)
|
||||
}
|
||||
}
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"explain", "--path", "/backends/child/extra_params/enabled", "--config", path})
|
||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "true") || !strings.Contains(stdout, "false") || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("parameter explanation lost history or exposed credential bytes: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
|
||||
code, stdout, stderr = runCLICommand(t, configCommand, []string{"sources", "--config", path})
|
||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("config sources failed: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
var sources []configurationSource
|
||||
if err := json.Unmarshal([]byte(stdout), &sources); err != nil || len(sources) != 1 || sources[0].Layer != "base" {
|
||||
t.Fatalf("inheritance fabricated a source record: %+v (%v)", sources, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidateRendersReservedInheritanceFailure(t *testing.T) {
|
||||
path := writeAppConfigFile(t, `backends:
|
||||
worker:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
extra_params:
|
||||
extends: missing
|
||||
`)
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inheritance.missing") || !strings.Contains(stderr, "/backends/worker/extra_params/extends") {
|
||||
t.Fatalf("reserved inheritance failure was not safely reported: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigInspectionKeepsReferencesAndOrdinaryParameters(t *testing.T) {
|
||||
const name = "SCRIPTORIUM_CONFIG_TEST_API_KEY"
|
||||
const secret = "synthetic-secret-value-should-not-appear"
|
||||
path := writeAppConfigFile(t, `backends:
|
||||
synthetic:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_CONFIG_TEST_API_KEY
|
||||
extra_params:
|
||||
visible_parameter: synthetic-ordinary-value
|
||||
`)
|
||||
t.Setenv(name, secret)
|
||||
for _, present := range []bool{true, false} {
|
||||
if !present {
|
||||
if err := os.Unsetenv(name); err != nil {
|
||||
t.Fatal(err)
|
||||
t.Run(frameworkSource.name, func(t *testing.T) {
|
||||
path := writeAppConfigFile(t, "promptkit_sources:\n - file: "+frameworkSource.path+"\n")
|
||||
for _, args := range [][]string{
|
||||
{"validate"},
|
||||
{"show"},
|
||||
{"sources"},
|
||||
{"explain", "--path", "/promptkit_sources/0/file"},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, append(args, "--config", path))
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("application inspection acquired configured framework source: args=%q code=%d out=%q err=%q", args, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"show", "--config", path})
|
||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, name) || !strings.Contains(stdout, "synthetic-ordinary-value") || strings.Contains(stdout, secret) {
|
||||
t.Fatalf("credential boundary changed with present=%t: code=%d out=%q err=%q", present, code, stdout, stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
result, err := appconfig.Load(appconfig.LoadOptions{Path: path, Explicit: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := appconfig.Settings(result.Config)
|
||||
if len(settings.Backends) != 1 || settings.Backends[0].ExtraParams["visible_parameter"] != "synthetic-ordinary-value" {
|
||||
t.Fatalf("runtime backend parameters changed: %+v", settings.Backends)
|
||||
}
|
||||
|
||||
func TestConfigCommandsRejectRetiredFrameworkFields(t *testing.T) {
|
||||
for _, field := range []string{"prompt_dir", "profile_dir", "backends"} {
|
||||
t.Run(field, func(t *testing.T) {
|
||||
path := writeAppConfigFile(t, field+": {}\n")
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", path})
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("retired application field %q was accepted: code=%d out=%q err=%q", field, code, stdout, stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -213,51 +135,36 @@ func TestConfigCommandFailuresDoNotProduceSuccessfulOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigExplainUsesResolvedHistoryAndKeepsParametersVisible(t *testing.T) {
|
||||
func TestConfigExplainUsesResolvedHistory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := filepath.Join(dir, "config.yml")
|
||||
for path, body := range map[string]string{
|
||||
base: `prompt_dir: " ./prompts/../prompts "
|
||||
base: `promptkit_sources:
|
||||
- directory: " ./prompts/../prompts "
|
||||
server:
|
||||
addr: :8080
|
||||
backends:
|
||||
"team/blue~1":
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_KEY
|
||||
extra_params:
|
||||
"0": base-parameter
|
||||
`,
|
||||
filepath.Join(dir, "overlays", "local.yml"): `server:
|
||||
addr: :9090
|
||||
backends:
|
||||
"team/blue~1":
|
||||
extra_params:
|
||||
"0": overlay-parameter
|
||||
`,
|
||||
filepath.Join(dir, "overlays", "local.yml"): "server:\n addr: :9090\n",
|
||||
} {
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const keyValue = "synthetic-key-bytes-must-not-appear"
|
||||
t.Setenv("SCRIPTORIUM_EXPLAIN_KEY", keyValue)
|
||||
args := []string{"--config", base, "--config-overlay", "local", "--addr", ":9090"}
|
||||
for _, tc := range []struct {
|
||||
pointer string
|
||||
want []string
|
||||
}{
|
||||
{pointer: "/server/addr", want: []string{":8080", ":9090", "--addr"}},
|
||||
{pointer: "/prompt_dir", want: []string{"normalized application path", "prompts"}},
|
||||
{pointer: "/backends/team~1blue~01/extra_params/0", want: []string{"base-parameter", "overlay-parameter"}},
|
||||
{pointer: "/backends/team~1blue~01/api_key_env", want: []string{"SCRIPTORIUM_EXPLAIN_KEY"}},
|
||||
{pointer: "", want: []string{"base-parameter", "overlay-parameter", "SCRIPTORIUM_EXPLAIN_KEY"}},
|
||||
{pointer: "/promptkit_sources/0/directory", want: []string{"normalized application path", "prompts"}},
|
||||
{pointer: "", want: []string{"promptkit_sources", "server", "addr"}},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", tc.pointer}, args...))
|
||||
if code != ExitOK || stderr != "" || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("explain %q failed or disclosed key: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr)
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("explain %q failed: code=%d out=%q err=%q", tc.pointer, code, stdout, stderr)
|
||||
}
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(stdout, want) {
|
||||
@@ -265,13 +172,6 @@ backends:
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := os.Unsetenv("SCRIPTORIUM_EXPLAIN_KEY"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, append([]string{"explain", "--path", "/backends/team~1blue~01/api_key_env"}, args...))
|
||||
if code != ExitOK || stderr != "" || !strings.Contains(stdout, "SCRIPTORIUM_EXPLAIN_KEY") || strings.Contains(stdout, keyValue) {
|
||||
t.Fatalf("absent credential changed reference explanation: code=%d out=%q err=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigExplainRejectsUnknownOrMalformedPaths(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user