Explain framework configuration paths safely

This commit is contained in:
2026-10-04 12:46:12 +00:00
parent 2ba2ce1b91
commit 411a67b241
3 changed files with 248 additions and 9 deletions

View File

@@ -11,6 +11,7 @@ import (
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
)
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
@@ -143,11 +144,21 @@ func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t
format: text
validation_mode: none
`)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"configuration", "show", "--promptkit-source", frameworkFile,
})
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath})
if code != ExitOK || stdout != "" || stderr != "" {
t.Fatalf("application validation loaded framework definitions: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
for _, args := range [][]string{
{"configuration", "validate", "--config", configPath},
{"configuration", "show", "--config", configPath},
{"configuration", "sources", "--config", configPath},
{"configuration", "explain", "--config", configPath, "--path", ""},
} {
code, stdout, stderr = runCLICommand(t, inspectCommand, args)
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid final framework definition was reported as success: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
}
}
}
@@ -171,6 +182,195 @@ func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
}
stderr.Reset()
code = inspectCommand([]string{"configuration", "explain", "--config", configPath, "--path", ""}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
t.Fatalf("explanation writer failure was not reported: code=%d stderr=%q", code, stderr.String())
}
}
func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) {
root := t.TempDir()
lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-displaced-canary
password: backend-password-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
base:
backend: local
model: inherited-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-displaced-canary
password: profile-password-displaced-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
"team/summary":
extends: base
model: lower-model
prompts:
explainable:
"1":
default_profile: "team/summary"
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
upperRoot := t.TempDir()
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-effective-canary
password: backend-password-effective-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
"team/summary":
model: effective-model
extra_params:
token: profile-token-effective-canary
password: profile-password-effective-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper))
const secret = "environment-value-must-never-be-emitted"
t.Setenv("SCRIPTORIUM_EXPLAIN_REFERENCE", secret)
for _, tc := range []struct {
path string
want []string
unwant []string
}{
{
path: "/backends/local/extra_params/token",
unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"},
},
{
path: "/backends/local/extra_params/password",
unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"},
},
{
path: "/profiles/team~1summary/model",
want: []string{"effective-model", "inherited-model", "lower-model"},
unwant: []string{},
},
{
path: "/profiles/team~1summary/extra_params/token",
unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"},
},
{
path: "/profiles/team~1summary/extra_params/password",
unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"},
},
{
path: "/profiles/team~1summary/api_key_env",
want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"},
unwant: []string{secret},
},
{
path: "/profiles/team~1summary/extra_params/provider.option",
want: []string{"dotted-visible-value"},
unwant: []string{"provider/option"},
},
{
path: "",
want: []string{"profiles:", "effective-model"},
unwant: []string{"profile-token-displaced-canary", "backend-password-effective-canary"},
},
} {
args := []string{"configuration", "explain", "--config", configPath, "--path", tc.path}
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitOK || stderr != "" {
t.Fatalf("explain %q failed: code=%d stdout=%q stderr=%q", tc.path, code, stdout, stderr)
}
if stdout == "" {
t.Fatalf("explain %q produced no output", tc.path)
}
for _, value := range tc.want {
if !strings.Contains(stdout, value) {
t.Errorf("explain %q omitted %q: %s", tc.path, value, stdout)
}
}
for _, value := range tc.unwant {
if strings.Contains(stdout, value) {
t.Errorf("explain %q disclosed %q: %s", tc.path, value, stdout)
}
}
if strings.Contains(stdout, secret) {
t.Errorf("explain %q disclosed the resolved environment value: %s", tc.path, stdout)
}
}
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"configuration", "show", "--config", configPath})
if code != ExitOK || stderr != "" {
t.Fatalf("framework show failed: code=%d stderr=%q", code, stderr)
}
for _, value := range []string{"SCRIPTORIUM_EXPLAIN_REFERENCE", "backend-visible-value", "profile-visible-value", "dotted-visible-value"} {
if !strings.Contains(stdout, value) {
t.Errorf("effective report omitted visible configuration %q: %s", value, stdout)
}
}
for _, value := range []string{
"backend-token-displaced-canary", "backend-token-effective-canary",
"backend-password-displaced-canary", "backend-password-effective-canary",
"profile-token-displaced-canary", "profile-token-effective-canary",
"profile-password-displaced-canary", "profile-password-effective-canary", secret,
} {
if strings.Contains(stdout, value) {
t.Errorf("effective report disclosed classified or resolved value %q: %s", value, stdout)
}
}
profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{
"profile", "--config", configPath, "--profile", "team/summary", "--format", "json",
})
if profileCode != ExitOK || profileError != "" {
t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError)
}
var profile renderformat.ProfileInspection
if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil {
t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput)
}
if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" {
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams)
}
if strings.Contains(profileOutput, secret) {
t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput)
}
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"})
if code != ExitOK || stderr != "" {
t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr)
}
if !strings.Contains(rendered, "profile-visible-value") || strings.Contains(rendered, secret) {
t.Fatalf("prepared output did not preserve ordinary provider data or disclosed environment data: %s", rendered)
}
}
func TestFrameworkConfigurationExplainRejectsMissingMalformedAndUnknownPaths(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
for _, args := range [][]string{
{"configuration", "explain"},
{"configuration", "explain", "--path", "not-a-pointer"},
{"configuration", "explain", "--path", "/profiles/team~2summary/model"},
{"configuration", "explain", "--config", configPath, "--path", "/profiles/missing/model"},
} {
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid explanation path was accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
}
}
}
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {