Explain framework configuration paths safely
This commit is contained in:
@@ -3,8 +3,8 @@
|
|||||||
## Status, Baseline, And Authority
|
## Status, Baseline, And Authority
|
||||||
|
|
||||||
Planned on 2026-10-04 against Scriptorium
|
Planned on 2026-10-04 against Scriptorium
|
||||||
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–9 are complete; stages
|
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–10 are complete; stages
|
||||||
10–13 are pending.
|
11–13 are pending.
|
||||||
This plan is for a Luna coding agent: implement the stages in sequential order,
|
This plan is for a Luna coding agent: implement the stages in sequential order,
|
||||||
normally one stage per implementation prompt. The
|
normally one stage per implementation prompt. The
|
||||||
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
||||||
@@ -565,7 +565,19 @@ scope with application inspection or altering existing presentation contracts.
|
|||||||
|
|
||||||
## Stage 10: Add Literal-Path Explanations And Prove Scoped Redaction
|
## Stage 10: Add Literal-Path Explanations And Prove Scoped Redaction
|
||||||
|
|
||||||
Status: Pending. Depends on Stage 9.
|
Status: Complete. `inspect configuration explain` now requires an explicitly
|
||||||
|
provided JSON Pointer, accepts the empty root, parses with ConfigKit, detects
|
||||||
|
unknown paths before writing, and delegates explanations to the captured
|
||||||
|
PromptKit inspection view. Tests cover inherited and overlaid history, escaped
|
||||||
|
slash IDs, dotted keys, root output, malformed/unknown paths, native writer
|
||||||
|
errors, effective and historical `token`/`password` redaction, visible
|
||||||
|
environment-reference names and ordinary values, and the unchanged disclosure
|
||||||
|
behavior of profile inspection and prepared output. Application validation
|
||||||
|
remains independent of invalid framework definitions, while framework
|
||||||
|
validation rejects them without a partial report. Focused inspection tests,
|
||||||
|
`go test ./...`, `go test -race ./...`, `go vet ./...`, the CLI build,
|
||||||
|
`gofmt`, `git diff --check`, and all four operations against
|
||||||
|
`examples/config.yml` pass.
|
||||||
|
|
||||||
Files: framework inspection parser/handler/tests from Stage 9.
|
Files: framework inspection parser/handler/tests from Stage 9.
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,9 @@ import (
|
|||||||
|
|
||||||
type frameworkConfigurationConfig struct {
|
type frameworkConfigurationConfig struct {
|
||||||
mode, configPath, configOverlay, schemaDir string
|
mode, configPath, configOverlay, schemaDir string
|
||||||
|
path string
|
||||||
configExplicit bool
|
configExplicit bool
|
||||||
|
pathExplicit bool
|
||||||
overrides []configkit.Override
|
overrides []configkit.Override
|
||||||
diagnostics []configkit.Diagnostic
|
diagnostics []configkit.Diagnostic
|
||||||
}
|
}
|
||||||
@@ -39,6 +41,14 @@ func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Write
|
|||||||
writeApplicationError(stderr, "inspect configuration parse", err)
|
writeApplicationError(stderr, "inspect configuration parse", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
var explanationPath configkit.Path
|
||||||
|
if cfg.mode == "explain" {
|
||||||
|
explanationPath, err = configkit.ParsePath(cfg.path)
|
||||||
|
if err != nil {
|
||||||
|
writeApplicationError(stderr, "inspect configuration path", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
}
|
||||||
settings, err := resolveAppSettingsForFrameworkConfiguration(cfg)
|
settings, err := resolveAppSettingsForFrameworkConfiguration(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeApplicationError(stderr, "inspect configuration", err)
|
writeApplicationError(stderr, "inspect configuration", err)
|
||||||
@@ -64,6 +74,18 @@ func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Write
|
|||||||
return ExitOK
|
return ExitOK
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if cfg.mode == "explain" {
|
||||||
|
if _, ok := view.Explain(explanationPath); !ok {
|
||||||
|
writeApplicationError(stderr, "inspect configuration path", fmt.Errorf("unknown path %q", cfg.path))
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if err := view.WriteExplanation(stdout, explanationPath); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect configuration output error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
return ExitOK
|
||||||
|
}
|
||||||
|
|
||||||
switch cfg.mode {
|
switch cfg.mode {
|
||||||
case "show":
|
case "show":
|
||||||
err = view.WriteEffective(stdout)
|
err = view.WriteEffective(stdout)
|
||||||
@@ -84,7 +106,7 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf
|
|||||||
}
|
}
|
||||||
cfg := &frameworkConfigurationConfig{mode: args[0]}
|
cfg := &frameworkConfigurationConfig{mode: args[0]}
|
||||||
switch cfg.mode {
|
switch cfg.mode {
|
||||||
case "validate", "show", "sources":
|
case "validate", "show", "sources", "explain":
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode)
|
return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode)
|
||||||
}
|
}
|
||||||
@@ -95,6 +117,7 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf
|
|||||||
var promptkitSources promptKitSourceFlagValue
|
var promptkitSources promptKitSourceFlagValue
|
||||||
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
||||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||||
|
fs.StringVar(&cfg.path, "path", "", "JSON Pointer to explain (required for explain)")
|
||||||
if err := fs.Parse(args[1:]); err != nil {
|
if err := fs.Parse(args[1:]); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -102,6 +125,10 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf
|
|||||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||||
}
|
}
|
||||||
cfg.configExplicit = flagWasSet(fs, "config")
|
cfg.configExplicit = flagWasSet(fs, "config")
|
||||||
|
cfg.pathExplicit = flagWasSet(fs, "path")
|
||||||
|
if cfg.mode == "explain" && !cfg.pathExplicit {
|
||||||
|
return nil, errors.New("--path is required for explain")
|
||||||
|
}
|
||||||
cfg.overrides = applicationOverrides(fs)
|
cfg.overrides = applicationOverrides(fs)
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
|
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
|
||||||
@@ -143,11 +144,21 @@ func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
`)
|
`)
|
||||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||||
"configuration", "show", "--promptkit-source", frameworkFile,
|
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath})
|
||||||
})
|
if code != ExitOK || stdout != "" || stderr != "" {
|
||||||
|
t.Fatalf("application validation loaded framework definitions: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"configuration", "validate", "--config", configPath},
|
||||||
|
{"configuration", "show", "--config", configPath},
|
||||||
|
{"configuration", "sources", "--config", configPath},
|
||||||
|
{"configuration", "explain", "--config", configPath, "--path", ""},
|
||||||
|
} {
|
||||||
|
code, stdout, stderr = runCLICommand(t, inspectCommand, args)
|
||||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||||
t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("invalid final framework definition was reported as success: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,6 +182,195 @@ func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors
|
|||||||
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
||||||
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
||||||
}
|
}
|
||||||
|
stderr.Reset()
|
||||||
|
code = inspectCommand([]string{"configuration", "explain", "--config", configPath, "--path", ""}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
|
||||||
|
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
||||||
|
t.Fatalf("explanation writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends:
|
||||||
|
local:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||||
|
extra_params:
|
||||||
|
token: backend-token-displaced-canary
|
||||||
|
password: backend-password-canary
|
||||||
|
visible_option: backend-visible-value
|
||||||
|
provider.option: dotted-visible-value
|
||||||
|
profiles:
|
||||||
|
base:
|
||||||
|
backend: local
|
||||||
|
model: inherited-model
|
||||||
|
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||||
|
extra_params:
|
||||||
|
token: profile-token-displaced-canary
|
||||||
|
password: profile-password-displaced-canary
|
||||||
|
visible_option: profile-visible-value
|
||||||
|
provider.option: dotted-visible-value
|
||||||
|
"team/summary":
|
||||||
|
extends: base
|
||||||
|
model: lower-model
|
||||||
|
prompts:
|
||||||
|
explainable:
|
||||||
|
"1":
|
||||||
|
default_profile: "team/summary"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: hello
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
upperRoot := t.TempDir()
|
||||||
|
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends:
|
||||||
|
local:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||||
|
extra_params:
|
||||||
|
token: backend-token-effective-canary
|
||||||
|
password: backend-password-effective-canary
|
||||||
|
visible_option: backend-visible-value
|
||||||
|
provider.option: dotted-visible-value
|
||||||
|
profiles:
|
||||||
|
"team/summary":
|
||||||
|
model: effective-model
|
||||||
|
extra_params:
|
||||||
|
token: profile-token-effective-canary
|
||||||
|
password: profile-password-effective-canary
|
||||||
|
visible_option: profile-visible-value
|
||||||
|
provider.option: dotted-visible-value
|
||||||
|
`)
|
||||||
|
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper))
|
||||||
|
const secret = "environment-value-must-never-be-emitted"
|
||||||
|
t.Setenv("SCRIPTORIUM_EXPLAIN_REFERENCE", secret)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
path string
|
||||||
|
want []string
|
||||||
|
unwant []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
path: "/backends/local/extra_params/token",
|
||||||
|
unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "/backends/local/extra_params/password",
|
||||||
|
unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "/profiles/team~1summary/model",
|
||||||
|
want: []string{"effective-model", "inherited-model", "lower-model"},
|
||||||
|
unwant: []string{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "/profiles/team~1summary/extra_params/token",
|
||||||
|
unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "/profiles/team~1summary/extra_params/password",
|
||||||
|
unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "/profiles/team~1summary/api_key_env",
|
||||||
|
want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"},
|
||||||
|
unwant: []string{secret},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "/profiles/team~1summary/extra_params/provider.option",
|
||||||
|
want: []string{"dotted-visible-value"},
|
||||||
|
unwant: []string{"provider/option"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "",
|
||||||
|
want: []string{"profiles:", "effective-model"},
|
||||||
|
unwant: []string{"profile-token-displaced-canary", "backend-password-effective-canary"},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
args := []string{"configuration", "explain", "--config", configPath, "--path", tc.path}
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||||
|
if code != ExitOK || stderr != "" {
|
||||||
|
t.Fatalf("explain %q failed: code=%d stdout=%q stderr=%q", tc.path, code, stdout, stderr)
|
||||||
|
}
|
||||||
|
if stdout == "" {
|
||||||
|
t.Fatalf("explain %q produced no output", tc.path)
|
||||||
|
}
|
||||||
|
for _, value := range tc.want {
|
||||||
|
if !strings.Contains(stdout, value) {
|
||||||
|
t.Errorf("explain %q omitted %q: %s", tc.path, value, stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, value := range tc.unwant {
|
||||||
|
if strings.Contains(stdout, value) {
|
||||||
|
t.Errorf("explain %q disclosed %q: %s", tc.path, value, stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(stdout, secret) {
|
||||||
|
t.Errorf("explain %q disclosed the resolved environment value: %s", tc.path, stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"configuration", "show", "--config", configPath})
|
||||||
|
if code != ExitOK || stderr != "" {
|
||||||
|
t.Fatalf("framework show failed: code=%d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
for _, value := range []string{"SCRIPTORIUM_EXPLAIN_REFERENCE", "backend-visible-value", "profile-visible-value", "dotted-visible-value"} {
|
||||||
|
if !strings.Contains(stdout, value) {
|
||||||
|
t.Errorf("effective report omitted visible configuration %q: %s", value, stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, value := range []string{
|
||||||
|
"backend-token-displaced-canary", "backend-token-effective-canary",
|
||||||
|
"backend-password-displaced-canary", "backend-password-effective-canary",
|
||||||
|
"profile-token-displaced-canary", "profile-token-effective-canary",
|
||||||
|
"profile-password-displaced-canary", "profile-password-effective-canary", secret,
|
||||||
|
} {
|
||||||
|
if strings.Contains(stdout, value) {
|
||||||
|
t.Errorf("effective report disclosed classified or resolved value %q: %s", value, stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"profile", "--config", configPath, "--profile", "team/summary", "--format", "json",
|
||||||
|
})
|
||||||
|
if profileCode != ExitOK || profileError != "" {
|
||||||
|
t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError)
|
||||||
|
}
|
||||||
|
var profile renderformat.ProfileInspection
|
||||||
|
if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil {
|
||||||
|
t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput)
|
||||||
|
}
|
||||||
|
if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" {
|
||||||
|
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
if strings.Contains(profileOutput, secret) {
|
||||||
|
t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput)
|
||||||
|
}
|
||||||
|
|
||||||
|
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"})
|
||||||
|
if code != ExitOK || stderr != "" {
|
||||||
|
t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rendered, "profile-visible-value") || strings.Contains(rendered, secret) {
|
||||||
|
t.Fatalf("prepared output did not preserve ordinary provider data or disclosed environment data: %s", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameworkConfigurationExplainRejectsMissingMalformedAndUnknownPaths(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"configuration", "explain"},
|
||||||
|
{"configuration", "explain", "--path", "not-a-pointer"},
|
||||||
|
{"configuration", "explain", "--path", "/profiles/team~2summary/model"},
|
||||||
|
{"configuration", "explain", "--config", configPath, "--path", "/profiles/missing/model"},
|
||||||
|
} {
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||||
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||||
|
t.Fatalf("invalid explanation path was accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {
|
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {
|
||||||
|
|||||||
Reference in New Issue
Block a user