Explain framework configuration paths safely

This commit is contained in:
2026-10-04 12:46:12 +00:00
parent 2ba2ce1b91
commit 411a67b241
3 changed files with 248 additions and 9 deletions

View File

@@ -3,8 +3,8 @@
## Status, Baseline, And Authority ## Status, Baseline, And Authority
Planned on 2026-10-04 against Scriptorium Planned on 2026-10-04 against Scriptorium
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–9 are complete; stages `82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–10 are complete; stages
10–13 are pending. 11–13 are pending.
This plan is for a Luna coding agent: implement the stages in sequential order, This plan is for a Luna coding agent: implement the stages in sequential order,
normally one stage per implementation prompt. The normally one stage per implementation prompt. The
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed [feature roadmap](promptkit.md) owns the target state, scope, and confirmed
@@ -565,7 +565,19 @@ scope with application inspection or altering existing presentation contracts.
## Stage 10: Add Literal-Path Explanations And Prove Scoped Redaction ## Stage 10: Add Literal-Path Explanations And Prove Scoped Redaction
Status: Pending. Depends on Stage 9. Status: Complete. `inspect configuration explain` now requires an explicitly
provided JSON Pointer, accepts the empty root, parses with ConfigKit, detects
unknown paths before writing, and delegates explanations to the captured
PromptKit inspection view. Tests cover inherited and overlaid history, escaped
slash IDs, dotted keys, root output, malformed/unknown paths, native writer
errors, effective and historical `token`/`password` redaction, visible
environment-reference names and ordinary values, and the unchanged disclosure
behavior of profile inspection and prepared output. Application validation
remains independent of invalid framework definitions, while framework
validation rejects them without a partial report. Focused inspection tests,
`go test ./...`, `go test -race ./...`, `go vet ./...`, the CLI build,
`gofmt`, `git diff --check`, and all four operations against
`examples/config.yml` pass.
Files: framework inspection parser/handler/tests from Stage 9. Files: framework inspection parser/handler/tests from Stage 9.

View File

@@ -12,7 +12,9 @@ import (
type frameworkConfigurationConfig struct { type frameworkConfigurationConfig struct {
mode, configPath, configOverlay, schemaDir string mode, configPath, configOverlay, schemaDir string
path string
configExplicit bool configExplicit bool
pathExplicit bool
overrides []configkit.Override overrides []configkit.Override
diagnostics []configkit.Diagnostic diagnostics []configkit.Diagnostic
} }
@@ -39,6 +41,14 @@ func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Write
writeApplicationError(stderr, "inspect configuration parse", err) writeApplicationError(stderr, "inspect configuration parse", err)
return ExitRuntimeError return ExitRuntimeError
} }
var explanationPath configkit.Path
if cfg.mode == "explain" {
explanationPath, err = configkit.ParsePath(cfg.path)
if err != nil {
writeApplicationError(stderr, "inspect configuration path", err)
return ExitRuntimeError
}
}
settings, err := resolveAppSettingsForFrameworkConfiguration(cfg) settings, err := resolveAppSettingsForFrameworkConfiguration(cfg)
if err != nil { if err != nil {
writeApplicationError(stderr, "inspect configuration", err) writeApplicationError(stderr, "inspect configuration", err)
@@ -64,6 +74,18 @@ func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Write
return ExitOK return ExitOK
} }
if cfg.mode == "explain" {
if _, ok := view.Explain(explanationPath); !ok {
writeApplicationError(stderr, "inspect configuration path", fmt.Errorf("unknown path %q", cfg.path))
return ExitRuntimeError
}
if err := view.WriteExplanation(stdout, explanationPath); err != nil {
fmt.Fprintf(stderr, "inspect configuration output error: %v\n", err)
return ExitRuntimeError
}
return ExitOK
}
switch cfg.mode { switch cfg.mode {
case "show": case "show":
err = view.WriteEffective(stdout) err = view.WriteEffective(stdout)
@@ -84,7 +106,7 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf
} }
cfg := &frameworkConfigurationConfig{mode: args[0]} cfg := &frameworkConfigurationConfig{mode: args[0]}
switch cfg.mode { switch cfg.mode {
case "validate", "show", "sources": case "validate", "show", "sources", "explain":
default: default:
return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode) return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode)
} }
@@ -95,6 +117,7 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf
var promptkitSources promptKitSourceFlagValue var promptkitSources promptKitSourceFlagValue
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)") fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas") fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
fs.StringVar(&cfg.path, "path", "", "JSON Pointer to explain (required for explain)")
if err := fs.Parse(args[1:]); err != nil { if err := fs.Parse(args[1:]); err != nil {
return nil, err return nil, err
} }
@@ -102,6 +125,10 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
} }
cfg.configExplicit = flagWasSet(fs, "config") cfg.configExplicit = flagWasSet(fs, "config")
cfg.pathExplicit = flagWasSet(fs, "path")
if cfg.mode == "explain" && !cfg.pathExplicit {
return nil, errors.New("--path is required for explain")
}
cfg.overrides = applicationOverrides(fs) cfg.overrides = applicationOverrides(fs)
return cfg, nil return cfg, nil
} }

View File

@@ -11,6 +11,7 @@ import (
"testing" "testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit" configkit "gitea.maximumdirect.net/eric/ConfigKit"
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
) )
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) { func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
@@ -143,11 +144,21 @@ func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t
format: text format: text
validation_mode: none validation_mode: none
`) `)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
"configuration", "show", "--promptkit-source", frameworkFile, code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath})
}) if code != ExitOK || stdout != "" || stderr != "" {
t.Fatalf("application validation loaded framework definitions: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
for _, args := range [][]string{
{"configuration", "validate", "--config", configPath},
{"configuration", "show", "--config", configPath},
{"configuration", "sources", "--config", configPath},
{"configuration", "explain", "--config", configPath, "--path", ""},
} {
code, stdout, stderr = runCLICommand(t, inspectCommand, args)
if code != ExitRuntimeError || stdout != "" || stderr == "" { if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr) t.Fatalf("invalid final framework definition was reported as success: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
}
} }
} }
@@ -171,6 +182,195 @@ func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) { if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String()) t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
} }
stderr.Reset()
code = inspectCommand([]string{"configuration", "explain", "--config", configPath, "--path", ""}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
t.Fatalf("explanation writer failure was not reported: code=%d stderr=%q", code, stderr.String())
}
}
func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) {
root := t.TempDir()
lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-displaced-canary
password: backend-password-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
base:
backend: local
model: inherited-model
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: profile-token-displaced-canary
password: profile-password-displaced-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
"team/summary":
extends: base
model: lower-model
prompts:
explainable:
"1":
default_profile: "team/summary"
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
upperRoot := t.TempDir()
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
extra_params:
token: backend-token-effective-canary
password: backend-password-effective-canary
visible_option: backend-visible-value
provider.option: dotted-visible-value
profiles:
"team/summary":
model: effective-model
extra_params:
token: profile-token-effective-canary
password: profile-password-effective-canary
visible_option: profile-visible-value
provider.option: dotted-visible-value
`)
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper))
const secret = "environment-value-must-never-be-emitted"
t.Setenv("SCRIPTORIUM_EXPLAIN_REFERENCE", secret)
for _, tc := range []struct {
path string
want []string
unwant []string
}{
{
path: "/backends/local/extra_params/token",
unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"},
},
{
path: "/backends/local/extra_params/password",
unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"},
},
{
path: "/profiles/team~1summary/model",
want: []string{"effective-model", "inherited-model", "lower-model"},
unwant: []string{},
},
{
path: "/profiles/team~1summary/extra_params/token",
unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"},
},
{
path: "/profiles/team~1summary/extra_params/password",
unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"},
},
{
path: "/profiles/team~1summary/api_key_env",
want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"},
unwant: []string{secret},
},
{
path: "/profiles/team~1summary/extra_params/provider.option",
want: []string{"dotted-visible-value"},
unwant: []string{"provider/option"},
},
{
path: "",
want: []string{"profiles:", "effective-model"},
unwant: []string{"profile-token-displaced-canary", "backend-password-effective-canary"},
},
} {
args := []string{"configuration", "explain", "--config", configPath, "--path", tc.path}
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitOK || stderr != "" {
t.Fatalf("explain %q failed: code=%d stdout=%q stderr=%q", tc.path, code, stdout, stderr)
}
if stdout == "" {
t.Fatalf("explain %q produced no output", tc.path)
}
for _, value := range tc.want {
if !strings.Contains(stdout, value) {
t.Errorf("explain %q omitted %q: %s", tc.path, value, stdout)
}
}
for _, value := range tc.unwant {
if strings.Contains(stdout, value) {
t.Errorf("explain %q disclosed %q: %s", tc.path, value, stdout)
}
}
if strings.Contains(stdout, secret) {
t.Errorf("explain %q disclosed the resolved environment value: %s", tc.path, stdout)
}
}
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"configuration", "show", "--config", configPath})
if code != ExitOK || stderr != "" {
t.Fatalf("framework show failed: code=%d stderr=%q", code, stderr)
}
for _, value := range []string{"SCRIPTORIUM_EXPLAIN_REFERENCE", "backend-visible-value", "profile-visible-value", "dotted-visible-value"} {
if !strings.Contains(stdout, value) {
t.Errorf("effective report omitted visible configuration %q: %s", value, stdout)
}
}
for _, value := range []string{
"backend-token-displaced-canary", "backend-token-effective-canary",
"backend-password-displaced-canary", "backend-password-effective-canary",
"profile-token-displaced-canary", "profile-token-effective-canary",
"profile-password-displaced-canary", "profile-password-effective-canary", secret,
} {
if strings.Contains(stdout, value) {
t.Errorf("effective report disclosed classified or resolved value %q: %s", value, stdout)
}
}
profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{
"profile", "--config", configPath, "--profile", "team/summary", "--format", "json",
})
if profileCode != ExitOK || profileError != "" {
t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError)
}
var profile renderformat.ProfileInspection
if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil {
t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput)
}
if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" {
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams)
}
if strings.Contains(profileOutput, secret) {
t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput)
}
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"})
if code != ExitOK || stderr != "" {
t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr)
}
if !strings.Contains(rendered, "profile-visible-value") || strings.Contains(rendered, secret) {
t.Fatalf("prepared output did not preserve ordinary provider data or disclosed environment data: %s", rendered)
}
}
func TestFrameworkConfigurationExplainRejectsMissingMalformedAndUnknownPaths(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
for _, args := range [][]string{
{"configuration", "explain"},
{"configuration", "explain", "--path", "not-a-pointer"},
{"configuration", "explain", "--path", "/profiles/team~2summary/model"},
{"configuration", "explain", "--config", configPath, "--path", "/profiles/missing/model"},
} {
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid explanation path was accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
}
}
} }
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int { func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {