From 411a67b241681b408069abf764512a2a804ed83c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 12:46:12 +0000 Subject: [PATCH] Explain framework configuration paths safely --- docs/roadmap/implementation.md | 18 +- internal/adapter/cli/framework_config.go | 29 ++- internal/adapter/cli/framework_config_test.go | 210 +++++++++++++++++- 3 files changed, 248 insertions(+), 9 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d2d255d..94c9cc6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -3,8 +3,8 @@ ## Status, Baseline, And Authority Planned on 2026-10-04 against Scriptorium -`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–9 are complete; stages -10–13 are pending. +`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–10 are complete; stages +11–13 are pending. This plan is for a Luna coding agent: implement the stages in sequential order, normally one stage per implementation prompt. The [feature roadmap](promptkit.md) owns the target state, scope, and confirmed @@ -565,7 +565,19 @@ scope with application inspection or altering existing presentation contracts. ## Stage 10: Add Literal-Path Explanations And Prove Scoped Redaction -Status: Pending. Depends on Stage 9. +Status: Complete. `inspect configuration explain` now requires an explicitly +provided JSON Pointer, accepts the empty root, parses with ConfigKit, detects +unknown paths before writing, and delegates explanations to the captured +PromptKit inspection view. Tests cover inherited and overlaid history, escaped +slash IDs, dotted keys, root output, malformed/unknown paths, native writer +errors, effective and historical `token`/`password` redaction, visible +environment-reference names and ordinary values, and the unchanged disclosure +behavior of profile inspection and prepared output. Application validation +remains independent of invalid framework definitions, while framework +validation rejects them without a partial report. Focused inspection tests, +`go test ./...`, `go test -race ./...`, `go vet ./...`, the CLI build, +`gofmt`, `git diff --check`, and all four operations against +`examples/config.yml` pass. Files: framework inspection parser/handler/tests from Stage 9. diff --git a/internal/adapter/cli/framework_config.go b/internal/adapter/cli/framework_config.go index 1de27ee..be7a5d9 100644 --- a/internal/adapter/cli/framework_config.go +++ b/internal/adapter/cli/framework_config.go @@ -12,7 +12,9 @@ import ( type frameworkConfigurationConfig struct { mode, configPath, configOverlay, schemaDir string + path string configExplicit bool + pathExplicit bool overrides []configkit.Override diagnostics []configkit.Diagnostic } @@ -39,6 +41,14 @@ func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Write writeApplicationError(stderr, "inspect configuration parse", err) return ExitRuntimeError } + var explanationPath configkit.Path + if cfg.mode == "explain" { + explanationPath, err = configkit.ParsePath(cfg.path) + if err != nil { + writeApplicationError(stderr, "inspect configuration path", err) + return ExitRuntimeError + } + } settings, err := resolveAppSettingsForFrameworkConfiguration(cfg) if err != nil { writeApplicationError(stderr, "inspect configuration", err) @@ -64,6 +74,18 @@ func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Write return ExitOK } + if cfg.mode == "explain" { + if _, ok := view.Explain(explanationPath); !ok { + writeApplicationError(stderr, "inspect configuration path", fmt.Errorf("unknown path %q", cfg.path)) + return ExitRuntimeError + } + if err := view.WriteExplanation(stdout, explanationPath); err != nil { + fmt.Fprintf(stderr, "inspect configuration output error: %v\n", err) + return ExitRuntimeError + } + return ExitOK + } + switch cfg.mode { case "show": err = view.WriteEffective(stdout) @@ -84,7 +106,7 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf } cfg := &frameworkConfigurationConfig{mode: args[0]} switch cfg.mode { - case "validate", "show", "sources": + case "validate", "show", "sources", "explain": default: return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode) } @@ -95,6 +117,7 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf var promptkitSources promptKitSourceFlagValue fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)") fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas") + fs.StringVar(&cfg.path, "path", "", "JSON Pointer to explain (required for explain)") if err := fs.Parse(args[1:]); err != nil { return nil, err } @@ -102,6 +125,10 @@ func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConf return nil, fmt.Errorf("unexpected positional args: %v", fs.Args()) } cfg.configExplicit = flagWasSet(fs, "config") + cfg.pathExplicit = flagWasSet(fs, "path") + if cfg.mode == "explain" && !cfg.pathExplicit { + return nil, errors.New("--path is required for explain") + } cfg.overrides = applicationOverrides(fs) return cfg, nil } diff --git a/internal/adapter/cli/framework_config_test.go b/internal/adapter/cli/framework_config_test.go index 9877e6c..8f8afa9 100644 --- a/internal/adapter/cli/framework_config_test.go +++ b/internal/adapter/cli/framework_config_test.go @@ -11,6 +11,7 @@ import ( "testing" configkit "gitea.maximumdirect.net/eric/ConfigKit" + renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" ) func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) { @@ -143,11 +144,21 @@ func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t format: text validation_mode: none `) - code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ - "configuration", "show", "--promptkit-source", frameworkFile, - }) - if code != ExitRuntimeError || stdout != "" || stderr == "" { - t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr) + configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) + code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath}) + if code != ExitOK || stdout != "" || stderr != "" { + t.Fatalf("application validation loaded framework definitions: code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + for _, args := range [][]string{ + {"configuration", "validate", "--config", configPath}, + {"configuration", "show", "--config", configPath}, + {"configuration", "sources", "--config", configPath}, + {"configuration", "explain", "--config", configPath, "--path", ""}, + } { + code, stdout, stderr = runCLICommand(t, inspectCommand, args) + if code != ExitRuntimeError || stdout != "" || stderr == "" { + t.Fatalf("invalid final framework definition was reported as success: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) + } } } @@ -171,6 +182,195 @@ func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) { t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String()) } + stderr.Reset() + code = inspectCommand([]string{"configuration", "explain", "--config", configPath, "--path", ""}, frameworkConfigurationFailingWriter{err: writerError}, &stderr) + if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) { + t.Fatalf("explanation writer failure was not reported: code=%d stderr=%q", code, stderr.String()) + } +} + +func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) { + root := t.TempDir() + lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends: + local: + endpoint: http://127.0.0.1:1/v1 + api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE + extra_params: + token: backend-token-displaced-canary + password: backend-password-canary + visible_option: backend-visible-value + provider.option: dotted-visible-value +profiles: + base: + backend: local + model: inherited-model + api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE + extra_params: + token: profile-token-displaced-canary + password: profile-password-displaced-canary + visible_option: profile-visible-value + provider.option: dotted-visible-value + "team/summary": + extends: base + model: lower-model +prompts: + explainable: + "1": + default_profile: "team/summary" + messages: + - role: user + content: hello + output: + format: text + validation_mode: none +`) + upperRoot := t.TempDir() + upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends: + local: + endpoint: http://127.0.0.1:1/v1 + api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE + extra_params: + token: backend-token-effective-canary + password: backend-password-effective-canary + visible_option: backend-visible-value + provider.option: dotted-visible-value +profiles: + "team/summary": + model: effective-model + extra_params: + token: profile-token-effective-canary + password: profile-password-effective-canary + visible_option: profile-visible-value + provider.option: dotted-visible-value +`) + configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper)) + const secret = "environment-value-must-never-be-emitted" + t.Setenv("SCRIPTORIUM_EXPLAIN_REFERENCE", secret) + + for _, tc := range []struct { + path string + want []string + unwant []string + }{ + { + path: "/backends/local/extra_params/token", + unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"}, + }, + { + path: "/backends/local/extra_params/password", + unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"}, + }, + { + path: "/profiles/team~1summary/model", + want: []string{"effective-model", "inherited-model", "lower-model"}, + unwant: []string{}, + }, + { + path: "/profiles/team~1summary/extra_params/token", + unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"}, + }, + { + path: "/profiles/team~1summary/extra_params/password", + unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"}, + }, + { + path: "/profiles/team~1summary/api_key_env", + want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"}, + unwant: []string{secret}, + }, + { + path: "/profiles/team~1summary/extra_params/provider.option", + want: []string{"dotted-visible-value"}, + unwant: []string{"provider/option"}, + }, + { + path: "", + want: []string{"profiles:", "effective-model"}, + unwant: []string{"profile-token-displaced-canary", "backend-password-effective-canary"}, + }, + } { + args := []string{"configuration", "explain", "--config", configPath, "--path", tc.path} + code, stdout, stderr := runCLICommand(t, inspectCommand, args) + if code != ExitOK || stderr != "" { + t.Fatalf("explain %q failed: code=%d stdout=%q stderr=%q", tc.path, code, stdout, stderr) + } + if stdout == "" { + t.Fatalf("explain %q produced no output", tc.path) + } + for _, value := range tc.want { + if !strings.Contains(stdout, value) { + t.Errorf("explain %q omitted %q: %s", tc.path, value, stdout) + } + } + for _, value := range tc.unwant { + if strings.Contains(stdout, value) { + t.Errorf("explain %q disclosed %q: %s", tc.path, value, stdout) + } + } + if strings.Contains(stdout, secret) { + t.Errorf("explain %q disclosed the resolved environment value: %s", tc.path, stdout) + } + } + + code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"configuration", "show", "--config", configPath}) + if code != ExitOK || stderr != "" { + t.Fatalf("framework show failed: code=%d stderr=%q", code, stderr) + } + for _, value := range []string{"SCRIPTORIUM_EXPLAIN_REFERENCE", "backend-visible-value", "profile-visible-value", "dotted-visible-value"} { + if !strings.Contains(stdout, value) { + t.Errorf("effective report omitted visible configuration %q: %s", value, stdout) + } + } + for _, value := range []string{ + "backend-token-displaced-canary", "backend-token-effective-canary", + "backend-password-displaced-canary", "backend-password-effective-canary", + "profile-token-displaced-canary", "profile-token-effective-canary", + "profile-password-displaced-canary", "profile-password-effective-canary", secret, + } { + if strings.Contains(stdout, value) { + t.Errorf("effective report disclosed classified or resolved value %q: %s", value, stdout) + } + } + + profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{ + "profile", "--config", configPath, "--profile", "team/summary", "--format", "json", + }) + if profileCode != ExitOK || profileError != "" { + t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError) + } + var profile renderformat.ProfileInspection + if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil { + t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput) + } + if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" { + t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams) + } + if strings.Contains(profileOutput, secret) { + t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput) + } + + code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"}) + if code != ExitOK || stderr != "" { + t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr) + } + if !strings.Contains(rendered, "profile-visible-value") || strings.Contains(rendered, secret) { + t.Fatalf("prepared output did not preserve ordinary provider data or disclosed environment data: %s", rendered) + } +} + +func TestFrameworkConfigurationExplainRejectsMissingMalformedAndUnknownPaths(t *testing.T) { + configPath := writeAppConfigFile(t, "{}\n") + for _, args := range [][]string{ + {"configuration", "explain"}, + {"configuration", "explain", "--path", "not-a-pointer"}, + {"configuration", "explain", "--path", "/profiles/team~2summary/model"}, + {"configuration", "explain", "--config", configPath, "--path", "/profiles/missing/model"}, + } { + code, stdout, stderr := runCLICommand(t, inspectCommand, args) + if code != ExitRuntimeError || stdout != "" || stderr == "" { + t.Fatalf("invalid explanation path was accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) + } + } } func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {