Add scoped PromptKit configuration reports

This commit is contained in:
2026-10-04 13:47:00 +00:00
parent c848befb53
commit 79154882b8
5 changed files with 187 additions and 10 deletions

View File

@@ -13,6 +13,8 @@ notarius help
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name]
notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer>
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
~~~
@@ -188,8 +190,16 @@ notarius config effective [--config path/to/config.yml] [--config-overlay name]
~~~
This command writes the selected composed configuration as deterministic,
redacted YAML. Standard output contains only that YAML: it does not validate a
pipeline, prepare modules, materialize references, or contact a provider.
redacted YAML. By default, it reports the Notarius configuration. Set
**--scope promptkit** to report PromptKit's captured effective configuration;
**--scope notarius** is equivalent to the default. Only the PromptKit scope
constructs the PromptKit inspection engine, using the selected configuration
and registered application assets. Engine construction validates and captures
the definitions. Standard output contains only the selected report: the
command does not validate a pipeline, prepare modules, materialize references,
resolve credentials, or contact a provider. PromptKit output can include
ordinary inline prompt and provider-parameter data; credential-shaped provider
parameters are redacted.
## config explain
@@ -197,11 +207,15 @@ pipeline, prepare modules, materialize references, or contact a provider.
notarius config explain [--config path/to/config.yml] [--config-overlay name] <json-pointer>
~~~
This command writes ConfigKit's redacted, human-readable explanation for one
canonical JSON Pointer. Supply `""` explicitly to explain the root. A malformed
pointer is a syntax error; a well-formed unknown path is a command failure.
The explanation can describe a value displaced by a selected overlay. Standard
output contains only the explanation.
This command writes a redacted, human-readable explanation for one canonical
JSON Pointer. Supply `""` explicitly to explain the root. By default, it
explains Notarius configuration; **--scope promptkit** selects PromptKit
configuration history, while **--scope notarius** is equivalent to the default.
A malformed pointer is a syntax error; a well-formed unknown path is a command
failure. The explanation can describe values displaced by source overlays or
imports. Standard output contains only the selected explanation. PromptKit
scope constructs and validates its captured definitions, and its report can
include ordinary inline prompt and provider-parameter data.
## pipelines list

View File

@@ -576,6 +576,16 @@ its requested report without transcript input, credentials, or provider work.
Validation: `GOWORK=off go test ./internal/cli`; `git diff --check`.
> Completion: complete. Added opt-in `promptkit` scope for effective and
> explanation reports while preserving the default and explicit `notarius`
> output. The PromptKit path builds its inspector from the selected normalized
> configuration and registered assets, and reports malformed/unknown/history
> selectors and writer failures with the established exit classes. CLI
> coverage verifies imported and fallback origins, escaped pointers,
> credential redaction, and no LLM factory invocation. Updated root help and
> [CLI reference](../cli.md). `GOWORK=off go test ./internal/cli` and
> `git diff --check` pass.
Prompt size: two scoped branches, not resolved-profile parsing.
## Stage 12 — Add Resolved Profile Inspection To The CLI

View File

@@ -3,6 +3,7 @@ package cli
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
@@ -10,6 +11,7 @@ import (
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int {
@@ -17,6 +19,7 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
overlay := fs.String("config-overlay", "", "configuration overlay")
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -25,12 +28,28 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
if *scope != "notarius" && *scope != "promptkit" {
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
return 2
}
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
}
if err := loaded.WriteEffective(stdout); err != nil {
if *scope == "notarius" {
if err := loaded.WriteEffective(stdout); err != nil {
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
return 1
}
return 0
}
inspector, err := newPromptKitInspection(loaded.Config, opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
return 1
}
if err := inspector.WriteEffectiveConfiguration(stdout); err != nil {
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
return 1
}
@@ -42,6 +61,7 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
overlay := fs.String("config-overlay", "", "configuration overlay")
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -50,7 +70,37 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer")
return 2
}
if *scope != "notarius" && *scope != "promptkit" {
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
return 2
}
pointer := fs.Arg(0)
if *scope == "promptkit" {
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
}
inspector, err := newPromptKitInspection(loaded.Config, opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
return 1
}
found, err := inspector.WriteConfigurationExplanation(stdout, pointer)
if err != nil {
if errors.Is(err, frameworkllm.ErrInvalidPromptKitConfigurationPointer) {
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
return 2
}
fmt.Fprintf(stderr, "notarius: write PromptKit configuration explanation: %v\n", err)
return 1
}
if !found {
fmt.Fprintf(stderr, "notarius: PromptKit configuration pointer %q is unknown\n", pointer)
return 1
}
return 0
}
parsed, err := configkit.ParsePath(pointer)
if err != nil || parsed.String() != pointer {
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
@@ -68,6 +118,13 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
return 0
}
func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) {
return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{
Source: promptKitProfileSourceConfig(cfg),
Assets: opts.promptKitAssets,
})
}
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
fs.SetOutput(io.Discard)

View File

@@ -276,6 +276,102 @@ func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
}
}
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
t.Parallel()
root := t.TempDir()
configPath := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
base:
endpoint: https://example.invalid/v1
model: base-model
extra_params:
Authorization: base-auth-secret
ordinary: visible-value
`)
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
imports:
- path: base.yml
profiles:
"escaped/id":
model: imported-model
dnd-extraction:
model: imported-override
`)
assets, err := productionPromptAssets()
if err != nil {
t.Fatal(err)
}
factoryCalls := 0
opts := commandContractOptions(t)
opts.promptKitAssets = assets
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, errors.New("inspection must not construct an LLM client")
}
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
}
defaultReport := stdout.String()
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
}
if factoryCalls != 0 {
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
}
}
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
t.Parallel()
path := writeCommandConfig(t, "session", "other")
opts := commandContractOptions(t)
for _, test := range []struct {
name string
args []string
want int
}{
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
}
})
}
var stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
}
}
type failingInspectionWriter struct{}
func (failingInspectionWriter) Write([]byte) (int, error) {

View File

@@ -35,8 +35,8 @@ const usage = `Usage:
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius config effective --config path/to/config.yml [--config-overlay name]
notarius config explain --config path/to/config.yml [--config-overlay name] <json-pointer>
notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name]
notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer>
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
`