Add scoped PromptKit configuration reports
This commit is contained in:
28
docs/cli.md
28
docs/cli.md
@@ -13,6 +13,8 @@ notarius help
|
|||||||
notarius --version
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
|
notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name]
|
||||||
|
notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
||||||
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
@@ -188,8 +190,16 @@ notarius config effective [--config path/to/config.yml] [--config-overlay name]
|
|||||||
~~~
|
~~~
|
||||||
|
|
||||||
This command writes the selected composed configuration as deterministic,
|
This command writes the selected composed configuration as deterministic,
|
||||||
redacted YAML. Standard output contains only that YAML: it does not validate a
|
redacted YAML. By default, it reports the Notarius configuration. Set
|
||||||
pipeline, prepare modules, materialize references, or contact a provider.
|
**--scope promptkit** to report PromptKit's captured effective configuration;
|
||||||
|
**--scope notarius** is equivalent to the default. Only the PromptKit scope
|
||||||
|
constructs the PromptKit inspection engine, using the selected configuration
|
||||||
|
and registered application assets. Engine construction validates and captures
|
||||||
|
the definitions. Standard output contains only the selected report: the
|
||||||
|
command does not validate a pipeline, prepare modules, materialize references,
|
||||||
|
resolve credentials, or contact a provider. PromptKit output can include
|
||||||
|
ordinary inline prompt and provider-parameter data; credential-shaped provider
|
||||||
|
parameters are redacted.
|
||||||
|
|
||||||
## config explain
|
## config explain
|
||||||
|
|
||||||
@@ -197,11 +207,15 @@ pipeline, prepare modules, materialize references, or contact a provider.
|
|||||||
notarius config explain [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
notarius config explain [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
This command writes ConfigKit's redacted, human-readable explanation for one
|
This command writes a redacted, human-readable explanation for one canonical
|
||||||
canonical JSON Pointer. Supply `""` explicitly to explain the root. A malformed
|
JSON Pointer. Supply `""` explicitly to explain the root. By default, it
|
||||||
pointer is a syntax error; a well-formed unknown path is a command failure.
|
explains Notarius configuration; **--scope promptkit** selects PromptKit
|
||||||
The explanation can describe a value displaced by a selected overlay. Standard
|
configuration history, while **--scope notarius** is equivalent to the default.
|
||||||
output contains only the explanation.
|
A malformed pointer is a syntax error; a well-formed unknown path is a command
|
||||||
|
failure. The explanation can describe values displaced by source overlays or
|
||||||
|
imports. Standard output contains only the selected explanation. PromptKit
|
||||||
|
scope constructs and validates its captured definitions, and its report can
|
||||||
|
include ordinary inline prompt and provider-parameter data.
|
||||||
|
|
||||||
## pipelines list
|
## pipelines list
|
||||||
|
|
||||||
|
|||||||
@@ -576,6 +576,16 @@ its requested report without transcript input, credentials, or provider work.
|
|||||||
|
|
||||||
Validation: `GOWORK=off go test ./internal/cli`; `git diff --check`.
|
Validation: `GOWORK=off go test ./internal/cli`; `git diff --check`.
|
||||||
|
|
||||||
|
> Completion: complete. Added opt-in `promptkit` scope for effective and
|
||||||
|
> explanation reports while preserving the default and explicit `notarius`
|
||||||
|
> output. The PromptKit path builds its inspector from the selected normalized
|
||||||
|
> configuration and registered assets, and reports malformed/unknown/history
|
||||||
|
> selectors and writer failures with the established exit classes. CLI
|
||||||
|
> coverage verifies imported and fallback origins, escaped pointers,
|
||||||
|
> credential redaction, and no LLM factory invocation. Updated root help and
|
||||||
|
> [CLI reference](../cli.md). `GOWORK=off go test ./internal/cli` and
|
||||||
|
> `git diff --check` pass.
|
||||||
|
|
||||||
Prompt size: two scoped branches, not resolved-profile parsing.
|
Prompt size: two scoped branches, not resolved-profile parsing.
|
||||||
|
|
||||||
## Stage 12 — Add Resolved Profile Inspection To The CLI
|
## Stage 12 — Add Resolved Profile Inspection To The CLI
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -10,6 +11,7 @@ import (
|
|||||||
|
|
||||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
|
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int {
|
func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
@@ -17,6 +19,7 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
configPath := fs.String("config", "", "config file path")
|
configPath := fs.String("config", "", "config file path")
|
||||||
overlay := fs.String("config-overlay", "", "configuration overlay")
|
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||||
|
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
@@ -25,12 +28,28 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
|
|||||||
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
|
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if *scope != "notarius" && *scope != "promptkit" {
|
||||||
|
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeConfigLoadError(stderr, err)
|
writeConfigLoadError(stderr, err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
if err := loaded.WriteEffective(stdout); err != nil {
|
if *scope == "notarius" {
|
||||||
|
if err := loaded.WriteEffective(stdout); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if err := inspector.WriteEffectiveConfiguration(stdout); err != nil {
|
||||||
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: write effective configuration: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
@@ -42,6 +61,7 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
|||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
configPath := fs.String("config", "", "config file path")
|
configPath := fs.String("config", "", "config file path")
|
||||||
overlay := fs.String("config-overlay", "", "configuration overlay")
|
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||||
|
scope := fs.String("scope", "notarius", "configuration scope: notarius or promptkit")
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
return 2
|
return 2
|
||||||
@@ -50,7 +70,37 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
|||||||
fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer")
|
fmt.Fprintln(stderr, "notarius: config explain requires exactly one JSON Pointer")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if *scope != "notarius" && *scope != "promptkit" {
|
||||||
|
fmt.Fprintf(stderr, "notarius: invalid configuration scope %q (want notarius or promptkit)\n", *scope)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
pointer := fs.Arg(0)
|
pointer := fs.Arg(0)
|
||||||
|
if *scope == "promptkit" {
|
||||||
|
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||||
|
if err != nil {
|
||||||
|
writeConfigLoadError(stderr, err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
found, err := inspector.WriteConfigurationExplanation(stdout, pointer)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, frameworkllm.ErrInvalidPromptKitConfigurationPointer) {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stderr, "notarius: write PromptKit configuration explanation: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
fmt.Fprintf(stderr, "notarius: PromptKit configuration pointer %q is unknown\n", pointer)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
parsed, err := configkit.ParsePath(pointer)
|
parsed, err := configkit.ParsePath(pointer)
|
||||||
if err != nil || parsed.String() != pointer {
|
if err != nil || parsed.String() != pointer {
|
||||||
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
|
||||||
@@ -68,6 +118,13 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) {
|
||||||
|
return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{
|
||||||
|
Source: promptKitProfileSourceConfig(cfg),
|
||||||
|
Assets: opts.promptKitAssets,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
|
func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
|
||||||
fs.SetOutput(io.Discard)
|
fs.SetOutput(io.Discard)
|
||||||
|
|||||||
@@ -276,6 +276,102 @@ func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
root := t.TempDir()
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
|
||||||
|
base:
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: base-model
|
||||||
|
extra_params:
|
||||||
|
Authorization: base-auth-secret
|
||||||
|
ordinary: visible-value
|
||||||
|
`)
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
|
||||||
|
imports:
|
||||||
|
- path: base.yml
|
||||||
|
profiles:
|
||||||
|
"escaped/id":
|
||||||
|
model: imported-model
|
||||||
|
dnd-extraction:
|
||||||
|
model: imported-override
|
||||||
|
`)
|
||||||
|
assets, err := productionPromptAssets()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
factoryCalls := 0
|
||||||
|
opts := commandContractOptions(t)
|
||||||
|
opts.promptKitAssets = assets
|
||||||
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
|
factoryCalls++
|
||||||
|
return nil, nil, errors.New("inspection must not construct an LLM client")
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
defaultReport := stdout.String()
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
stdout.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
if factoryCalls != 0 {
|
||||||
|
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
path := writeCommandConfig(t, "session", "other")
|
||||||
|
opts := commandContractOptions(t)
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
|
||||||
|
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
|
||||||
|
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
|
||||||
|
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
||||||
|
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type failingInspectionWriter struct{}
|
type failingInspectionWriter struct{}
|
||||||
|
|
||||||
func (failingInspectionWriter) Write([]byte) (int, error) {
|
func (failingInspectionWriter) Write([]byte) (int, error) {
|
||||||
|
|||||||
@@ -35,8 +35,8 @@ const usage = `Usage:
|
|||||||
notarius --version
|
notarius --version
|
||||||
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
||||||
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius config effective --config path/to/config.yml [--config-overlay name]
|
notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name]
|
||||||
notarius config explain --config path/to/config.yml [--config-overlay name] <json-pointer>
|
notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer>
|
||||||
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
|
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
|
||||||
`
|
`
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user