Add resolved PromptKit profile inspection

This commit is contained in:
2026-10-04 13:54:17 +00:00
parent 79154882b8
commit db8acbea43
6 changed files with 222 additions and 1 deletions

View File

@@ -15,6 +15,7 @@ notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name]
notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer> notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer>
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json] notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
~~~ ~~~
@@ -217,6 +218,40 @@ imports. Standard output contains only the selected explanation. PromptKit
scope constructs and validates its captured definitions, and its report can scope constructs and validates its captured definitions, and its report can
include ordinary inline prompt and provider-parameter data. include ordinary inline prompt and provider-parameter data.
## config profile
~~~
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
~~~
This command inspects one exact, nonblank PromptKit profile ID and writes one
JSON object followed by a newline:
The following illustrates the JSON shape; field values vary with the selected
profile and its configured backend.
~~~json
{"profile_id":"example","api_key_required":true,"effective_model_params":{"backend_id":"openrouter","endpoint":"https://provider.example/v1","model":"example-model","temperature":0,"max_tokens":0,"top_p":0,"timeout_seconds":600,"service_tier":"","reasoning_effort":"","api_key_env":"PROVIDER_API_KEY","extra_params":null}}
~~~
The object reports the resolved target, including backend-completed endpoint
and credential-name settings. An empty **backend_id** identifies an
endpoint-only profile. Credential names and **api_key_required** are configured
metadata; the command does not look up credential values or report whether an
environment variable is set. Credential-shaped keys in **extra_params** are
redacted.
This is the named profile's configured target, not a prediction of provider
defaults or the profile selected by an individual request. A request may omit
its profile and let its prompt select a default, and a run may supply a profile
override; this command applies neither behavior. Zero-valued fields show the
effective value but do not distinguish omission from an explicit zero. Use
[effective configuration](#config-effective) and
[configuration explanation](#config-explain) to inspect source presence and
history. Profile/configuration failures use exit status 1; malformed command
syntax, a blank ID, or an incorrect argument count use exit status 2. No
transcript or model call is involved.
## pipelines list ## pipelines list
~~~ ~~~

View File

@@ -613,6 +613,17 @@ call; preflight/run behavior remains unchanged.
Validation: `GOWORK=off go test ./internal/cli ./internal/framework/llm`; Validation: `GOWORK=off go test ./internal/cli ./internal/framework/llm`;
`git diff --check`. `git diff --check`.
> Completion: complete. Added `config profile` with exact nonblank IDs and one
> neutral JSON object/newline containing the captured resolved target and
> `APIKeyRequired` metadata. Tests cover backend-completed and endpoint-only
> targets, inheritance, null/zero values, credential-shaped parameter
> redaction, required but unset credentials, literal whitespace-bearing IDs,
> safe unknown-profile handling, writer failure, and selected overlays. The
> command performs no credential lookup or model call, and profile IDs now
> remain exact through the inspection adapter. Updated root help and the
> [CLI reference](../cli.md). `GOWORK=off go test ./internal/cli
> ./internal/framework/llm` and `git diff --check` pass.
Prompt size: one focused subcommand and its contract. Prompt size: one focused subcommand and its contract.
## Stage 13 — Maintain Examples And Operator Guidance ## Stage 13 — Maintain Examples And Operator Guidance

View File

@@ -118,6 +118,51 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
return 0 return 0
} }
func runConfigProfile(args []string, stdout, stderr io.Writer, opts Options) int {
fs := flag.NewFlagSet("config profile", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
overlay := fs.String("config-overlay", "", "configuration overlay")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
if fs.NArg() != 1 || strings.TrimSpace(fs.Arg(0)) == "" {
fmt.Fprintln(stderr, "notarius: config profile requires exactly one nonblank profile ID")
return 2
}
profileID := fs.Arg(0)
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
}
inspector, err := newPromptKitInspection(loaded.Config, opts)
if err != nil {
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
return 1
}
profile, err := inspector.InspectProfile(context.Background(), profileID)
if err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 1
}
result := struct {
ProfileID string `json:"profile_id"`
APIKeyRequired bool `json:"api_key_required"`
EffectiveModelParams frameworkllm.PromptKitEffectiveModelParams `json:"effective_model_params"`
}{
ProfileID: profile.ProfileID,
APIKeyRequired: profile.CredentialRequired,
EffectiveModelParams: profile.EffectiveModelParams,
}
if err := json.NewEncoder(stdout).Encode(result); err != nil {
fmt.Fprintf(stderr, "notarius: write PromptKit profile inspection: %v\n", err)
return 1
}
return 0
}
func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) { func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) {
return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{ return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{
Source: promptKitProfileSourceConfig(cfg), Source: promptKitProfileSourceConfig(cfg),

View File

@@ -3,6 +3,7 @@ package cli
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"errors" "errors"
"io" "io"
"path/filepath" "path/filepath"
@@ -372,6 +373,133 @@ func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.
} }
} }
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) {
t.Parallel()
root := t.TempDir()
profilePath := filepath.Join(root, "profiles.yml")
writeConfigKitFile(t, profilePath, `backends:
managed:
endpoint: https://provider.example.invalid/v1
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
profiles:
inherited:
backend: managed
model: inherited-model
temperature: 0.4
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
api_key_required: true
extra_params:
Authorization: private-authorization
ordinary: visible-value
selected:
extends: inherited
temperature: 0
extra_params: null
endpoint-only:
endpoint: https://endpoint.example.invalid/v1
model: endpoint-model
api_key_required: false
" literal profile ":
endpoint: https://literal.example.invalid/v1
model: literal-model
`)
configPath := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n")
assets, err := productionPromptAssets()
if err != nil {
t.Fatal(err)
}
lookupCalls := 0
factoryCalls := 0
opts := commandContractOptions(t)
opts.promptKitAssets = assets
opts.LookupEnv = func(string) (string, bool) {
lookupCalls++
return "", false
}
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
factoryCalls++
return nil, nil, errors.New("profile inspection must not construct an LLM client")
}
type profileReport struct {
ProfileID string `json:"profile_id"`
APIKeyRequired bool `json:"api_key_required"`
Target struct {
BackendID string `json:"backend_id"`
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
} `json:"effective_model_params"`
}
run := func(profileID string) (int, string, string, profileReport) {
t.Helper()
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts)
var report profileReport
if code == 0 {
if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 {
t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String())
}
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatalf("decode profile JSON %q: %v", stdout.String(), err)
}
}
return code, stdout.String(), stderr.String(), report
}
code, output, diagnostic, selected := run("selected")
if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil {
t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected)
}
code, output, diagnostic, inherited := run("inherited")
if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") {
t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited)
}
code, output, diagnostic, endpointOnly := run("endpoint-only")
if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" {
t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly)
}
code, output, diagnostic, literal := run(" literal profile ")
if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" {
t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal)
}
if lookupCalls != 0 || factoryCalls != 0 {
t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls)
}
}
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) {
t.Parallel()
configPath := writeCommandConfig(t, "session", "other")
for _, test := range []struct {
name string
args []string
want int
}{
{name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2},
{name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2},
{name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2},
{name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 {
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
}
})
}
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n")
configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
var stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") {
t.Fatalf("profile writer failure = %d, %q", code, stderr.String())
}
}
type failingInspectionWriter struct{} type failingInspectionWriter struct{}
func (failingInspectionWriter) Write([]byte) (int, error) { func (failingInspectionWriter) Write([]byte) (int, error) {

View File

@@ -37,6 +37,7 @@ const usage = `Usage:
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b] notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name]
notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer> notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer>
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json] notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
` `
@@ -1059,6 +1060,8 @@ func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
return runConfigEffective(args[1:], stdout, stderr, opts) return runConfigEffective(args[1:], stdout, stderr, opts)
case "explain": case "explain":
return runConfigExplain(args[1:], stdout, stderr, opts) return runConfigExplain(args[1:], stdout, stderr, opts)
case "profile":
return runConfigProfile(args[1:], stdout, stderr, opts)
default: default:
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0]) fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
writeUsage(stderr) writeUsage(stderr)

View File

@@ -94,7 +94,6 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
if i == nil || i.engine == nil { if i == nil || i.engine == nil {
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil") return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
} }
profileID = strings.TrimSpace(profileID)
inspection, err := i.engine.InspectProfile(ctx, profileID) inspection, err := i.engine.InspectProfile(ctx, profileID)
if err != nil { if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil { if ctxErr := ctx.Err(); ctxErr != nil {