Add resolved PromptKit profile inspection
This commit is contained in:
35
docs/cli.md
35
docs/cli.md
@@ -15,6 +15,7 @@ notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
|
|||||||
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name]
|
notarius config effective [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name]
|
||||||
notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
notarius config explain [--scope notarius|promptkit] [--config path/to/config.yml] [--config-overlay name] <json-pointer>
|
||||||
|
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||||
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
|
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
@@ -217,6 +218,40 @@ imports. Standard output contains only the selected explanation. PromptKit
|
|||||||
scope constructs and validates its captured definitions, and its report can
|
scope constructs and validates its captured definitions, and its report can
|
||||||
include ordinary inline prompt and provider-parameter data.
|
include ordinary inline prompt and provider-parameter data.
|
||||||
|
|
||||||
|
## config profile
|
||||||
|
|
||||||
|
~~~
|
||||||
|
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||||
|
~~~
|
||||||
|
|
||||||
|
This command inspects one exact, nonblank PromptKit profile ID and writes one
|
||||||
|
JSON object followed by a newline:
|
||||||
|
|
||||||
|
The following illustrates the JSON shape; field values vary with the selected
|
||||||
|
profile and its configured backend.
|
||||||
|
|
||||||
|
~~~json
|
||||||
|
{"profile_id":"example","api_key_required":true,"effective_model_params":{"backend_id":"openrouter","endpoint":"https://provider.example/v1","model":"example-model","temperature":0,"max_tokens":0,"top_p":0,"timeout_seconds":600,"service_tier":"","reasoning_effort":"","api_key_env":"PROVIDER_API_KEY","extra_params":null}}
|
||||||
|
~~~
|
||||||
|
|
||||||
|
The object reports the resolved target, including backend-completed endpoint
|
||||||
|
and credential-name settings. An empty **backend_id** identifies an
|
||||||
|
endpoint-only profile. Credential names and **api_key_required** are configured
|
||||||
|
metadata; the command does not look up credential values or report whether an
|
||||||
|
environment variable is set. Credential-shaped keys in **extra_params** are
|
||||||
|
redacted.
|
||||||
|
|
||||||
|
This is the named profile's configured target, not a prediction of provider
|
||||||
|
defaults or the profile selected by an individual request. A request may omit
|
||||||
|
its profile and let its prompt select a default, and a run may supply a profile
|
||||||
|
override; this command applies neither behavior. Zero-valued fields show the
|
||||||
|
effective value but do not distinguish omission from an explicit zero. Use
|
||||||
|
[effective configuration](#config-effective) and
|
||||||
|
[configuration explanation](#config-explain) to inspect source presence and
|
||||||
|
history. Profile/configuration failures use exit status 1; malformed command
|
||||||
|
syntax, a blank ID, or an incorrect argument count use exit status 2. No
|
||||||
|
transcript or model call is involved.
|
||||||
|
|
||||||
## pipelines list
|
## pipelines list
|
||||||
|
|
||||||
~~~
|
~~~
|
||||||
|
|||||||
@@ -613,6 +613,17 @@ call; preflight/run behavior remains unchanged.
|
|||||||
Validation: `GOWORK=off go test ./internal/cli ./internal/framework/llm`;
|
Validation: `GOWORK=off go test ./internal/cli ./internal/framework/llm`;
|
||||||
`git diff --check`.
|
`git diff --check`.
|
||||||
|
|
||||||
|
> Completion: complete. Added `config profile` with exact nonblank IDs and one
|
||||||
|
> neutral JSON object/newline containing the captured resolved target and
|
||||||
|
> `APIKeyRequired` metadata. Tests cover backend-completed and endpoint-only
|
||||||
|
> targets, inheritance, null/zero values, credential-shaped parameter
|
||||||
|
> redaction, required but unset credentials, literal whitespace-bearing IDs,
|
||||||
|
> safe unknown-profile handling, writer failure, and selected overlays. The
|
||||||
|
> command performs no credential lookup or model call, and profile IDs now
|
||||||
|
> remain exact through the inspection adapter. Updated root help and the
|
||||||
|
> [CLI reference](../cli.md). `GOWORK=off go test ./internal/cli
|
||||||
|
> ./internal/framework/llm` and `git diff --check` pass.
|
||||||
|
|
||||||
Prompt size: one focused subcommand and its contract.
|
Prompt size: one focused subcommand and its contract.
|
||||||
|
|
||||||
## Stage 13 — Maintain Examples And Operator Guidance
|
## Stage 13 — Maintain Examples And Operator Guidance
|
||||||
|
|||||||
@@ -118,6 +118,51 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runConfigProfile(args []string, stdout, stderr io.Writer, opts Options) int {
|
||||||
|
fs := flag.NewFlagSet("config profile", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
configPath := fs.String("config", "", "config file path")
|
||||||
|
overlay := fs.String("config-overlay", "", "configuration overlay")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
if fs.NArg() != 1 || strings.TrimSpace(fs.Arg(0)) == "" {
|
||||||
|
fmt.Fprintln(stderr, "notarius: config profile requires exactly one nonblank profile ID")
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
profileID := fs.Arg(0)
|
||||||
|
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
|
||||||
|
if err != nil {
|
||||||
|
writeConfigLoadError(stderr, err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
inspector, err := newPromptKitInspection(loaded.Config, opts)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: inspect PromptKit configuration: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
profile, err := inspector.InspectProfile(context.Background(), profileID)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
result := struct {
|
||||||
|
ProfileID string `json:"profile_id"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
EffectiveModelParams frameworkllm.PromptKitEffectiveModelParams `json:"effective_model_params"`
|
||||||
|
}{
|
||||||
|
ProfileID: profile.ProfileID,
|
||||||
|
APIKeyRequired: profile.CredentialRequired,
|
||||||
|
EffectiveModelParams: profile.EffectiveModelParams,
|
||||||
|
}
|
||||||
|
if err := json.NewEncoder(stdout).Encode(result); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "notarius: write PromptKit profile inspection: %v\n", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) {
|
func newPromptKitInspection(cfg config.Config, opts Options) (*frameworkllm.PromptKitProfileInspector, error) {
|
||||||
return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{
|
return frameworkllm.NewPromptKitProfileInspector(frameworkllm.PromptKitProfileInspectorConfig{
|
||||||
Source: promptKitProfileSourceConfig(cfg),
|
Source: promptKitProfileSourceConfig(cfg),
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -372,6 +373,133 @@ func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
root := t.TempDir()
|
||||||
|
profilePath := filepath.Join(root, "profiles.yml")
|
||||||
|
writeConfigKitFile(t, profilePath, `backends:
|
||||||
|
managed:
|
||||||
|
endpoint: https://provider.example.invalid/v1
|
||||||
|
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||||
|
profiles:
|
||||||
|
inherited:
|
||||||
|
backend: managed
|
||||||
|
model: inherited-model
|
||||||
|
temperature: 0.4
|
||||||
|
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||||
|
api_key_required: true
|
||||||
|
extra_params:
|
||||||
|
Authorization: private-authorization
|
||||||
|
ordinary: visible-value
|
||||||
|
selected:
|
||||||
|
extends: inherited
|
||||||
|
temperature: 0
|
||||||
|
extra_params: null
|
||||||
|
endpoint-only:
|
||||||
|
endpoint: https://endpoint.example.invalid/v1
|
||||||
|
model: endpoint-model
|
||||||
|
api_key_required: false
|
||||||
|
" literal profile ":
|
||||||
|
endpoint: https://literal.example.invalid/v1
|
||||||
|
model: literal-model
|
||||||
|
`)
|
||||||
|
configPath := filepath.Join(root, "notarius.yml")
|
||||||
|
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||||
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n")
|
||||||
|
assets, err := productionPromptAssets()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lookupCalls := 0
|
||||||
|
factoryCalls := 0
|
||||||
|
opts := commandContractOptions(t)
|
||||||
|
opts.promptKitAssets = assets
|
||||||
|
opts.LookupEnv = func(string) (string, bool) {
|
||||||
|
lookupCalls++
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
|
factoryCalls++
|
||||||
|
return nil, nil, errors.New("profile inspection must not construct an LLM client")
|
||||||
|
}
|
||||||
|
|
||||||
|
type profileReport struct {
|
||||||
|
ProfileID string `json:"profile_id"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
Target struct {
|
||||||
|
BackendID string `json:"backend_id"`
|
||||||
|
Endpoint string `json:"endpoint"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Temperature float64 `json:"temperature"`
|
||||||
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
|
} `json:"effective_model_params"`
|
||||||
|
}
|
||||||
|
run := func(profileID string) (int, string, string, profileReport) {
|
||||||
|
t.Helper()
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts)
|
||||||
|
var report profileReport
|
||||||
|
if code == 0 {
|
||||||
|
if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 {
|
||||||
|
t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String())
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||||
|
t.Fatalf("decode profile JSON %q: %v", stdout.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return code, stdout.String(), stderr.String(), report
|
||||||
|
}
|
||||||
|
code, output, diagnostic, selected := run("selected")
|
||||||
|
if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil {
|
||||||
|
t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected)
|
||||||
|
}
|
||||||
|
code, output, diagnostic, inherited := run("inherited")
|
||||||
|
if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") {
|
||||||
|
t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited)
|
||||||
|
}
|
||||||
|
code, output, diagnostic, endpointOnly := run("endpoint-only")
|
||||||
|
if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" {
|
||||||
|
t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly)
|
||||||
|
}
|
||||||
|
code, output, diagnostic, literal := run(" literal profile ")
|
||||||
|
if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" {
|
||||||
|
t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal)
|
||||||
|
}
|
||||||
|
if lookupCalls != 0 || factoryCalls != 0 {
|
||||||
|
t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
configPath := writeCommandConfig(t, "session", "other")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2},
|
||||||
|
{name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2},
|
||||||
|
{name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2},
|
||||||
|
{name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||||
|
writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n")
|
||||||
|
configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") {
|
||||||
|
t.Fatalf("profile writer failure = %d, %q", code, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type failingInspectionWriter struct{}
|
type failingInspectionWriter struct{}
|
||||||
|
|
||||||
func (failingInspectionWriter) Write([]byte) (int, error) {
|
func (failingInspectionWriter) Write([]byte) (int, error) {
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ const usage = `Usage:
|
|||||||
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||||
notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name]
|
notarius config effective [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name]
|
||||||
notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer>
|
notarius config explain [--scope notarius|promptkit] --config path/to/config.yml [--config-overlay name] <json-pointer>
|
||||||
|
notarius config profile [--config path/to/config.yml] [--config-overlay name] <profile-id>
|
||||||
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
|
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
|
||||||
`
|
`
|
||||||
|
|
||||||
@@ -1059,6 +1060,8 @@ func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
|
|||||||
return runConfigEffective(args[1:], stdout, stderr, opts)
|
return runConfigEffective(args[1:], stdout, stderr, opts)
|
||||||
case "explain":
|
case "explain":
|
||||||
return runConfigExplain(args[1:], stdout, stderr, opts)
|
return runConfigExplain(args[1:], stdout, stderr, opts)
|
||||||
|
case "profile":
|
||||||
|
return runConfigProfile(args[1:], stdout, stderr, opts)
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
|
fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
|
||||||
writeUsage(stderr)
|
writeUsage(stderr)
|
||||||
|
|||||||
@@ -94,7 +94,6 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
|
|||||||
if i == nil || i.engine == nil {
|
if i == nil || i.engine == nil {
|
||||||
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
|
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
|
||||||
}
|
}
|
||||||
profileID = strings.TrimSpace(profileID)
|
|
||||||
inspection, err := i.engine.InspectProfile(ctx, profileID)
|
inspection, err := i.engine.InspectProfile(ctx, profileID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user