Add resolved PromptKit profile inspection
This commit is contained in:
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
@@ -372,6 +373,133 @@ func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) {
|
||||
t.Parallel()
|
||||
root := t.TempDir()
|
||||
profilePath := filepath.Join(root, "profiles.yml")
|
||||
writeConfigKitFile(t, profilePath, `backends:
|
||||
managed:
|
||||
endpoint: https://provider.example.invalid/v1
|
||||
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||
profiles:
|
||||
inherited:
|
||||
backend: managed
|
||||
model: inherited-model
|
||||
temperature: 0.4
|
||||
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
||||
api_key_required: true
|
||||
extra_params:
|
||||
Authorization: private-authorization
|
||||
ordinary: visible-value
|
||||
selected:
|
||||
extends: inherited
|
||||
temperature: 0
|
||||
extra_params: null
|
||||
endpoint-only:
|
||||
endpoint: https://endpoint.example.invalid/v1
|
||||
model: endpoint-model
|
||||
api_key_required: false
|
||||
" literal profile ":
|
||||
endpoint: https://literal.example.invalid/v1
|
||||
model: literal-model
|
||||
`)
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n")
|
||||
assets, err := productionPromptAssets()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lookupCalls := 0
|
||||
factoryCalls := 0
|
||||
opts := commandContractOptions(t)
|
||||
opts.promptKitAssets = assets
|
||||
opts.LookupEnv = func(string) (string, bool) {
|
||||
lookupCalls++
|
||||
return "", false
|
||||
}
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, errors.New("profile inspection must not construct an LLM client")
|
||||
}
|
||||
|
||||
type profileReport struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
Target struct {
|
||||
BackendID string `json:"backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
} `json:"effective_model_params"`
|
||||
}
|
||||
run := func(profileID string) (int, string, string, profileReport) {
|
||||
t.Helper()
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts)
|
||||
var report profileReport
|
||||
if code == 0 {
|
||||
if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 {
|
||||
t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String())
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||
t.Fatalf("decode profile JSON %q: %v", stdout.String(), err)
|
||||
}
|
||||
}
|
||||
return code, stdout.String(), stderr.String(), report
|
||||
}
|
||||
code, output, diagnostic, selected := run("selected")
|
||||
if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil {
|
||||
t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected)
|
||||
}
|
||||
code, output, diagnostic, inherited := run("inherited")
|
||||
if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") {
|
||||
t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited)
|
||||
}
|
||||
code, output, diagnostic, endpointOnly := run("endpoint-only")
|
||||
if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" {
|
||||
t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly)
|
||||
}
|
||||
code, output, diagnostic, literal := run(" literal profile ")
|
||||
if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" {
|
||||
t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal)
|
||||
}
|
||||
if lookupCalls != 0 || factoryCalls != 0 {
|
||||
t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
configPath := writeCommandConfig(t, "session", "other")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
want int
|
||||
}{
|
||||
{name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2},
|
||||
{name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2},
|
||||
{name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2},
|
||||
{name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||
writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n")
|
||||
configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
||||
var stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") {
|
||||
t.Fatalf("profile writer failure = %d, %q", code, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
type failingInspectionWriter struct{}
|
||||
|
||||
func (failingInspectionWriter) Write([]byte) (int, error) {
|
||||
|
||||
Reference in New Issue
Block a user