Adapt inspection to native PromptKit records
This commit is contained in:
@@ -103,6 +103,8 @@ scaffolding, or conversation and coordination features.
|
||||
|
||||
## Stage 3: Adapt existing inspection commands in place
|
||||
|
||||
**Status: Complete.**
|
||||
|
||||
1. In `internal/adapter/cli/inspect.go` call
|
||||
`Engine.InspectPrompt(ctx, id)`. Remove the version flag, config member,
|
||||
usage text, and output field. Leave the command families, format flags,
|
||||
|
||||
@@ -27,11 +27,11 @@ func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := filepath.Join(root, "framework.yml")
|
||||
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||
local:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
|
||||
profiles:
|
||||
protected:
|
||||
- id: protected
|
||||
backend: local
|
||||
model: offline-model
|
||||
api_key_required: true
|
||||
@@ -59,24 +59,21 @@ func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T)
|
||||
if err := os.MkdirAll(importDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit:
|
||||
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `config:
|
||||
imports:
|
||||
- path: catalog/fragment.yml
|
||||
required: true
|
||||
backends:
|
||||
local:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
profiles:
|
||||
selected:
|
||||
backend: local
|
||||
model: selected-model
|
||||
`)
|
||||
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
|
||||
imported:
|
||||
- id: imported
|
||||
model: imported-model
|
||||
`)
|
||||
directory := t.TempDir()
|
||||
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
|
||||
directory:
|
||||
- id: directory
|
||||
model: directory-model
|
||||
`)
|
||||
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
|
||||
@@ -135,14 +132,13 @@ profiles:
|
||||
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
|
||||
unused:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
- id: unused
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||
code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath})
|
||||
@@ -191,57 +187,59 @@ func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors
|
||||
|
||||
func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends:
|
||||
local:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: backend-token-displaced-canary
|
||||
password: backend-password-canary
|
||||
visible_option: backend-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
profiles:
|
||||
base:
|
||||
backend: local
|
||||
model: inherited-model
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: profile-token-displaced-canary
|
||||
password: profile-password-displaced-canary
|
||||
visible_option: profile-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
"team/summary":
|
||||
extends: base
|
||||
model: lower-model
|
||||
prompts:
|
||||
explainable:
|
||||
"1":
|
||||
default_profile: "team/summary"
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
lower := writeFrameworkSourceFile(t, root, "lower.yml", `collections:
|
||||
support:
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: backend-token-displaced-canary
|
||||
password: backend-password-canary
|
||||
visible_option: backend-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
profiles:
|
||||
- id: summary
|
||||
backend: local
|
||||
model: lower-model
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: profile-token-displaced-canary
|
||||
password: profile-password-displaced-canary
|
||||
visible_option: profile-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
prompts:
|
||||
- id: explainable
|
||||
default_profile: support/summary
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
upperRoot := t.TempDir()
|
||||
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends:
|
||||
local:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: backend-token-effective-canary
|
||||
password: backend-password-effective-canary
|
||||
visible_option: backend-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
profiles:
|
||||
"team/summary":
|
||||
model: effective-model
|
||||
extra_params:
|
||||
token: profile-token-effective-canary
|
||||
password: profile-password-effective-canary
|
||||
visible_option: profile-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `collections:
|
||||
support:
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: backend-token-effective-canary
|
||||
password: backend-password-effective-canary
|
||||
visible_option: backend-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
profiles:
|
||||
- id: summary
|
||||
backend: local
|
||||
model: effective-model
|
||||
api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE
|
||||
extra_params:
|
||||
token: profile-token-effective-canary
|
||||
password: profile-password-effective-canary
|
||||
visible_option: profile-visible-value
|
||||
provider.option: dotted-visible-value
|
||||
`)
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper))
|
||||
const secret = "environment-value-must-never-be-emitted"
|
||||
@@ -253,33 +251,33 @@ profiles:
|
||||
unwant []string
|
||||
}{
|
||||
{
|
||||
path: "/backends/local/extra_params/token",
|
||||
path: "/collections/support/backends/0/extra_params/token",
|
||||
unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/backends/local/extra_params/password",
|
||||
path: "/collections/support/backends/0/extra_params/password",
|
||||
unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/profiles/team~1summary/model",
|
||||
want: []string{"effective-model", "inherited-model", "lower-model"},
|
||||
path: "/collections/support/profiles/0/model",
|
||||
want: []string{"effective-model", "lower-model"},
|
||||
unwant: []string{},
|
||||
},
|
||||
{
|
||||
path: "/profiles/team~1summary/extra_params/token",
|
||||
path: "/collections/support/profiles/0/extra_params/token",
|
||||
unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/profiles/team~1summary/extra_params/password",
|
||||
path: "/collections/support/profiles/0/extra_params/password",
|
||||
unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"},
|
||||
},
|
||||
{
|
||||
path: "/profiles/team~1summary/api_key_env",
|
||||
path: "/collections/support/profiles/0/api_key_env",
|
||||
want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"},
|
||||
unwant: []string{secret},
|
||||
},
|
||||
{
|
||||
path: "/profiles/team~1summary/extra_params/provider.option",
|
||||
path: "/collections/support/profiles/0/extra_params/provider.option",
|
||||
want: []string{"dotted-visible-value"},
|
||||
unwant: []string{"provider/option"},
|
||||
},
|
||||
@@ -333,7 +331,7 @@ profiles:
|
||||
}
|
||||
|
||||
profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{
|
||||
"profile", "--config", configPath, "--profile", "team/summary", "--format", "json",
|
||||
"profile", "--config", configPath, "--profile", "support/summary", "--format", "json",
|
||||
})
|
||||
if profileCode != ExitOK || profileError != "" {
|
||||
t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError)
|
||||
@@ -342,14 +340,14 @@ profiles:
|
||||
if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil {
|
||||
t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput)
|
||||
}
|
||||
if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" {
|
||||
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams)
|
||||
if pointerValue(profile.ProfileModelParams.Model) != "effective-model" || pointerValue(profile.ProfileModelParams.ExtraParams)["visible_option"] != "profile-visible-value" {
|
||||
t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.ProfileModelParams)
|
||||
}
|
||||
if strings.Contains(profileOutput, secret) {
|
||||
t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput)
|
||||
}
|
||||
|
||||
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"})
|
||||
code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "support/explainable"})
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr)
|
||||
}
|
||||
|
||||
@@ -51,8 +51,8 @@ func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("inspect profile after later source replacement: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.Endpoint != "http://127.0.0.1:1/v1" || inspection.EffectiveModelParams.Model != "upper-model" || inspection.EffectiveModelParams.Temperature != 0.5 {
|
||||
t.Fatalf("later source did not replace lower values before validation: %+v", inspection.EffectiveModelParams)
|
||||
if pointerValue(inspection.Endpoint) != "http://127.0.0.1:1/v1" || pointerValue(inspection.Model) != "upper-model" || pointerValue(inspection.Temperature) != 0.5 {
|
||||
t.Fatalf("later source did not replace lower values before validation: %+v", inspection)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,12 +14,12 @@ import (
|
||||
)
|
||||
|
||||
type promptInspectionConfig struct {
|
||||
configPath, promptID, promptVersion, schemaDir, outputPath string
|
||||
configOverlay string
|
||||
configExplicit bool
|
||||
outputFormat appformat.OutputFormat
|
||||
overrides []configkit.Override
|
||||
diagnostics []configkit.Diagnostic
|
||||
configPath, promptID, schemaDir, outputPath string
|
||||
configOverlay string
|
||||
configExplicit bool
|
||||
outputFormat appformat.OutputFormat
|
||||
overrides []configkit.Override
|
||||
diagnostics []configkit.Diagnostic
|
||||
}
|
||||
|
||||
type profileInspectionConfig struct {
|
||||
@@ -68,7 +68,7 @@ func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "engine diagnostic write error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID, cfg.promptVersion)
|
||||
inspection, err := engine.InspectPrompt(context.Background(), cfg.promptID)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
@@ -177,7 +177,6 @@ func parsePromptInspectionArgs(args []string) (*promptInspectionConfig, error) {
|
||||
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||
fs.StringVar(&cfg.promptID, "prompt", "", "prompt ID to inspect")
|
||||
fs.StringVar(&cfg.promptVersion, "prompt-version", "", "optional prompt version")
|
||||
rawFormat := ""
|
||||
fs.StringVar(&rawFormat, "format", "", "output format: text or json")
|
||||
fs.StringVar(&cfg.outputPath, "out", "", "optional output file path")
|
||||
|
||||
@@ -734,7 +734,7 @@ func printUsage(w io.Writer) {
|
||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--out path] [--execution-report path] [--timeout 10m] [--run-timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] --prompt ID [--input name=path] [--profile ID] [--session-id ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--reasoning-effort VALUE] [--var k=v] [--format text|json] [--out path] [--timeout 10m] [--transient-retries N]")
|
||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--addr %s] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--prompt-version VERSION] [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
|
||||
fmt.Fprintln(w, " inspect prompt: scriptorium inspect prompt --prompt ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
|
||||
fmt.Fprintln(w, " inspect profile: scriptorium inspect profile --profile ID [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--format text|json] [--out path]")
|
||||
fmt.Fprintln(w, " inspect configuration: scriptorium inspect configuration <validate|show|sources|explain> [--config PATH] [--config-overlay NAME] [--promptkit-source PATH ...] [--schema-dir DIR] [--path JSON_POINTER (required for explain)]")
|
||||
fmt.Fprintln(w, " config: scriptorium config <validate|show|sources|explain> [--config PATH] [--config-overlay NAME] [application overrides] [--path /pointer for explain]")
|
||||
|
||||
@@ -1442,26 +1442,29 @@ func TestRenderCommandRejectsOverlongSessionID(t *testing.T) {
|
||||
func TestInspectPromptCommandFormatsDefinitionWithoutProfileOrGeneration(t *testing.T) {
|
||||
lib := newCLITestLibrary(t)
|
||||
writePromptDefinition(t, lib.promptDir, "inspect.yaml", `prompts:
|
||||
inspect:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
- id: inspect
|
||||
inputs: []
|
||||
messages:
|
||||
- role: user
|
||||
content: "private message"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "inspect", "--format", "json"})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||
}
|
||||
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) {
|
||||
if !strings.Contains(stdout, `"prompt_id": "inspect"`) || !strings.Contains(stdout, `"inputs": []`) || strings.Contains(stdout, "private message") || strings.Contains(stdout, "prompt_version") {
|
||||
t.Fatalf("unexpected inspection output: %s", stdout)
|
||||
}
|
||||
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{"prompt", "--config", lib.configPath, "--promptkit-source", lib.promptDir, "--prompt", "missing"})
|
||||
if code != ExitRuntimeError || stdout != "" || !strings.Contains(stderr, "inspect error") {
|
||||
t.Fatalf("expected failed inspection without output, got code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
if _, err := parsePromptInspectionArgs([]string{"--prompt", "inspect", "--prompt-version", "1"}); err == nil {
|
||||
t.Fatal("inspection accepted removed prompt version flag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectionParsersPreserveExplicitEmptyConfigPath(t *testing.T) {
|
||||
@@ -1821,28 +1824,34 @@ func TestExecutionReportPathValidationAndPreInvocationFailure(t *testing.T) {
|
||||
|
||||
func intPointer(value int) *int { return &value }
|
||||
|
||||
func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||
func pointerValue[T any](value *T) T {
|
||||
if value == nil {
|
||||
var zero T
|
||||
return zero
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func TestPromptkitPromptInspectionSelectsIDsAndContracts(t *testing.T) {
|
||||
fixtureRoot := definitionFixtureRoot(t)
|
||||
configPath := writeDefinitionFixtureConfig(t, fixtureRoot)
|
||||
tests := []struct {
|
||||
promptID string
|
||||
version string
|
||||
format string
|
||||
mode string
|
||||
}{
|
||||
{promptID: "compat.complete", version: "1.0.0", format: "json", mode: "json_schema"},
|
||||
{promptID: "compat.complete", version: "2.0.0", format: "text", mode: "basic"},
|
||||
{promptID: "compat.json", version: "1.0.0", format: "json", mode: "json"},
|
||||
{promptID: "compat.none", version: "1.0.0", format: "text", mode: "none"},
|
||||
{promptID: "compat.complete", format: "json", mode: "json_schema"},
|
||||
{promptID: "compat.complete-text", format: "text", mode: "basic"},
|
||||
{promptID: "compat.json", format: "json", mode: "json"},
|
||||
{promptID: "compat.none", format: "text", mode: "none"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.promptID+"@"+tc.version, func(t *testing.T) {
|
||||
t.Run(tc.promptID, func(t *testing.T) {
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"prompt",
|
||||
"--config", configPath,
|
||||
"--prompt", tc.promptID,
|
||||
"--prompt-version", tc.version,
|
||||
"--format", "json",
|
||||
})
|
||||
if code != ExitOK {
|
||||
@@ -1852,7 +1861,7 @@ func TestPromptkitPromptInspectionSelectsVersionsAndContracts(t *testing.T) {
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatalf("decode prompt inspection: %v\nbody=%s", err, stdout)
|
||||
}
|
||||
if inspection.PromptVersion != tc.version || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
|
||||
if inspection.PromptID != tc.promptID || inspection.OutputContract.Format != tc.format || inspection.OutputContract.ValidationMode != tc.mode {
|
||||
t.Fatalf("unexpected prompt inspection: %+v", inspection)
|
||||
}
|
||||
})
|
||||
@@ -1891,12 +1900,12 @@ func TestPromptkitProfileInspectionResolvesSupportedTargets(t *testing.T) {
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatalf("decode profile inspection: %v\nbody=%s", err, stdout)
|
||||
}
|
||||
params := inspection.EffectiveModelParams
|
||||
if inspection.ProfileID != tc.profileID || params.BackendID != tc.wantBackend || params.Model != tc.wantModel || params.APIKeyEnv != tc.wantAPIKeyEnv {
|
||||
params := inspection.ProfileModelParams
|
||||
if inspection.ProfileID != tc.profileID || pointerValue(params.BackendID) != tc.wantBackend || pointerValue(params.Model) != tc.wantModel || pointerValue(params.APIKeyEnv) != tc.wantAPIKeyEnv {
|
||||
t.Fatalf("unexpected effective profile: %+v", inspection)
|
||||
}
|
||||
if tc.profileID == "custom-derived" {
|
||||
if params.ServiceTier != "flex" || params.ReasoningEffort != "high" || params.TimeoutSeconds != 45 || len(params.ExtraParams) == 0 {
|
||||
if pointerValue(params.ServiceTier) != "flex" || pointerValue(params.ReasoningEffort) != "high" || pointerValue(params.TimeoutSeconds) != 45 || len(pointerValue(params.ExtraParams)) == 0 {
|
||||
t.Fatalf("inherited profile controls were not resolved: %+v", params)
|
||||
}
|
||||
}
|
||||
@@ -1948,8 +1957,8 @@ prompts:
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatalf("decode combined-source profile inspection: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.BackendID != "fixture" || inspection.EffectiveModelParams.Model != "fixture-model" {
|
||||
t.Fatalf("combined source did not provide the selected profile: %+v", inspection.EffectiveModelParams)
|
||||
if pointerValue(inspection.ProfileModelParams.BackendID) != "fixture" || pointerValue(inspection.ProfileModelParams.Model) != "fixture-model" {
|
||||
t.Fatalf("combined source did not provide the selected profile: %+v", inspection.ProfileModelParams)
|
||||
}
|
||||
|
||||
invalidCombined := strings.Replace(combined, `content: "valid unused prompt"`, `content: "{{"`, 1)
|
||||
@@ -1989,11 +1998,11 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.ProfileID != profileID || inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Model == "" {
|
||||
if inspection.ProfileID != profileID || pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) == "" {
|
||||
t.Fatalf("maintained profile did not resolve: %+v", inspection)
|
||||
}
|
||||
if profileID == "promptkit-openrouter-auto" && inspection.EffectiveModelParams.Model != "openrouter/auto" {
|
||||
t.Fatalf("unexpected auto target: %+v", inspection.EffectiveModelParams)
|
||||
if profileID == "promptkit-openrouter-auto" && pointerValue(inspection.ProfileModelParams.Model) != "openrouter/auto" {
|
||||
t.Fatalf("unexpected auto target: %+v", inspection.ProfileModelParams)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -2040,7 +2049,7 @@ func TestMaintainedProfilesResolveAcrossSelectionAndSourcePrecedence(t *testing.
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Model != tc.model {
|
||||
if pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || pointerValue(inspection.ProfileModelParams.Model) != tc.model {
|
||||
t.Fatalf("unexpected effective target: %+v", inspection)
|
||||
}
|
||||
})
|
||||
@@ -2071,7 +2080,7 @@ func TestProfileInspectionHonorsDirectoryPrecedenceOutputAndFailures(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("read profile inspection output: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(output), `"model": "override-model"`) || !strings.Contains(string(output), `"backend_id": ""`) {
|
||||
if !strings.Contains(string(output), `"model": "override-model"`) || strings.Contains(string(output), `"backend_id"`) {
|
||||
t.Fatalf("profile source override was not used: %s", output)
|
||||
}
|
||||
|
||||
@@ -2159,8 +2168,8 @@ func TestConfiguredBackendValidationComesFromPromptkit(t *testing.T) {
|
||||
if err := json.Unmarshal([]byte(stdout), &inspection); err != nil {
|
||||
t.Fatalf("decode built-in profile inspection: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.BackendID != "openrouter" || inspection.EffectiveModelParams.Endpoint != "http://localhost:11434/v1" {
|
||||
t.Fatalf("caller overlay did not update the built-in backend: %+v", inspection.EffectiveModelParams)
|
||||
if pointerValue(inspection.ProfileModelParams.BackendID) != "openrouter" || inspection.ProfileModelParams.Endpoint != nil {
|
||||
t.Fatalf("profile inspection unexpectedly completed backend settings: %+v", inspection.ProfileModelParams)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -11,46 +11,45 @@ import (
|
||||
)
|
||||
|
||||
type PromptInspection struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version"`
|
||||
PromptHash string `json:"prompt_hash"`
|
||||
DefaultProfileID string `json:"default_profile_id"`
|
||||
Inputs []PromptInspectionInput `json:"inputs"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptHash string `json:"prompt_hash"`
|
||||
DefaultProfileID *string `json:"default_profile_id,omitempty"`
|
||||
Inputs *[]PromptInspectionInput `json:"inputs,omitempty"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
}
|
||||
|
||||
type PromptInspectionInput struct {
|
||||
Name string `json:"name"`
|
||||
Required bool `json:"required"`
|
||||
ContentType string `json:"content_type"`
|
||||
Description string `json:"description"`
|
||||
Name string `json:"name"`
|
||||
Required *bool `json:"required,omitempty"`
|
||||
ContentType *string `json:"content_type,omitempty"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
type OutputContract struct {
|
||||
Format string `json:"format"`
|
||||
ValidationMode string `json:"validation_mode"`
|
||||
SchemaPath string `json:"schema_path"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
Format string `json:"format"`
|
||||
ValidationMode string `json:"validation_mode"`
|
||||
SchemaPath *string `json:"schema_path,omitempty"`
|
||||
RepairAttempts *int `json:"repair_attempts,omitempty"`
|
||||
}
|
||||
|
||||
type ProfileInspection struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
EffectiveModelParams ProfileModelParams `json:"effective_model_params"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
ProfileID string `json:"profile_id"`
|
||||
ProfileModelParams ProfileModelParams `json:"profile_model_params"`
|
||||
APIKeyRequired *bool `json:"api_key_required,omitempty"`
|
||||
}
|
||||
|
||||
type ProfileModelParams struct {
|
||||
BackendID string `json:"backend_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
BackendID *string `json:"backend_id,omitempty"`
|
||||
Endpoint *string `json:"endpoint,omitempty"`
|
||||
Model *string `json:"model,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier *string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv *string `json:"api_key_env,omitempty"`
|
||||
ExtraParams *map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
|
||||
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
|
||||
@@ -58,12 +57,15 @@ func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat Outp
|
||||
return nil, errors.New("prompt inspection is nil")
|
||||
}
|
||||
dto := PromptInspection{
|
||||
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
|
||||
Inputs: make([]PromptInspectionInput, len(value.Inputs)),
|
||||
PromptID: value.PromptID, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
|
||||
OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts},
|
||||
}
|
||||
for i, input := range value.Inputs {
|
||||
dto.Inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
|
||||
if value.Inputs != nil {
|
||||
inputs := make([]PromptInspectionInput, len(*value.Inputs))
|
||||
for i, input := range *value.Inputs {
|
||||
inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
|
||||
}
|
||||
dto.Inputs = &inputs
|
||||
}
|
||||
switch outputFormat {
|
||||
case OutputFormatJSON:
|
||||
@@ -74,16 +76,36 @@ func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat Outp
|
||||
return append(data, '\n'), nil
|
||||
case OutputFormatText:
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "prompt_id: %s\nprompt_version: %s\nprompt_hash: %s\ndefault_profile_id: %s\ninputs:", dto.PromptID, dto.PromptVersion, dto.PromptHash, dto.DefaultProfileID)
|
||||
if len(dto.Inputs) == 0 {
|
||||
fmt.Fprintln(&b, " []")
|
||||
} else {
|
||||
fmt.Fprintln(&b)
|
||||
for _, input := range dto.Inputs {
|
||||
fmt.Fprintf(&b, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", input.Name, input.Required, input.ContentType, strconv.Quote(input.Description))
|
||||
fmt.Fprintf(&b, "prompt_id: %s\nprompt_hash: %s\n", dto.PromptID, dto.PromptHash)
|
||||
if dto.DefaultProfileID != nil {
|
||||
fmt.Fprintf(&b, "default_profile_id: %s\n", *dto.DefaultProfileID)
|
||||
}
|
||||
if dto.Inputs != nil {
|
||||
if len(*dto.Inputs) == 0 {
|
||||
fmt.Fprintln(&b, "inputs: []")
|
||||
} else {
|
||||
fmt.Fprintln(&b, "inputs:")
|
||||
for _, input := range *dto.Inputs {
|
||||
fmt.Fprintf(&b, " - name: %s\n", input.Name)
|
||||
if input.Required != nil {
|
||||
fmt.Fprintf(&b, " required: %t\n", *input.Required)
|
||||
}
|
||||
if input.ContentType != nil {
|
||||
fmt.Fprintf(&b, " content_type: %s\n", *input.ContentType)
|
||||
}
|
||||
if input.Description != nil {
|
||||
fmt.Fprintf(&b, " description: %s\n", strconv.Quote(*input.Description))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode, dto.OutputContract.SchemaPath, dto.OutputContract.RepairAttempts)
|
||||
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode)
|
||||
if dto.OutputContract.SchemaPath != nil {
|
||||
fmt.Fprintf(&b, " schema_path: %s\n", *dto.OutputContract.SchemaPath)
|
||||
}
|
||||
if dto.OutputContract.RepairAttempts != nil {
|
||||
fmt.Fprintf(&b, " repair_attempts: %d\n", *dto.OutputContract.RepairAttempts)
|
||||
}
|
||||
return b.Bytes(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
@@ -94,12 +116,11 @@ func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat Ou
|
||||
if value == nil {
|
||||
return nil, errors.New("profile inspection is nil")
|
||||
}
|
||||
target := value.EffectiveModelParams
|
||||
params := map[string]any{}
|
||||
for key, item := range target.ExtraParams {
|
||||
params[key] = item
|
||||
}
|
||||
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, EffectiveModelParams: ProfileModelParams{BackendID: target.BackendID, Endpoint: target.Endpoint, Model: target.Model, Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP, TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort, APIKeyEnv: target.APIKeyEnv, ExtraParams: params}}
|
||||
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, ProfileModelParams: ProfileModelParams{
|
||||
BackendID: value.Backend, Endpoint: value.Endpoint, Model: value.Model, Temperature: value.Temperature,
|
||||
MaxTokens: value.MaxTokens, TopP: value.TopP, TimeoutSeconds: value.TimeoutSeconds, ServiceTier: value.ServiceTier,
|
||||
ReasoningEffort: value.ReasoningEffort, APIKeyEnv: value.APIKeyEnv, ExtraParams: value.ExtraParams,
|
||||
}}
|
||||
switch outputFormat {
|
||||
case OutputFormatJSON:
|
||||
data, err := json.MarshalIndent(dto, "", " ")
|
||||
@@ -109,12 +130,48 @@ func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat Ou
|
||||
return append(data, '\n'), nil
|
||||
case OutputFormatText:
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "profile_id: %s\neffective_model_params:\n backend_id: %s\n endpoint: %s\n model: %s\n temperature: %g\n max_tokens: %d\n top_p: %g\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n extra_params: ", dto.ProfileID, dto.EffectiveModelParams.BackendID, dto.EffectiveModelParams.Endpoint, dto.EffectiveModelParams.Model, dto.EffectiveModelParams.Temperature, dto.EffectiveModelParams.MaxTokens, dto.EffectiveModelParams.TopP, dto.EffectiveModelParams.TimeoutSeconds, dto.EffectiveModelParams.ServiceTier, dto.EffectiveModelParams.ReasoningEffort, dto.EffectiveModelParams.APIKeyEnv)
|
||||
paramsJSON, err := json.Marshal(dto.EffectiveModelParams.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
fmt.Fprintf(&b, "profile_id: %s\nprofile_model_params:\n", dto.ProfileID)
|
||||
p := dto.ProfileModelParams
|
||||
if p.BackendID != nil {
|
||||
fmt.Fprintf(&b, " backend_id: %s\n", *p.BackendID)
|
||||
}
|
||||
if p.Endpoint != nil {
|
||||
fmt.Fprintf(&b, " endpoint: %s\n", *p.Endpoint)
|
||||
}
|
||||
if p.Model != nil {
|
||||
fmt.Fprintf(&b, " model: %s\n", *p.Model)
|
||||
}
|
||||
if p.Temperature != nil {
|
||||
fmt.Fprintf(&b, " temperature: %g\n", *p.Temperature)
|
||||
}
|
||||
if p.MaxTokens != nil {
|
||||
fmt.Fprintf(&b, " max_tokens: %d\n", *p.MaxTokens)
|
||||
}
|
||||
if p.TopP != nil {
|
||||
fmt.Fprintf(&b, " top_p: %g\n", *p.TopP)
|
||||
}
|
||||
if p.TimeoutSeconds != nil {
|
||||
fmt.Fprintf(&b, " timeout_seconds: %d\n", *p.TimeoutSeconds)
|
||||
}
|
||||
if p.ServiceTier != nil {
|
||||
fmt.Fprintf(&b, " service_tier: %s\n", *p.ServiceTier)
|
||||
}
|
||||
if p.ReasoningEffort != nil {
|
||||
fmt.Fprintf(&b, " reasoning_effort: %s\n", *p.ReasoningEffort)
|
||||
}
|
||||
if p.APIKeyEnv != nil {
|
||||
fmt.Fprintf(&b, " api_key_env: %s\n", *p.APIKeyEnv)
|
||||
}
|
||||
if p.ExtraParams != nil {
|
||||
paramsJSON, err := json.Marshal(p.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Fprintf(&b, " extra_params: %s\n", paramsJSON)
|
||||
}
|
||||
if dto.APIKeyRequired != nil {
|
||||
fmt.Fprintf(&b, "api_key_required: %t\n", *dto.APIKeyRequired)
|
||||
}
|
||||
fmt.Fprintf(&b, "%s\napi_key_required: %t\n", paramsJSON, dto.APIKeyRequired)
|
||||
return b.Bytes(), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
|
||||
|
||||
@@ -8,117 +8,88 @@ import (
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestFormatPromptInspectionPreservesDeclaredOrderAndEmptyValues(t *testing.T) {
|
||||
value := &promptkit.PromptInspection{PromptID: "p", PromptVersion: "1", PromptHash: "hash", Inputs: []promptkit.PromptInputDefinition{{Name: "first", Required: true, ContentType: "text/plain", Description: "first input"}, {Name: "second"}}, OutputContract: promptkit.OutputContract{Format: "text", ValidationMode: promptkit.ValidationNone}}
|
||||
text, err := FormatPromptInspection(value, OutputFormatText)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(text), "default_profile_id: \ninputs:\n - name: first") || strings.Index(string(text), "name: first") > strings.Index(string(text), "name: second") {
|
||||
t.Fatalf("unexpected text inspection: %s", text)
|
||||
}
|
||||
jsonOutput, err := FormatPromptInspection(value, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(string(jsonOutput), "\n") || !strings.Contains(string(jsonOutput), `"default_profile_id": ""`) {
|
||||
t.Fatalf("unexpected json inspection: %s", jsonOutput)
|
||||
}
|
||||
}
|
||||
func ptr[T any](value T) *T { return &value }
|
||||
|
||||
func TestFormatPromptInspectionRejectsNil(t *testing.T) {
|
||||
func TestPromptInspectionOptionalFieldsAndDisclosure(t *testing.T) {
|
||||
inputs := []promptkit.PromptInputInspection{{Name: "first", Required: ptr(false), ContentType: ptr(""), Description: ptr("")}, {Name: "second"}}
|
||||
value := &promptkit.PromptInspection{PromptID: "support/p", PromptHash: "hash", DefaultProfileID: ptr(""), Inputs: &inputs,
|
||||
Messages: []promptkit.PromptMessageInspection{{Role: "user", Content: "private message"}}, Description: ptr("private description"), SessionID: ptr("private session"),
|
||||
OutputContract: promptkit.PromptOutputInspection{Format: "text", ValidationMode: promptkit.ValidationNone, SchemaPath: ptr(""), RepairAttempts: ptr(0)}}
|
||||
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
|
||||
output, err := FormatPromptInspection(value, format)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(output)
|
||||
for _, forbidden := range []string{"private message", "private description", "private session", "prompt_version"} {
|
||||
if strings.Contains(s, forbidden) {
|
||||
t.Fatalf("inspection exposed %q: %s", forbidden, s)
|
||||
}
|
||||
}
|
||||
if strings.Index(s, "first") > strings.Index(s, "second") {
|
||||
t.Fatalf("input order changed: %s", s)
|
||||
}
|
||||
if !strings.Contains(s, "default_profile_id") || !strings.Contains(s, "repair_attempts") || !strings.Contains(s, "required") {
|
||||
t.Fatalf("present empty values missing: %s", s)
|
||||
}
|
||||
}
|
||||
value.DefaultProfileID, value.Inputs = nil, nil
|
||||
value.OutputContract.SchemaPath, value.OutputContract.RepairAttempts = nil, nil
|
||||
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
|
||||
output, err := FormatPromptInspection(value, format)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, absent := range []string{"default_profile_id", "inputs", "schema_path", "repair_attempts"} {
|
||||
if strings.Contains(string(output), absent) {
|
||||
t.Fatalf("absent %s rendered: %s", absent, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
empty := []promptkit.PromptInputInspection{}
|
||||
value.Inputs = &empty
|
||||
output, err := FormatPromptInspection(value, OutputFormatJSON)
|
||||
if err != nil || !strings.Contains(string(output), `"inputs": []`) {
|
||||
t.Fatalf("empty inputs lost: %s %v", output, err)
|
||||
}
|
||||
if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil {
|
||||
t.Fatal("expected nil inspection error")
|
||||
t.Fatal("nil prompt accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatProfileInspectionPreservesSafeEffectiveValues(t *testing.T) {
|
||||
const secret = "sentinel-secret-must-not-appear"
|
||||
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
|
||||
|
||||
value := &promptkit.ProfileInspection{
|
||||
ProfileID: "custom-derived",
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
BackendID: "fixture-custom",
|
||||
Endpoint: "http://127.0.0.1:11434/v1",
|
||||
Model: "fixture-model",
|
||||
Temperature: 0.25,
|
||||
MaxTokens: 640,
|
||||
TopP: 0.9,
|
||||
TimeoutSeconds: 45,
|
||||
ServiceTier: "flex",
|
||||
ReasoningEffort: "high",
|
||||
APIKeyEnv: "FIXTURE_PROFILE_API_KEY",
|
||||
ExtraParams: map[string]any{
|
||||
"zeta": true,
|
||||
"alpha": map[string]any{"nested": []any{"first", 2.0}},
|
||||
},
|
||||
},
|
||||
APIKeyRequired: true,
|
||||
func TestProfileInspectionPreservesOnlyPresentValues(t *testing.T) {
|
||||
params := map[string]any{}
|
||||
value := &promptkit.ProfileInspection{ProfileID: "support/fast", Backend: ptr("backend"), Model: ptr(""), Temperature: ptr(0.0), MaxTokens: ptr(0), APIKeyRequired: ptr(false), ExtraParams: ¶ms}
|
||||
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
|
||||
output, err := FormatProfileInspection(value, format)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(output)
|
||||
for _, key := range []string{"profile_model_params", "backend_id", "model", "temperature", "max_tokens", "api_key_required", "extra_params"} {
|
||||
if !strings.Contains(s, key) {
|
||||
t.Fatalf("present %s missing: %s", key, s)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"effective_model_params", "endpoint", "api_key_env", "top_p"} {
|
||||
if strings.Contains(s, key) {
|
||||
t.Fatalf("absent %s appeared: %s", key, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
jsonOutput, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||
output, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondJSONOutput, err := FormatProfileInspection(value, OutputFormatJSON)
|
||||
if err != nil {
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(output, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(jsonOutput) != string(secondJSONOutput) || !strings.HasSuffix(string(jsonOutput), "\n") {
|
||||
t.Fatalf("expected deterministic newline-terminated JSON, got %q", jsonOutput)
|
||||
}
|
||||
if strings.Contains(string(jsonOutput), secret) {
|
||||
t.Fatalf("inspection exposed an environment secret: %s", jsonOutput)
|
||||
}
|
||||
|
||||
var decoded ProfileInspection
|
||||
if err := json.Unmarshal(jsonOutput, &decoded); err != nil {
|
||||
t.Fatalf("decode profile inspection: %v", err)
|
||||
}
|
||||
if decoded.ProfileID != "custom-derived" || decoded.EffectiveModelParams.BackendID != "fixture-custom" {
|
||||
t.Fatalf("unexpected profile identity: %+v", decoded)
|
||||
}
|
||||
if decoded.EffectiveModelParams.APIKeyEnv != "FIXTURE_PROFILE_API_KEY" || !decoded.APIKeyRequired {
|
||||
t.Fatalf("unexpected credential metadata: %+v", decoded)
|
||||
}
|
||||
alpha, ok := decoded.EffectiveModelParams.ExtraParams["alpha"].(map[string]any)
|
||||
if !ok || len(alpha["nested"].([]any)) != 2 {
|
||||
t.Fatalf("nested extra parameters were not preserved: %#v", decoded.EffectiveModelParams.ExtraParams)
|
||||
}
|
||||
|
||||
textOutput, err := FormatProfileInspection(value, OutputFormatText)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
textValue := string(textOutput)
|
||||
if !strings.Contains(textValue, "backend_id: fixture-custom") ||
|
||||
!strings.Contains(textValue, "api_key_env: FIXTURE_PROFILE_API_KEY") ||
|
||||
!strings.Contains(textValue, `extra_params: {"alpha":{"nested":["first",2]},"zeta":true}`) ||
|
||||
!strings.Contains(textValue, "api_key_required: true") {
|
||||
t.Fatalf("unexpected text inspection: %s", textValue)
|
||||
}
|
||||
if strings.Contains(textValue, secret) {
|
||||
t.Fatalf("text inspection exposed an environment secret: %s", textValue)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatProfileInspectionPreservesEmptyBackendAndRejectsNil(t *testing.T) {
|
||||
output, err := FormatProfileInspection(&promptkit.ProfileInspection{
|
||||
ProfileID: "endpoint-only",
|
||||
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||
Endpoint: "http://127.0.0.1:8000/v1",
|
||||
Model: "fixture-model",
|
||||
},
|
||||
}, OutputFormatJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(output), `"backend_id": ""`) ||
|
||||
!strings.Contains(string(output), `"extra_params": {}`) {
|
||||
t.Fatalf("expected explicit empty backend and object extra params, got %s", output)
|
||||
p := decoded["profile_model_params"].(map[string]any)
|
||||
if p["backend_id"] != "backend" || p["model"] != "" || p["temperature"] != float64(0) || decoded["api_key_required"] != false || len(p["extra_params"].(map[string]any)) != 0 {
|
||||
t.Fatalf("wrong values: %#v", decoded)
|
||||
}
|
||||
if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil {
|
||||
t.Fatal("expected nil profile inspection error")
|
||||
t.Fatal("nil profile accepted")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user