Adapt inspection to native PromptKit records

This commit is contained in:
2026-11-18 19:08:41 +00:00
parent 77f168c660
commit de7c866662
8 changed files with 306 additions and 270 deletions

View File

@@ -11,46 +11,45 @@ import (
)
type PromptInspection struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"`
PromptHash string `json:"prompt_hash"`
DefaultProfileID string `json:"default_profile_id"`
Inputs []PromptInspectionInput `json:"inputs"`
OutputContract OutputContract `json:"output_contract"`
PromptID string `json:"prompt_id"`
PromptHash string `json:"prompt_hash"`
DefaultProfileID *string `json:"default_profile_id,omitempty"`
Inputs *[]PromptInspectionInput `json:"inputs,omitempty"`
OutputContract OutputContract `json:"output_contract"`
}
type PromptInspectionInput struct {
Name string `json:"name"`
Required bool `json:"required"`
ContentType string `json:"content_type"`
Description string `json:"description"`
Name string `json:"name"`
Required *bool `json:"required,omitempty"`
ContentType *string `json:"content_type,omitempty"`
Description *string `json:"description,omitempty"`
}
type OutputContract struct {
Format string `json:"format"`
ValidationMode string `json:"validation_mode"`
SchemaPath string `json:"schema_path"`
RepairAttempts int `json:"repair_attempts"`
Format string `json:"format"`
ValidationMode string `json:"validation_mode"`
SchemaPath *string `json:"schema_path,omitempty"`
RepairAttempts *int `json:"repair_attempts,omitempty"`
}
type ProfileInspection struct {
ProfileID string `json:"profile_id"`
EffectiveModelParams ProfileModelParams `json:"effective_model_params"`
APIKeyRequired bool `json:"api_key_required"`
ProfileID string `json:"profile_id"`
ProfileModelParams ProfileModelParams `json:"profile_model_params"`
APIKeyRequired *bool `json:"api_key_required,omitempty"`
}
type ProfileModelParams struct {
BackendID string `json:"backend_id"`
Endpoint string `json:"endpoint"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
TopP float64 `json:"top_p"`
TimeoutSeconds int `json:"timeout_seconds"`
ServiceTier string `json:"service_tier"`
ReasoningEffort string `json:"reasoning_effort"`
APIKeyEnv string `json:"api_key_env"`
ExtraParams map[string]any `json:"extra_params"`
BackendID *string `json:"backend_id,omitempty"`
Endpoint *string `json:"endpoint,omitempty"`
Model *string `json:"model,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier *string `json:"service_tier,omitempty"`
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
APIKeyEnv *string `json:"api_key_env,omitempty"`
ExtraParams *map[string]any `json:"extra_params,omitempty"`
}
func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat OutputFormat) ([]byte, error) {
@@ -58,12 +57,15 @@ func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat Outp
return nil, errors.New("prompt inspection is nil")
}
dto := PromptInspection{
PromptID: value.PromptID, PromptVersion: value.PromptVersion, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
Inputs: make([]PromptInspectionInput, len(value.Inputs)),
PromptID: value.PromptID, PromptHash: value.PromptHash, DefaultProfileID: value.DefaultProfileID,
OutputContract: OutputContract{Format: string(value.OutputContract.Format), ValidationMode: string(value.OutputContract.ValidationMode), SchemaPath: value.OutputContract.SchemaPath, RepairAttempts: value.OutputContract.RepairAttempts},
}
for i, input := range value.Inputs {
dto.Inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
if value.Inputs != nil {
inputs := make([]PromptInspectionInput, len(*value.Inputs))
for i, input := range *value.Inputs {
inputs[i] = PromptInspectionInput{Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description}
}
dto.Inputs = &inputs
}
switch outputFormat {
case OutputFormatJSON:
@@ -74,16 +76,36 @@ func FormatPromptInspection(value *promptkit.PromptInspection, outputFormat Outp
return append(data, '\n'), nil
case OutputFormatText:
var b bytes.Buffer
fmt.Fprintf(&b, "prompt_id: %s\nprompt_version: %s\nprompt_hash: %s\ndefault_profile_id: %s\ninputs:", dto.PromptID, dto.PromptVersion, dto.PromptHash, dto.DefaultProfileID)
if len(dto.Inputs) == 0 {
fmt.Fprintln(&b, " []")
} else {
fmt.Fprintln(&b)
for _, input := range dto.Inputs {
fmt.Fprintf(&b, " - name: %s\n required: %t\n content_type: %s\n description: %s\n", input.Name, input.Required, input.ContentType, strconv.Quote(input.Description))
fmt.Fprintf(&b, "prompt_id: %s\nprompt_hash: %s\n", dto.PromptID, dto.PromptHash)
if dto.DefaultProfileID != nil {
fmt.Fprintf(&b, "default_profile_id: %s\n", *dto.DefaultProfileID)
}
if dto.Inputs != nil {
if len(*dto.Inputs) == 0 {
fmt.Fprintln(&b, "inputs: []")
} else {
fmt.Fprintln(&b, "inputs:")
for _, input := range *dto.Inputs {
fmt.Fprintf(&b, " - name: %s\n", input.Name)
if input.Required != nil {
fmt.Fprintf(&b, " required: %t\n", *input.Required)
}
if input.ContentType != nil {
fmt.Fprintf(&b, " content_type: %s\n", *input.ContentType)
}
if input.Description != nil {
fmt.Fprintf(&b, " description: %s\n", strconv.Quote(*input.Description))
}
}
}
}
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n schema_path: %s\n repair_attempts: %d\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode, dto.OutputContract.SchemaPath, dto.OutputContract.RepairAttempts)
fmt.Fprintf(&b, "output_contract:\n format: %s\n validation_mode: %s\n", dto.OutputContract.Format, dto.OutputContract.ValidationMode)
if dto.OutputContract.SchemaPath != nil {
fmt.Fprintf(&b, " schema_path: %s\n", *dto.OutputContract.SchemaPath)
}
if dto.OutputContract.RepairAttempts != nil {
fmt.Fprintf(&b, " repair_attempts: %d\n", *dto.OutputContract.RepairAttempts)
}
return b.Bytes(), nil
default:
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)
@@ -94,12 +116,11 @@ func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat Ou
if value == nil {
return nil, errors.New("profile inspection is nil")
}
target := value.EffectiveModelParams
params := map[string]any{}
for key, item := range target.ExtraParams {
params[key] = item
}
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, EffectiveModelParams: ProfileModelParams{BackendID: target.BackendID, Endpoint: target.Endpoint, Model: target.Model, Temperature: target.Temperature, MaxTokens: target.MaxTokens, TopP: target.TopP, TimeoutSeconds: target.TimeoutSeconds, ServiceTier: target.ServiceTier, ReasoningEffort: target.ReasoningEffort, APIKeyEnv: target.APIKeyEnv, ExtraParams: params}}
dto := ProfileInspection{ProfileID: value.ProfileID, APIKeyRequired: value.APIKeyRequired, ProfileModelParams: ProfileModelParams{
BackendID: value.Backend, Endpoint: value.Endpoint, Model: value.Model, Temperature: value.Temperature,
MaxTokens: value.MaxTokens, TopP: value.TopP, TimeoutSeconds: value.TimeoutSeconds, ServiceTier: value.ServiceTier,
ReasoningEffort: value.ReasoningEffort, APIKeyEnv: value.APIKeyEnv, ExtraParams: value.ExtraParams,
}}
switch outputFormat {
case OutputFormatJSON:
data, err := json.MarshalIndent(dto, "", " ")
@@ -109,12 +130,48 @@ func FormatProfileInspection(value *promptkit.ProfileInspection, outputFormat Ou
return append(data, '\n'), nil
case OutputFormatText:
var b bytes.Buffer
fmt.Fprintf(&b, "profile_id: %s\neffective_model_params:\n backend_id: %s\n endpoint: %s\n model: %s\n temperature: %g\n max_tokens: %d\n top_p: %g\n timeout_seconds: %d\n service_tier: %s\n reasoning_effort: %s\n api_key_env: %s\n extra_params: ", dto.ProfileID, dto.EffectiveModelParams.BackendID, dto.EffectiveModelParams.Endpoint, dto.EffectiveModelParams.Model, dto.EffectiveModelParams.Temperature, dto.EffectiveModelParams.MaxTokens, dto.EffectiveModelParams.TopP, dto.EffectiveModelParams.TimeoutSeconds, dto.EffectiveModelParams.ServiceTier, dto.EffectiveModelParams.ReasoningEffort, dto.EffectiveModelParams.APIKeyEnv)
paramsJSON, err := json.Marshal(dto.EffectiveModelParams.ExtraParams)
if err != nil {
return nil, err
fmt.Fprintf(&b, "profile_id: %s\nprofile_model_params:\n", dto.ProfileID)
p := dto.ProfileModelParams
if p.BackendID != nil {
fmt.Fprintf(&b, " backend_id: %s\n", *p.BackendID)
}
if p.Endpoint != nil {
fmt.Fprintf(&b, " endpoint: %s\n", *p.Endpoint)
}
if p.Model != nil {
fmt.Fprintf(&b, " model: %s\n", *p.Model)
}
if p.Temperature != nil {
fmt.Fprintf(&b, " temperature: %g\n", *p.Temperature)
}
if p.MaxTokens != nil {
fmt.Fprintf(&b, " max_tokens: %d\n", *p.MaxTokens)
}
if p.TopP != nil {
fmt.Fprintf(&b, " top_p: %g\n", *p.TopP)
}
if p.TimeoutSeconds != nil {
fmt.Fprintf(&b, " timeout_seconds: %d\n", *p.TimeoutSeconds)
}
if p.ServiceTier != nil {
fmt.Fprintf(&b, " service_tier: %s\n", *p.ServiceTier)
}
if p.ReasoningEffort != nil {
fmt.Fprintf(&b, " reasoning_effort: %s\n", *p.ReasoningEffort)
}
if p.APIKeyEnv != nil {
fmt.Fprintf(&b, " api_key_env: %s\n", *p.APIKeyEnv)
}
if p.ExtraParams != nil {
paramsJSON, err := json.Marshal(p.ExtraParams)
if err != nil {
return nil, err
}
fmt.Fprintf(&b, " extra_params: %s\n", paramsJSON)
}
if dto.APIKeyRequired != nil {
fmt.Fprintf(&b, "api_key_required: %t\n", *dto.APIKeyRequired)
}
fmt.Fprintf(&b, "%s\napi_key_required: %t\n", paramsJSON, dto.APIKeyRequired)
return b.Bytes(), nil
default:
return nil, fmt.Errorf("%w: %q", ErrUnknownPreparedRunFormat, outputFormat)

View File

@@ -8,117 +8,88 @@ import (
"go.maximumdirect.net/promptkit"
)
func TestFormatPromptInspectionPreservesDeclaredOrderAndEmptyValues(t *testing.T) {
value := &promptkit.PromptInspection{PromptID: "p", PromptVersion: "1", PromptHash: "hash", Inputs: []promptkit.PromptInputDefinition{{Name: "first", Required: true, ContentType: "text/plain", Description: "first input"}, {Name: "second"}}, OutputContract: promptkit.OutputContract{Format: "text", ValidationMode: promptkit.ValidationNone}}
text, err := FormatPromptInspection(value, OutputFormatText)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(text), "default_profile_id: \ninputs:\n - name: first") || strings.Index(string(text), "name: first") > strings.Index(string(text), "name: second") {
t.Fatalf("unexpected text inspection: %s", text)
}
jsonOutput, err := FormatPromptInspection(value, OutputFormatJSON)
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(string(jsonOutput), "\n") || !strings.Contains(string(jsonOutput), `"default_profile_id": ""`) {
t.Fatalf("unexpected json inspection: %s", jsonOutput)
}
}
func ptr[T any](value T) *T { return &value }
func TestFormatPromptInspectionRejectsNil(t *testing.T) {
func TestPromptInspectionOptionalFieldsAndDisclosure(t *testing.T) {
inputs := []promptkit.PromptInputInspection{{Name: "first", Required: ptr(false), ContentType: ptr(""), Description: ptr("")}, {Name: "second"}}
value := &promptkit.PromptInspection{PromptID: "support/p", PromptHash: "hash", DefaultProfileID: ptr(""), Inputs: &inputs,
Messages: []promptkit.PromptMessageInspection{{Role: "user", Content: "private message"}}, Description: ptr("private description"), SessionID: ptr("private session"),
OutputContract: promptkit.PromptOutputInspection{Format: "text", ValidationMode: promptkit.ValidationNone, SchemaPath: ptr(""), RepairAttempts: ptr(0)}}
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
output, err := FormatPromptInspection(value, format)
if err != nil {
t.Fatal(err)
}
s := string(output)
for _, forbidden := range []string{"private message", "private description", "private session", "prompt_version"} {
if strings.Contains(s, forbidden) {
t.Fatalf("inspection exposed %q: %s", forbidden, s)
}
}
if strings.Index(s, "first") > strings.Index(s, "second") {
t.Fatalf("input order changed: %s", s)
}
if !strings.Contains(s, "default_profile_id") || !strings.Contains(s, "repair_attempts") || !strings.Contains(s, "required") {
t.Fatalf("present empty values missing: %s", s)
}
}
value.DefaultProfileID, value.Inputs = nil, nil
value.OutputContract.SchemaPath, value.OutputContract.RepairAttempts = nil, nil
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
output, err := FormatPromptInspection(value, format)
if err != nil {
t.Fatal(err)
}
for _, absent := range []string{"default_profile_id", "inputs", "schema_path", "repair_attempts"} {
if strings.Contains(string(output), absent) {
t.Fatalf("absent %s rendered: %s", absent, output)
}
}
}
empty := []promptkit.PromptInputInspection{}
value.Inputs = &empty
output, err := FormatPromptInspection(value, OutputFormatJSON)
if err != nil || !strings.Contains(string(output), `"inputs": []`) {
t.Fatalf("empty inputs lost: %s %v", output, err)
}
if _, err := FormatPromptInspection(nil, OutputFormatText); err == nil {
t.Fatal("expected nil inspection error")
t.Fatal("nil prompt accepted")
}
}
func TestFormatProfileInspectionPreservesSafeEffectiveValues(t *testing.T) {
const secret = "sentinel-secret-must-not-appear"
t.Setenv("FIXTURE_PROFILE_API_KEY", secret)
value := &promptkit.ProfileInspection{
ProfileID: "custom-derived",
EffectiveModelParams: promptkit.ExecutionTarget{
BackendID: "fixture-custom",
Endpoint: "http://127.0.0.1:11434/v1",
Model: "fixture-model",
Temperature: 0.25,
MaxTokens: 640,
TopP: 0.9,
TimeoutSeconds: 45,
ServiceTier: "flex",
ReasoningEffort: "high",
APIKeyEnv: "FIXTURE_PROFILE_API_KEY",
ExtraParams: map[string]any{
"zeta": true,
"alpha": map[string]any{"nested": []any{"first", 2.0}},
},
},
APIKeyRequired: true,
func TestProfileInspectionPreservesOnlyPresentValues(t *testing.T) {
params := map[string]any{}
value := &promptkit.ProfileInspection{ProfileID: "support/fast", Backend: ptr("backend"), Model: ptr(""), Temperature: ptr(0.0), MaxTokens: ptr(0), APIKeyRequired: ptr(false), ExtraParams: &params}
for _, format := range []OutputFormat{OutputFormatText, OutputFormatJSON} {
output, err := FormatProfileInspection(value, format)
if err != nil {
t.Fatal(err)
}
s := string(output)
for _, key := range []string{"profile_model_params", "backend_id", "model", "temperature", "max_tokens", "api_key_required", "extra_params"} {
if !strings.Contains(s, key) {
t.Fatalf("present %s missing: %s", key, s)
}
}
for _, key := range []string{"effective_model_params", "endpoint", "api_key_env", "top_p"} {
if strings.Contains(s, key) {
t.Fatalf("absent %s appeared: %s", key, s)
}
}
}
jsonOutput, err := FormatProfileInspection(value, OutputFormatJSON)
output, err := FormatProfileInspection(value, OutputFormatJSON)
if err != nil {
t.Fatal(err)
}
secondJSONOutput, err := FormatProfileInspection(value, OutputFormatJSON)
if err != nil {
var decoded map[string]any
if err := json.Unmarshal(output, &decoded); err != nil {
t.Fatal(err)
}
if string(jsonOutput) != string(secondJSONOutput) || !strings.HasSuffix(string(jsonOutput), "\n") {
t.Fatalf("expected deterministic newline-terminated JSON, got %q", jsonOutput)
}
if strings.Contains(string(jsonOutput), secret) {
t.Fatalf("inspection exposed an environment secret: %s", jsonOutput)
}
var decoded ProfileInspection
if err := json.Unmarshal(jsonOutput, &decoded); err != nil {
t.Fatalf("decode profile inspection: %v", err)
}
if decoded.ProfileID != "custom-derived" || decoded.EffectiveModelParams.BackendID != "fixture-custom" {
t.Fatalf("unexpected profile identity: %+v", decoded)
}
if decoded.EffectiveModelParams.APIKeyEnv != "FIXTURE_PROFILE_API_KEY" || !decoded.APIKeyRequired {
t.Fatalf("unexpected credential metadata: %+v", decoded)
}
alpha, ok := decoded.EffectiveModelParams.ExtraParams["alpha"].(map[string]any)
if !ok || len(alpha["nested"].([]any)) != 2 {
t.Fatalf("nested extra parameters were not preserved: %#v", decoded.EffectiveModelParams.ExtraParams)
}
textOutput, err := FormatProfileInspection(value, OutputFormatText)
if err != nil {
t.Fatal(err)
}
textValue := string(textOutput)
if !strings.Contains(textValue, "backend_id: fixture-custom") ||
!strings.Contains(textValue, "api_key_env: FIXTURE_PROFILE_API_KEY") ||
!strings.Contains(textValue, `extra_params: {"alpha":{"nested":["first",2]},"zeta":true}`) ||
!strings.Contains(textValue, "api_key_required: true") {
t.Fatalf("unexpected text inspection: %s", textValue)
}
if strings.Contains(textValue, secret) {
t.Fatalf("text inspection exposed an environment secret: %s", textValue)
}
}
func TestFormatProfileInspectionPreservesEmptyBackendAndRejectsNil(t *testing.T) {
output, err := FormatProfileInspection(&promptkit.ProfileInspection{
ProfileID: "endpoint-only",
EffectiveModelParams: promptkit.ExecutionTarget{
Endpoint: "http://127.0.0.1:8000/v1",
Model: "fixture-model",
},
}, OutputFormatJSON)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(output), `"backend_id": ""`) ||
!strings.Contains(string(output), `"extra_params": {}`) {
t.Fatalf("expected explicit empty backend and object extra params, got %s", output)
p := decoded["profile_model_params"].(map[string]any)
if p["backend_id"] != "backend" || p["model"] != "" || p["temperature"] != float64(0) || decoded["api_key_required"] != false || len(p["extra_params"].(map[string]any)) != 0 {
t.Fatalf("wrong values: %#v", decoded)
}
if _, err := FormatProfileInspection(nil, OutputFormatText); err == nil {
t.Fatal("expected nil profile inspection error")
t.Fatal("nil profile accepted")
}
}