Adopt native PromptKit resource inspection
This commit is contained in:
192
integrations/ckprompt/resource_inspection_test.go
Normal file
192
integrations/ckprompt/resource_inspection_test.go
Normal file
@@ -0,0 +1,192 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
promptkit "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func jsonValue(t *testing.T, value any) any {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var result any
|
||||
if err := decoder.Decode(&result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestResourceInspectionsForwardNativeReports(t *testing.T) {
|
||||
zero := 0
|
||||
no := false
|
||||
empty := ""
|
||||
inputs := []promptkit.PromptInputInspection{{Name: "topic", Required: &no}}
|
||||
extra := map[string]any{}
|
||||
prompt := &promptkit.PromptInspection{PromptID: "p", PromptHash: "hash", Description: &empty, Inputs: &inputs, Messages: []promptkit.PromptMessageInspection{{Role: "user", Content: "captured {{.topic}}"}}}
|
||||
profile := &promptkit.ProfileInspection{ProfileID: "p", Temperature: nil, MaxTokens: &zero, APIKeyRequired: &no, ExtraParams: &extra}
|
||||
backend := &promptkit.BackendInspection{BackendID: "b", Endpoint: "https://example.invalid", QueueCapacity: &zero, ExtraParams: &extra}
|
||||
var selections []ckconfig.Selection
|
||||
calls := 0
|
||||
suite := Suite[struct{}, struct{}]{
|
||||
InspectPrompt: func(_ context.Context, r PromptRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
||||
calls++
|
||||
selections = append(selections, r.Selection)
|
||||
if r.ID != "p" {
|
||||
t.Errorf("prompt ID %q", r.ID)
|
||||
}
|
||||
return commandkit.Acquired[*promptkit.PromptInspection]{Value: prompt}, nil
|
||||
},
|
||||
InspectProfile: func(_ context.Context, r ProfileRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
||||
calls++
|
||||
selections = append(selections, r.Selection)
|
||||
if r.ID != "p" {
|
||||
t.Errorf("profile ID %q", r.ID)
|
||||
}
|
||||
return commandkit.Acquired[*promptkit.ProfileInspection]{Value: profile}, nil
|
||||
},
|
||||
InspectBackend: func(_ context.Context, r BackendRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
|
||||
calls++
|
||||
selections = append(selections, r.Selection)
|
||||
if r.ID != "b" {
|
||||
t.Errorf("backend ID %q", r.ID)
|
||||
}
|
||||
return commandkit.Acquired[*promptkit.BackendInspection]{Value: backend}, nil
|
||||
},
|
||||
}
|
||||
app := newPromptTestApp(t, Command(suite), commandkit.Completion("example"))
|
||||
for _, kind := range []struct {
|
||||
name, id string
|
||||
value any
|
||||
}{{"prompt", "p", prompt}, {"profile", "p", profile}, {"backend", "b", backend}} {
|
||||
for _, format := range []string{"text", "yaml", "json"} {
|
||||
t.Run(kind.name+"/"+format, func(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
outcome := app.Execute(t.Context(), []string{"promptkit", "show", kind.name, kind.id, "--config=fixture", "--format=" + format}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("%#v, %s", outcome, out.String())
|
||||
}
|
||||
var got any
|
||||
var err error
|
||||
if format == "json" {
|
||||
d := json.NewDecoder(&out)
|
||||
d.UseNumber()
|
||||
err = d.Decode(&got)
|
||||
} else {
|
||||
got, err = commandkit.DecodeYAMLReport(&out)
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(got, jsonValue(t, kind.value)) {
|
||||
t.Fatalf("got %#v, %v; want %#v", got, err, jsonValue(t, kind.value))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, selection := range selections {
|
||||
if !selection.Config.Set || selection.Config.Value != "fixture" {
|
||||
t.Fatalf("selection %#v", selection)
|
||||
}
|
||||
}
|
||||
before := calls
|
||||
for _, argv := range [][]string{{"promptkit", "show", "backend", "--help"}, {"__complete", "promptkit", "show", "backend", ""}} {
|
||||
var help bytes.Buffer
|
||||
result := app.Execute(t.Context(), argv, commandkit.Streams{Out: &help, Err: io.Discard})
|
||||
if result.Status != 0 || calls != before {
|
||||
t.Fatalf("%v: %#v calls %d", argv, result, calls)
|
||||
}
|
||||
if argv[0] == "promptkit" && !strings.Contains(help.String(), "BACKEND_ID") {
|
||||
t.Fatalf("backend help = %q", help.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendInspectionOptionalAndNativeErrorIdentity(t *testing.T) {
|
||||
calls := 0
|
||||
engine := adapterEngine(t)
|
||||
suite := FromEngine(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
calls++
|
||||
return engine, nil
|
||||
})
|
||||
backendOnly := Suite[struct{}, struct{}]{InspectBackend: suite.InspectBackend}
|
||||
app := newPromptTestApp(t, Command(backendOnly))
|
||||
result := app.Execute(t.Context(), []string{"promptkit", "show", "backend", "missing"}, commandkit.Streams{Err: io.Discard})
|
||||
if !errors.Is(result.Cause, promptkit.ErrBackendNotFound) || result.Kind != commandkit.OutcomeOperational || calls != 1 {
|
||||
t.Fatalf("outcome %#v calls %d", result, calls)
|
||||
}
|
||||
without := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: suite.InspectPrompt}))
|
||||
result = without.Execute(t.Context(), []string{"promptkit", "show", "backend", "missing"}, commandkit.Streams{Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeUsage || calls != 1 {
|
||||
t.Fatalf("outcome %#v calls %d", result, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineInspectionReportsMatchNativeResults(t *testing.T) {
|
||||
t.Setenv("PROMPTKIT_ADAPTER_TEST_KEY", "private-credential-value")
|
||||
engine := adapterEngine(t)
|
||||
calls := 0
|
||||
native := FromEngine(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
calls++
|
||||
return engine, nil
|
||||
})
|
||||
suite := Suite[struct{}, struct{}]{InspectPrompt: native.InspectPrompt, InspectProfile: native.InspectProfile, InspectBackend: native.InspectBackend}
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
ctx := t.Context()
|
||||
prompt, err := engine.InspectPrompt(ctx, "briefing")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile, err := engine.InspectProfile(ctx, "default")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := engine.InspectBackend(ctx, "local")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name, id string
|
||||
want any
|
||||
}{{"prompt", "briefing", prompt}, {"profile", "default", profile}, {"backend", "local", backend}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
result := app.Execute(ctx, []string{"promptkit", "show", tc.name, tc.id, "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||
if bytes.Contains(out.Bytes(), []byte("private-credential-value")) {
|
||||
t.Fatal("resource report exposed a credential value")
|
||||
}
|
||||
var got any
|
||||
decoder := json.NewDecoder(&out)
|
||||
decoder.UseNumber()
|
||||
err := decoder.Decode(&got)
|
||||
if result.Status != 0 || err != nil || !reflect.DeepEqual(got, jsonValue(t, tc.want)) {
|
||||
t.Fatalf("result=%#v got=%#v err=%v want=%#v", result, got, err, jsonValue(t, tc.want))
|
||||
}
|
||||
})
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("loader calls %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendInspectionPreservesDiagnosticsAndCause(t *testing.T) {
|
||||
cause := errors.New("backend lookup failed")
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectBackend: func(context.Context, BackendRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.BackendInspection]{Diagnostics: []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityWarning, "loader warning")}}, cause
|
||||
}}))
|
||||
var diagnostics bytes.Buffer
|
||||
result := app.Execute(t.Context(), []string{"promptkit", "show", "backend", "example", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
||||
if !errors.Is(result.Cause, cause) || !strings.Contains(diagnostics.String(), "loader warning") {
|
||||
t.Fatalf("result=%#v diagnostics=%q", result, diagnostics.String())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user