193 lines
7.4 KiB
Go
193 lines
7.4 KiB
Go
package ckprompt
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/ckconfig"
|
|
commandkit "go.maximumdirect.net/commandkit"
|
|
promptkit "go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func jsonValue(t *testing.T, value any) any {
|
|
t.Helper()
|
|
data, err := json.Marshal(value)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
|
decoder.UseNumber()
|
|
var result any
|
|
if err := decoder.Decode(&result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func TestResourceInspectionsForwardNativeReports(t *testing.T) {
|
|
zero := 0
|
|
no := false
|
|
empty := ""
|
|
inputs := []promptkit.PromptInputInspection{{Name: "topic", Required: &no}}
|
|
extra := map[string]any{}
|
|
prompt := &promptkit.PromptInspection{PromptID: "p", PromptHash: "hash", Description: &empty, Inputs: &inputs, Messages: []promptkit.PromptMessageInspection{{Role: "user", Content: "captured {{.topic}}"}}}
|
|
profile := &promptkit.ProfileInspection{ProfileID: "p", Temperature: nil, MaxTokens: &zero, APIKeyRequired: &no, ExtraParams: &extra}
|
|
backend := &promptkit.BackendInspection{BackendID: "b", Endpoint: "https://example.invalid", QueueCapacity: &zero, ExtraParams: &extra}
|
|
var selections []ckconfig.Selection
|
|
calls := 0
|
|
suite := Suite[struct{}, struct{}]{
|
|
InspectPrompt: func(_ context.Context, r PromptRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
|
calls++
|
|
selections = append(selections, r.Selection)
|
|
if r.ID != "p" {
|
|
t.Errorf("prompt ID %q", r.ID)
|
|
}
|
|
return commandkit.Acquired[*promptkit.PromptInspection]{Value: prompt}, nil
|
|
},
|
|
InspectProfile: func(_ context.Context, r ProfileRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
|
calls++
|
|
selections = append(selections, r.Selection)
|
|
if r.ID != "p" {
|
|
t.Errorf("profile ID %q", r.ID)
|
|
}
|
|
return commandkit.Acquired[*promptkit.ProfileInspection]{Value: profile}, nil
|
|
},
|
|
InspectBackend: func(_ context.Context, r BackendRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
|
|
calls++
|
|
selections = append(selections, r.Selection)
|
|
if r.ID != "b" {
|
|
t.Errorf("backend ID %q", r.ID)
|
|
}
|
|
return commandkit.Acquired[*promptkit.BackendInspection]{Value: backend}, nil
|
|
},
|
|
}
|
|
app := newPromptTestApp(t, Command(suite), commandkit.Completion("example"))
|
|
for _, kind := range []struct {
|
|
name, id string
|
|
value any
|
|
}{{"prompt", "p", prompt}, {"profile", "p", profile}, {"backend", "b", backend}} {
|
|
for _, format := range []string{"text", "yaml", "json"} {
|
|
t.Run(kind.name+"/"+format, func(t *testing.T) {
|
|
var out bytes.Buffer
|
|
outcome := app.Execute(t.Context(), []string{"promptkit", "show", kind.name, kind.id, "--config=fixture", "--format=" + format}, commandkit.Streams{Out: &out, Err: io.Discard})
|
|
if outcome.Status != 0 {
|
|
t.Fatalf("%#v, %s", outcome, out.String())
|
|
}
|
|
var got any
|
|
var err error
|
|
if format == "json" {
|
|
d := json.NewDecoder(&out)
|
|
d.UseNumber()
|
|
err = d.Decode(&got)
|
|
} else {
|
|
got, err = commandkit.DecodeYAMLReport(&out)
|
|
}
|
|
if err != nil || !reflect.DeepEqual(got, jsonValue(t, kind.value)) {
|
|
t.Fatalf("got %#v, %v; want %#v", got, err, jsonValue(t, kind.value))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
for _, selection := range selections {
|
|
if !selection.Config.Set || selection.Config.Value != "fixture" {
|
|
t.Fatalf("selection %#v", selection)
|
|
}
|
|
}
|
|
before := calls
|
|
for _, argv := range [][]string{{"promptkit", "show", "backend", "--help"}, {"__complete", "promptkit", "show", "backend", ""}} {
|
|
var help bytes.Buffer
|
|
result := app.Execute(t.Context(), argv, commandkit.Streams{Out: &help, Err: io.Discard})
|
|
if result.Status != 0 || calls != before {
|
|
t.Fatalf("%v: %#v calls %d", argv, result, calls)
|
|
}
|
|
if argv[0] == "promptkit" && !strings.Contains(help.String(), "BACKEND_ID") {
|
|
t.Fatalf("backend help = %q", help.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBackendInspectionOptionalAndNativeErrorIdentity(t *testing.T) {
|
|
calls := 0
|
|
engine := adapterEngine(t)
|
|
suite := FromEngine(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
calls++
|
|
return engine, nil
|
|
})
|
|
backendOnly := Suite[struct{}, struct{}]{InspectBackend: suite.InspectBackend}
|
|
app := newPromptTestApp(t, Command(backendOnly))
|
|
result := app.Execute(t.Context(), []string{"promptkit", "show", "backend", "missing"}, commandkit.Streams{Err: io.Discard})
|
|
if !errors.Is(result.Cause, promptkit.ErrBackendNotFound) || result.Kind != commandkit.OutcomeOperational || calls != 1 {
|
|
t.Fatalf("outcome %#v calls %d", result, calls)
|
|
}
|
|
without := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectPrompt: suite.InspectPrompt}))
|
|
result = without.Execute(t.Context(), []string{"promptkit", "show", "backend", "missing"}, commandkit.Streams{Err: io.Discard})
|
|
if result.Kind != commandkit.OutcomeUsage || calls != 1 {
|
|
t.Fatalf("outcome %#v calls %d", result, calls)
|
|
}
|
|
}
|
|
|
|
func TestEngineInspectionReportsMatchNativeResults(t *testing.T) {
|
|
t.Setenv("PROMPTKIT_ADAPTER_TEST_KEY", "private-credential-value")
|
|
engine := adapterEngine(t)
|
|
calls := 0
|
|
native := FromEngine(func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
calls++
|
|
return engine, nil
|
|
})
|
|
suite := Suite[struct{}, struct{}]{InspectPrompt: native.InspectPrompt, InspectProfile: native.InspectProfile, InspectBackend: native.InspectBackend}
|
|
app := newPromptTestApp(t, Command(suite))
|
|
ctx := t.Context()
|
|
prompt, err := engine.InspectPrompt(ctx, "briefing")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
profile, err := engine.InspectProfile(ctx, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
backend, err := engine.InspectBackend(ctx, "local")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, tc := range []struct {
|
|
name, id string
|
|
want any
|
|
}{{"prompt", "briefing", prompt}, {"profile", "default", profile}, {"backend", "local", backend}} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
var out bytes.Buffer
|
|
result := app.Execute(ctx, []string{"promptkit", "show", tc.name, tc.id, "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
|
if bytes.Contains(out.Bytes(), []byte("private-credential-value")) {
|
|
t.Fatal("resource report exposed a credential value")
|
|
}
|
|
var got any
|
|
decoder := json.NewDecoder(&out)
|
|
decoder.UseNumber()
|
|
err := decoder.Decode(&got)
|
|
if result.Status != 0 || err != nil || !reflect.DeepEqual(got, jsonValue(t, tc.want)) {
|
|
t.Fatalf("result=%#v got=%#v err=%v want=%#v", result, got, err, jsonValue(t, tc.want))
|
|
}
|
|
})
|
|
}
|
|
if calls != 3 {
|
|
t.Fatalf("loader calls %d", calls)
|
|
}
|
|
}
|
|
|
|
func TestBackendInspectionPreservesDiagnosticsAndCause(t *testing.T) {
|
|
cause := errors.New("backend lookup failed")
|
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{InspectBackend: func(context.Context, BackendRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
|
|
return commandkit.Acquired[*promptkit.BackendInspection]{Diagnostics: []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityWarning, "loader warning")}}, cause
|
|
}}))
|
|
var diagnostics bytes.Buffer
|
|
result := app.Execute(t.Context(), []string{"promptkit", "show", "backend", "example", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
|
if !errors.Is(result.Cause, cause) || !strings.Contains(diagnostics.String(), "loader warning") {
|
|
t.Fatalf("result=%#v diagnostics=%q", result, diagnostics.String())
|
|
}
|
|
}
|