Files
commandkit/integrations/ckprompt/adapters_test.go

522 lines
21 KiB
Go

package ckprompt
import (
"bytes"
"context"
"errors"
"io"
"reflect"
"strings"
"testing"
"testing/fstest"
commandkit "gitea.maximumdirect.net/eric/CommandKit"
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
promptkit "gitea.maximumdirect.net/eric/promptkit"
)
const adapterPrompt = `prompts:
briefing:
"1":
default_profile: default
inputs:
- name: topic
required: true
content_type: text/plain
description: "Topic"
- name: audience
required: false
content_type: text/plain
description: ""
messages:
- role: user
content: "Hello"
output:
format: text
validation_mode: none
empty:
"1":
messages:
- role: user
content: "Empty inputs"
output:
format: text
validation_mode: none
backends:
local:
endpoint: "https://example.invalid/v1"
profiles:
default:
backend: local
model: example
temperature: 0
max_tokens: 0
api_key_env: PROMPTKIT_ADAPTER_TEST_KEY
extra_params:
token: "caller-owned-value"
nested:
visible: true
null_value: null
sequence: [first, null, 3]
requires-key:
backend: local
model: example
api_key_required: true
`
func adapterEngine(t *testing.T) *promptkit.Engine {
return adapterEngineFrom(t, "adapter fixture", adapterPrompt)
}
func adapterEngineFrom(t *testing.T, label, configuration string) *promptkit.Engine {
t.Helper()
inputs, err := promptkit.FilesystemInputs(label, fstest.MapFS{
"config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)},
}, "config")
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}},
}})
if err != nil {
t.Fatalf("construct PromptKit fixture: %v", err)
}
return engine
}
func TestConfigScopeUsesNativeInspectionFacetsAndInvocationValues(t *testing.T) {
type input struct{ token string }
type state struct{ token string }
type contextKey struct{}
engine := adapterEngine(t)
wantContext := context.WithValue(context.Background(), contextKey{}, "invocation")
wantInput := input{token: "input"}
wantState := state{token: "state"}
loaderCalls := 0
scope := ConfigScope(func(ctx context.Context, selection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) {
loaderCalls++
if ctx.Value(contextKey{}) != "invocation" || !selection.Config.Set || selection.Config.Value != "prompt-config.yml" || gotInput != wantInput || gotState != wantState {
t.Errorf("loader args = ctx=%v selection=%#v input=%#v state=%#v", ctx.Value(contextKey{}), selection, gotInput, gotState)
}
return engine, nil
})
if scope.Name != "promptkit" || len(scope.Operations) != 5 || scope.Acquire == nil {
t.Fatalf("configuration scope = %#v", scope)
}
acquired, err := scope.Acquire(wantContext, ckconfig.Request{
Operation: ckconfig.Show,
Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "prompt-config.yml", Set: true}},
}, wantInput, wantState)
if err != nil || loaderCalls != 1 {
t.Fatalf("scope acquisition = (%#v, %v), calls=%d", acquired, err, loaderCalls)
}
var effective bytes.Buffer
if err := acquired.Value.Effective.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "briefing:") {
t.Fatalf("native effective report = %q, %v", effective.String(), err)
}
if sources := acquired.Value.Sources.Sources(); len(sources) == 0 || sources[0].ID == "" {
t.Fatalf("native sources = %#v", sources)
}
if explanation, ok := acquired.Value.Explanation.Explain(configkit.Path{}); !ok || !explanation.Effective.Present {
t.Fatalf("native root explanation = %#v, found=%t", explanation, ok)
}
}
func TestConfigScopeRetainsNilLoaderIssueAfterCapabilitiesAreCleared(t *testing.T) {
scope := ConfigScope[int, int](nil)
if scope.Acquire != nil {
t.Fatal("nil loader installed an acquisition callback")
}
scope.Operations = nil
app, err := commandkit.New(commandkit.Identity{Name: "example"})
if err != nil {
t.Fatal(err)
}
declaration := ckconfig.Command(ckconfig.Suite[int, int]{Scopes: []ckconfig.Scope[int, int]{scope}})
if err := app.Register(declaration); err == nil || !strings.Contains(err.Error(), "PromptKit configuration loader is nil") {
t.Fatalf("narrowed nil-loader scope registered: %v", err)
}
}
func TestConfigScopeCompareUsesNativeDiffAndLoadsBothEqualSelections(t *testing.T) {
leftEngine := adapterEngine(t)
rightConfig := strings.Replace(adapterPrompt, `content: "Hello"`, `content: "Goodbye"`, 1)
if rightConfig == adapterPrompt {
t.Fatal("comparison fixture did not change the prompt content")
}
rightEngine := adapterEngineFrom(t, "right comparison fixture", rightConfig)
type input struct{ label string }
type call struct {
selection ckconfig.Selection
input input
state *int
}
var calls []call
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, gotInput input, state *int) (*promptkit.Engine, error) {
calls = append(calls, call{selection: selection, input: gotInput, state: state})
if gotInput.label == "before" {
return leftEngine, nil
}
return rightEngine, nil
})
if scope.Compare == nil || len(scope.Operations) != 5 || scope.Operations[4] != ckconfig.Diff {
t.Fatalf("configuration scope does not declare comparison: %#v", scope)
}
selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "same-selection.yml", Set: true}}
state := new(int)
request := ckconfig.ComparisonRequest[input]{
Left: ckconfig.ComparisonSide[input]{Selection: selection, Input: input{label: "before"}},
Right: ckconfig.ComparisonSide[input]{Selection: selection, Input: input{label: "after"}},
}
leftInspection, err := leftEngine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
rightInspection, err := rightEngine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
want, err := leftInspection.Diff(rightInspection)
if err != nil {
t.Fatal(err)
}
if len(want) == 0 {
t.Fatal("native PromptKit fixture comparison produced no differences")
}
got, err := scope.Compare(context.Background(), request, state)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got.Value, want) {
t.Fatalf("adapter differences = %#v, want native PromptKit result %#v", got.Value, want)
}
if len(calls) != 2 || calls[0].selection != selection || calls[1].selection != selection || calls[0].input.label != "before" || calls[1].input.label != "after" || calls[0].state != state || calls[1].state != state {
t.Fatalf("loader calls = %#v", calls)
}
}
func TestConfigScopeCompareStopsAfterLeftLoadFailure(t *testing.T) {
loadErr := errors.New("left engine load failed")
rightCalled := false
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) {
if selection.Config.Value == "left" {
return nil, loadErr
}
rightCalled = true
return adapterEngine(t), nil
})
request := ckconfig.ComparisonRequest[struct{}]{
Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}},
Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}},
}
acquired, err := scope.Compare(context.Background(), request, struct{}{})
if !errors.Is(err, loadErr) || rightCalled || acquired.Value != nil {
t.Fatalf("left failure = (%#v, %v), right called=%t", acquired, err, rightCalled)
}
}
func TestConfigScopeCompareRetainsRightLoadDiagnosticsAndErrorIdentity(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid right prompt config", []byte("broken: [")))
if loadErr == nil {
t.Fatal("invalid right configuration unexpectedly loaded")
}
leftEngine := adapterEngine(t)
var calls []string
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) {
calls = append(calls, selection.Config.Value)
if selection.Config.Value == "left" {
return leftEngine, nil
}
return nil, loadErr
})
request := ckconfig.ComparisonRequest[struct{}]{
Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}},
Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}},
}
acquired, err := scope.Compare(context.Background(), request, struct{}{})
var retained *configkit.LoadError
if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(calls) != 2 || len(acquired.Diagnostics) == 0 {
t.Fatalf("right failure = (%#v, %v), calls=%#v", acquired, err, calls)
}
var output bytes.Buffer
if writeErr := acquired.Diagnostics[0].Write(&output); writeErr != nil || !strings.Contains(output.String(), `Configuration scope "promptkit" (right):`) {
t.Fatalf("right diagnostic = %q, %v", output.String(), writeErr)
}
}
func TestConfigScopeCompareCancellationAfterLeftLoadStopsRight(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
leftEngine := adapterEngine(t)
rightCalled := false
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _ struct{}, _ struct{}) (*promptkit.Engine, error) {
if selection.Config.Value == "left" {
cancel()
return leftEngine, nil
}
rightCalled = true
return adapterEngine(t), nil
})
request := ckconfig.ComparisonRequest[struct{}]{
Left: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "left", Set: true}}},
Right: ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "right", Set: true}}},
}
acquired, err := scope.Compare(ctx, request, struct{}{})
if !errors.Is(err, context.Canceled) || rightCalled || acquired.Value != nil {
t.Fatalf("cancellation = (%#v, %v), right called=%t", acquired, err, rightCalled)
}
}
func TestFromEnginePassesRequestsAndRetainsNilLoaderIssue(t *testing.T) {
type input struct{ value string }
type state struct{ value string }
type contextKey struct{}
engine := adapterEngine(t)
wantContext := context.WithValue(context.Background(), contextKey{}, "prompt invocation")
wantInput := input{value: "typed input"}
wantState := state{value: "shared state"}
loaderCalls := 0
suite := FromEngine(func(ctx context.Context, selection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) {
loaderCalls++
if ctx.Value(contextKey{}) != "prompt invocation" || !selection.Overlay.Set || selection.Overlay.Value != "overlay.yml" || gotInput != wantInput || gotState != wantState {
t.Errorf("loader args = ctx=%v selection=%#v input=%#v state=%#v", ctx.Value(contextKey{}), selection, gotInput, gotState)
}
return engine, nil
})
selection := ckconfig.Selection{Overlay: commandkit.Supplied[string]{Value: "overlay.yml", Set: true}}
version := commandkit.Supplied[string]{Value: "1", Set: true}
prompt, err := suite.InspectPrompt(wantContext, PromptRequest{Selection: selection, ID: "briefing", Version: version}, wantInput, wantState)
if err != nil || prompt.Value == nil || prompt.Value.PromptID != "briefing" || prompt.Value.PromptVersion != "1" || loaderCalls != 1 {
t.Fatalf("prompt inspection = (%#v, %v), calls=%d", prompt, err, loaderCalls)
}
defaultPrompt, err := suite.InspectPrompt(wantContext, PromptRequest{Selection: selection, ID: "briefing"}, wantInput, wantState)
if err != nil || defaultPrompt.Value == nil || defaultPrompt.Value.PromptVersion != "1" || loaderCalls != 2 {
t.Fatalf("empty version default = (%#v, %v), calls=%d", defaultPrompt, err, loaderCalls)
}
profile, err := suite.InspectProfile(wantContext, ProfileRequest{Selection: selection, ID: "default"}, wantInput, wantState)
if err != nil || profile.Value == nil || profile.Value.ProfileID != "default" || loaderCalls != 3 {
t.Fatalf("profile inspection = (%#v, %v), calls=%d", profile, err, loaderCalls)
}
if version.Set != true || version.Value != "1" {
t.Fatalf("custom callback request lost version presence: %#v", version)
}
empty := FromEngine[int, int](nil)
empty.ListPrompts = nil
empty.ListProfiles = nil
empty.InspectPrompt = nil
empty.InspectProfile = nil
if len(empty.issues) != 1 || !strings.Contains(empty.issues[0].Error(), "PromptKit engine loader is nil") {
t.Fatalf("nil-loader issue was lost when callbacks were cleared: %#v", empty.issues)
}
}
func TestFromEngineListsNativeCatalogsWithInvocationValues(t *testing.T) {
type input struct{ label string }
type state struct{ label string }
type contextKey struct{}
engine := adapterEngine(t)
wantContext := context.WithValue(context.Background(), contextKey{}, "catalog invocation")
wantSelection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "selected.yml", Set: true}}
wantInput := input{label: "typed input"}
wantState := state{label: "shared state"}
loaderCalls := 0
suite := FromEngine(func(ctx context.Context, selection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) {
loaderCalls++
if ctx.Value(contextKey{}) != "catalog invocation" || selection != wantSelection || gotInput != wantInput || gotState != wantState {
t.Errorf("loader args = ctx=%v selection=%#v input=%#v state=%#v", ctx.Value(contextKey{}), selection, gotInput, gotState)
}
return engine, nil
})
if suite.ListPrompts == nil || suite.ListProfiles == nil {
t.Fatalf("FromEngine did not supply both catalog callbacks: %#v", suite)
}
wantPrompts, err := engine.ListPrompts(wantContext)
if err != nil {
t.Fatal(err)
}
wantProfiles, err := engine.ListProfiles(wantContext)
if err != nil {
t.Fatal(err)
}
prompts, err := suite.ListPrompts(wantContext, wantSelection, wantInput, wantState)
if err != nil || !reflect.DeepEqual(prompts.Value, wantPrompts) || len(prompts.Diagnostics) != 0 || loaderCalls != 1 {
t.Fatalf("prompt catalog = (%#v, %v), calls=%d, want %#v", prompts, err, loaderCalls, wantPrompts)
}
profiles, err := suite.ListProfiles(wantContext, wantSelection, wantInput, wantState)
if err != nil || !reflect.DeepEqual(profiles.Value, wantProfiles) || len(profiles.Diagnostics) != 0 || loaderCalls != 2 {
t.Fatalf("profile catalog = (%#v, %v), calls=%d, want %#v", profiles, err, loaderCalls, wantProfiles)
}
}
func TestFromEngineCatalogsRetainLoadDiagnosticsAndErrorIdentity(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid catalog configuration", []byte("broken: [")))
if loadErr == nil {
t.Fatal("invalid configuration unexpectedly loaded")
}
suite := FromEngine[int, int](func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
return nil, loadErr
})
for _, test := range []struct {
name string
call func() (int, error)
}{
{name: "prompts", call: func() (int, error) {
acquired, err := suite.ListPrompts(context.Background(), ckconfig.Selection{}, 0, 0)
var retained *configkit.LoadError
if !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 {
t.Errorf("prompt load failure = (%#v, %v)", acquired, err)
}
return len(acquired.Diagnostics), err
}},
{name: "profiles", call: func() (int, error) {
acquired, err := suite.ListProfiles(context.Background(), ckconfig.Selection{}, 0, 0)
var retained *configkit.LoadError
if !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 {
t.Errorf("profile load failure = (%#v, %v)", acquired, err)
}
return len(acquired.Diagnostics), err
}},
} {
t.Run(test.name, func(t *testing.T) {
diagnosticCount, err := test.call()
if !errors.Is(err, loadErr) || diagnosticCount == 0 {
t.Fatalf("catalog failure = (%d diagnostics, %v)", diagnosticCount, err)
}
})
}
}
func TestFromEngineCatalogsHonorCancellationAfterLoad(t *testing.T) {
engine := adapterEngine(t)
loaderCalls := 0
for _, test := range []struct {
name string
call func(Suite[int, int], context.Context) error
}{
{name: "prompts", call: func(suite Suite[int, int], ctx context.Context) error {
_, err := suite.ListPrompts(ctx, ckconfig.Selection{}, 0, 0)
return err
}},
{name: "profiles", call: func(suite Suite[int, int], ctx context.Context) error {
_, err := suite.ListProfiles(ctx, ckconfig.Selection{}, 0, 0)
return err
}},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancelSuite := FromEngine(func(_ context.Context, _ ckconfig.Selection, _, _ int) (*promptkit.Engine, error) {
loaderCalls++
cancel()
return engine, nil
})
if err := test.call(cancelSuite, ctx); !errors.Is(err, context.Canceled) {
t.Fatalf("catalog cancellation = %v", err)
}
})
}
if loaderCalls != 2 {
t.Fatalf("catalog loaders called = %d, want one per provider", loaderCalls)
}
}
func TestAdapterAcquisitionPreservesLoadAndContextErrors(t *testing.T) {
loadErr := errors.New("loader failure")
suite := FromEngine[int, int](func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
return nil, loadErr
})
acquired, err := suite.InspectProfile(context.Background(), ProfileRequest{ID: "default"}, 0, 0)
if err != loadErr || len(acquired.Diagnostics) != 0 {
t.Fatalf("loader failure = (%#v, %v), want original error", acquired, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
engine := adapterEngine(t)
called := false
suite = FromEngine[int, int](func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
called = true
return engine, nil
})
acquired, err = suite.InspectProfile(ctx, ProfileRequest{ID: "default"}, 0, 0)
if !errors.Is(err, context.Canceled) || acquired.Value != nil || !called {
t.Fatalf("cancelled profile acquisition = (%#v, %v), loader called=%t", acquired, err, called)
}
}
func TestFromEngineRetainsConfigKitLoadDiagnostics(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid PromptKit config", []byte("broken: [")))
if loadErr == nil {
t.Fatal("invalid configuration unexpectedly loaded")
}
suite := FromEngine[int, int](func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
return nil, loadErr
})
acquired, err := suite.InspectProfile(context.Background(), ProfileRequest{ID: "default"}, 0, 0)
var retained *configkit.LoadError
if !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 {
t.Fatalf("load failure = (%#v, %v), want original ConfigKit error and diagnostics", acquired, err)
}
if acquired.Diagnostics[0].Severity != commandkit.SeverityError {
t.Fatalf("load diagnostics = %#v, want error severity", acquired.Diagnostics)
}
}
func TestConfigScopeHonorsCancellationAfterLoad(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
engine := adapterEngine(t)
scope := ConfigScope(func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
cancel()
return engine, nil
})
acquired, err := scope.Acquire(ctx, ckconfig.Request{Operation: ckconfig.Show}, 0, 0)
if !errors.Is(err, context.Canceled) || acquired.Value.Effective != nil {
t.Fatalf("cancelled scope acquisition = (%#v, %v), want cancellation and no value", acquired, err)
}
}
func TestPromptKitDiagnosticsRemainDeferredAndPropagateWriterErrors(t *testing.T) {
writeErr := errors.New("diagnostic writer failed")
sources := []configkit.SourceInfo{{ID: "source"}}
records := []configkit.Diagnostic{{
Severity: configkit.SeverityWarning,
Message: "safe warning",
}}
diagnostics := promptKitDiagnostics("promptkit", "", sources, records)
if len(diagnostics) != 1 || diagnostics[0].Severity != commandkit.SeverityWarning {
t.Fatalf("diagnostics = %#v", diagnostics)
}
if err := diagnostics[0].Write(failingDiagnosticWriter{err: writeErr}); !errors.Is(err, writeErr) {
t.Fatalf("diagnostic write error = %v", err)
}
var output bytes.Buffer
if err := diagnostics[0].Write(&output); err != nil || !strings.Contains(output.String(), `Configuration scope "promptkit":`) || !strings.Contains(output.String(), "safe warning") || !strings.Contains(output.String(), `"source"`) {
t.Fatalf("diagnostic text = %q, %v", output.String(), err)
}
sideDiagnostics := promptKitDiagnostics("promptkit", "left", sources, records)
output.Reset()
if err := sideDiagnostics[0].Write(&output); err != nil || !strings.Contains(output.String(), `Configuration scope "promptkit" (left):`) {
t.Fatalf("comparison-side diagnostic text = %q, %v", output.String(), err)
}
}
type failingDiagnosticWriter struct{ err error }
func (writer failingDiagnosticWriter) Write([]byte) (int, error) { return 0, writer.err }
var _ io.Writer = failingDiagnosticWriter{}