Add PromptKit smoke test adapter
This commit is contained in:
@@ -91,7 +91,7 @@ func adapterEngine(t *testing.T) *promptkit.Engine {
|
||||
return adapterEngineFrom(t, "adapter fixture", adapterPrompt)
|
||||
}
|
||||
|
||||
func adapterEngineFrom(t *testing.T, label, configuration string) *promptkit.Engine {
|
||||
func adapterEngineFrom(t *testing.T, label, configuration string, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
inputs, err := promptkit.FilesystemInputs(label, fstest.MapFS{
|
||||
"config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)},
|
||||
@@ -101,13 +101,159 @@ func adapterEngineFrom(t *testing.T, label, configuration string) *promptkit.Eng
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
||||
}})
|
||||
}}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct PromptKit fixture: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
type recordingSmokeClient struct {
|
||||
request promptkit.GenerateRequest
|
||||
response *promptkit.GenerateResponse
|
||||
calls int
|
||||
}
|
||||
|
||||
func (client *recordingSmokeClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.calls++
|
||||
client.request = request
|
||||
return client.response, nil
|
||||
}
|
||||
|
||||
func TestWithSmokeTestingForwardsAuthoritativeNativeRequestAndRetainsReport(t *testing.T) {
|
||||
client := &recordingSmokeClient{
|
||||
response: &promptkit.GenerateResponse{Content: "fixture response"},
|
||||
}
|
||||
engine := adapterEngineFrom(t, "smoke adapter fixture", adapterPrompt, promptkit.WithLLMClient(client))
|
||||
type input struct{ label string }
|
||||
type state struct{ label string }
|
||||
type contextKey struct{}
|
||||
wantContext := context.WithValue(context.Background(), contextKey{}, "smoke invocation")
|
||||
wantInput := input{label: "typed input"}
|
||||
wantState := state{label: "shared state"}
|
||||
wantSelection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "selected.yml", Set: true}}
|
||||
loaderCalls, builderCalls := 0, 0
|
||||
seed := promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "unused", APIKey: "seed credential"}
|
||||
built := promptkit.SmokeTestRequest{
|
||||
Kind: promptkit.SmokeTestTextInference,
|
||||
PromptID: "briefing",
|
||||
APIKey: "builder credential",
|
||||
Execution: &promptkit.ExecutionTargetOverride{Model: "builder-model"},
|
||||
}
|
||||
suite := WithSmokeTesting(Suite[input, state]{},
|
||||
func(ctx context.Context, selection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) {
|
||||
loaderCalls++
|
||||
if ctx.Value(contextKey{}) != "smoke invocation" || selection != wantSelection || gotInput != wantInput || gotState != wantState {
|
||||
t.Errorf("loader args = ctx=%v selection=%#v input=%#v state=%#v", ctx.Value(contextKey{}), selection, gotInput, gotState)
|
||||
}
|
||||
return engine, nil
|
||||
},
|
||||
func(ctx context.Context, request SmokeRequest, gotInput input, gotState state) (promptkit.SmokeTestRequest, error) {
|
||||
builderCalls++
|
||||
if ctx.Value(contextKey{}) != "smoke invocation" || request.Selection != wantSelection || request.Request.Kind != promptkit.SmokeTestAuthentication || gotInput != wantInput || gotState != wantState {
|
||||
t.Errorf("builder args = ctx=%v request=%#v input=%#v state=%#v", ctx.Value(contextKey{}), request, gotInput, gotState)
|
||||
}
|
||||
return built, nil
|
||||
},
|
||||
)
|
||||
if suite.SmokeTest == nil {
|
||||
t.Fatal("WithSmokeTesting did not install a callback")
|
||||
}
|
||||
|
||||
acquired, err := suite.SmokeTest(wantContext, SmokeRequest{Selection: wantSelection, Request: seed}, wantInput, wantState)
|
||||
if err != nil || acquired.Value == nil || len(acquired.Diagnostics) != 0 || loaderCalls != 1 || builderCalls != 1 {
|
||||
t.Fatalf("smoke acquisition = (%#v, %v), loaders/builders=%d/%d", acquired, err, loaderCalls, builderCalls)
|
||||
}
|
||||
if acquired.Value.Kind != promptkit.SmokeTestTextInference || client.calls != 1 || client.request.Target.Model != "builder-model" || client.request.APIKey != "builder credential" {
|
||||
t.Fatalf("native smoke request/report = request %#v report %#v calls=%d", client.request, acquired.Value, client.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithSmokeTestingNilBuilderForwardsSeedAndBuilderFailureSkipsLoader(t *testing.T) {
|
||||
client := &recordingSmokeClient{response: &promptkit.GenerateResponse{Content: "fixture response"}}
|
||||
engine := adapterEngineFrom(t, "smoke adapter fixture", adapterPrompt, promptkit.WithLLMClient(client))
|
||||
seed := promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, PromptID: "briefing", APIKey: "seed credential"}
|
||||
loaderCalls := 0
|
||||
suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
loaderCalls++
|
||||
return engine, nil
|
||||
}, nil)
|
||||
acquired, err := suite.SmokeTest(context.Background(), SmokeRequest{Request: seed}, struct{}{}, struct{}{})
|
||||
if err != nil || acquired.Value == nil || client.calls != 1 || client.request.APIKey != "seed credential" || len(client.request.Prompt.Messages) != 1 {
|
||||
t.Fatalf("nil-builder smoke request = (%#v, %v), client=%#v", acquired, err, client)
|
||||
}
|
||||
acquired, err = suite.SmokeTest(context.Background(), SmokeRequest{Request: promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestKind("unknown")}}, struct{}{}, struct{}{})
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) || acquired.Value != nil {
|
||||
t.Fatalf("native smoke error = (%#v, %v), want invalid request", acquired, err)
|
||||
}
|
||||
|
||||
buildErr := errors.New("request builder failed")
|
||||
loaderCalls = 0
|
||||
suite = WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
loaderCalls++
|
||||
return engine, nil
|
||||
}, func(context.Context, SmokeRequest, struct{}, struct{}) (promptkit.SmokeTestRequest, error) {
|
||||
return promptkit.SmokeTestRequest{}, buildErr
|
||||
})
|
||||
acquired, err = suite.SmokeTest(context.Background(), SmokeRequest{}, struct{}{}, struct{}{})
|
||||
if err != buildErr || acquired.Value != nil || loaderCalls != 0 {
|
||||
t.Fatalf("builder failure acquisition = (%#v, %v), loader calls=%d", acquired, err, loaderCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithSmokeTestingRetainsNilLoaderIssueAndChecksCancellationBeforeBuild(t *testing.T) {
|
||||
sentinel := errors.New("existing suite issue")
|
||||
suite := WithSmokeTesting(Suite[int, int]{issues: []error{sentinel}}, nil, nil)
|
||||
if suite.SmokeTest != nil || len(suite.issues) != 2 || suite.issues[0] != sentinel || !strings.Contains(suite.issues[1].Error(), "engine loader is nil") {
|
||||
t.Fatalf("nil-loader smoke wiring = %#v", suite)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
buildCalls, loadCalls := 0, 0
|
||||
suite = WithSmokeTesting(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
||||
loadCalls++
|
||||
return nil, nil
|
||||
}, func(context.Context, SmokeRequest, int, int) (promptkit.SmokeTestRequest, error) {
|
||||
buildCalls++
|
||||
return promptkit.SmokeTestRequest{}, nil
|
||||
})
|
||||
acquired, err := suite.SmokeTest(ctx, SmokeRequest{}, 0, 0)
|
||||
if !errors.Is(err, context.Canceled) || acquired.Value != nil || buildCalls != 0 || loadCalls != 0 {
|
||||
t.Fatalf("pre-canceled smoke = (%#v, %v), builders/loaders=%d/%d", acquired, err, buildCalls, loadCalls)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
buildCalls, loadCalls = 0, 0
|
||||
suite = WithSmokeTesting(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
||||
loadCalls++
|
||||
return nil, nil
|
||||
}, func(context.Context, SmokeRequest, int, int) (promptkit.SmokeTestRequest, error) {
|
||||
buildCalls++
|
||||
cancel()
|
||||
return promptkit.SmokeTestRequest{}, nil
|
||||
})
|
||||
acquired, err = suite.SmokeTest(ctx, SmokeRequest{}, 0, 0)
|
||||
if !errors.Is(err, context.Canceled) || acquired.Value != nil || buildCalls != 1 || loadCalls != 0 {
|
||||
t.Fatalf("canceled-after-build smoke = (%#v, %v), builders/loaders=%d/%d", acquired, err, buildCalls, loadCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithSmokeTestingRetainsLoaderDiagnosticsAndErrorIdentity(t *testing.T) {
|
||||
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid smoke configuration", []byte("broken: [")))
|
||||
if loadErr == nil {
|
||||
t.Fatal("invalid configuration unexpectedly loaded")
|
||||
}
|
||||
suite := WithSmokeTesting(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
||||
return nil, loadErr
|
||||
}, nil)
|
||||
acquired, err := suite.SmokeTest(context.Background(), SmokeRequest{}, 0, 0)
|
||||
var retained *configkit.LoadError
|
||||
if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || acquired.Value != nil || len(acquired.Diagnostics) == 0 {
|
||||
t.Fatalf("smoke load failure = (%#v, %v)", acquired, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigScopeUsesNativeInspectionFacetsAndInvocationValues(t *testing.T) {
|
||||
type input struct{ token string }
|
||||
type state struct{ token string }
|
||||
|
||||
Reference in New Issue
Block a user