Verify PromptKit smoke command lifecycles
This commit is contained in:
@@ -126,6 +126,16 @@ func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) {
|
||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || output.Len() != 0 {
|
||||
t.Fatalf("empty acquisition outcome = (%#v, %q)", outcome, output.String())
|
||||
}
|
||||
|
||||
app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{}, promptkit.ErrInvalidRequest
|
||||
},
|
||||
}))
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("native semantic error classification = %#v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
type smokeShortWriter struct{}
|
||||
|
||||
266
integrations/ckprompt/smoke_lifecycle_test.go
Normal file
266
integrations/ckprompt/smoke_lifecycle_test.go
Normal file
@@ -0,0 +1,266 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
||||
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestSmokeTestCommandRetainsSingletonValuesAndResetsInvocations(t *testing.T) {
|
||||
type input struct{ campaign string }
|
||||
type received struct {
|
||||
selection ckconfig.Selection
|
||||
request promptkit.SmokeTestRequest
|
||||
input input
|
||||
}
|
||||
campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString,
|
||||
func(target *input, value commandkit.Supplied[string]) { target.campaign = value.Value })
|
||||
var requests []received
|
||||
suite := Suite[input, struct{}]{
|
||||
NewInput: func() input { return input{} },
|
||||
Selectors: []commandkit.Flag[input]{campaignFlag},
|
||||
SmokeTest: func(_ context.Context, request SmokeRequest, value input, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
requests = append(requests, received{selection: request.Selection, request: request.Request, input: value})
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
|
||||
},
|
||||
}
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
|
||||
first := app.Execute(context.Background(), []string{
|
||||
"promptkit", "test", "authentication", "--config=old.yml", "--config=final.yml",
|
||||
"--overlay=old-overlay.yml", "--overlay=final-overlay.yml", "--campaign=old", "--campaign=final",
|
||||
"--backend=old-backend", "--backend=final-backend", "--prompt=old-prompt", "--prompt=final-prompt",
|
||||
"--profile=old-profile", "--profile=final-profile", "--model=old-model", "--model=final-model",
|
||||
"--temperature=0", "--transient-retries=0",
|
||||
}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if first.Kind != commandkit.OutcomeSuccess || len(requests) != 1 {
|
||||
t.Fatalf("first smoke invocation = %#v, requests=%#v", first, requests)
|
||||
}
|
||||
wantSelection := ckconfig.Selection{
|
||||
Config: commandkit.Supplied[string]{Value: "final.yml", Set: true},
|
||||
Overlay: commandkit.Supplied[string]{Value: "final-overlay.yml", Set: true},
|
||||
}
|
||||
got := requests[0]
|
||||
if got.selection != wantSelection || got.input.campaign != "final" || got.request.BackendID != "final-backend" || got.request.PromptID != "final-prompt" || got.request.ProfileID != "final-profile" {
|
||||
t.Fatalf("last supplied values = %#v", got)
|
||||
}
|
||||
if got.request.Execution == nil || got.request.Execution.Model != "final-model" || got.request.Execution.Temperature == nil || *got.request.Execution.Temperature != 0 || got.request.TransientRetries == nil || *got.request.TransientRetries != 0 {
|
||||
t.Fatalf("last execution values = %#v", got.request)
|
||||
}
|
||||
|
||||
second := app.Execute(context.Background(), []string{"promptkit", "test", "reachability"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if second.Kind != commandkit.OutcomeSuccess || len(requests) != 2 {
|
||||
t.Fatalf("second smoke invocation = %#v, requests=%#v", second, requests)
|
||||
}
|
||||
got = requests[1]
|
||||
if got.selection.Config.Set || got.selection.Overlay.Set || got.input.campaign != "" || got.request.BackendID != "" || got.request.PromptID != "" || got.request.ProfileID != "" || got.request.Execution != nil || got.request.TransientRetries != nil {
|
||||
t.Fatalf("values leaked between invocations: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestCommandUsageAndInputFailuresStopProviderWork(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{name: "missing kind", args: []string{"promptkit", "test"}},
|
||||
{name: "extra operand", args: []string{"promptkit", "test", "authentication", "extra"}},
|
||||
{name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--temperature=warm"}},
|
||||
{name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--max-tokens=many"}},
|
||||
{name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--top-p=high"}},
|
||||
{name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--timeout-seconds=soon"}},
|
||||
{name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--transient-retries=several"}},
|
||||
{name: "unsupported format", args: []string{"promptkit", "test", "authentication", "--format=json"}},
|
||||
{name: "unknown flag", args: []string{"promptkit", "test", "authentication", "--unknown=value"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
newInputCalls, checkCalls, newStateCalls, providerCalls := 0, 0, 0, 0
|
||||
suite := Suite[int, struct{}]{
|
||||
NewInput: func() int { newInputCalls++; return 1 },
|
||||
NewState: func() struct{} { newStateCalls++; return struct{}{} },
|
||||
CheckInput: func(ckconfig.Selection, int) error {
|
||||
checkCalls++
|
||||
return nil
|
||||
},
|
||||
SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
providerCalls++
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
|
||||
},
|
||||
}
|
||||
outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), test.args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 || checkCalls != 0 || newStateCalls != 0 || providerCalls != 0 {
|
||||
t.Fatalf("preflight failure = %#v, input/check/state/provider=%d/%d/%d/%d", outcome, newInputCalls, checkCalls, newStateCalls, providerCalls)
|
||||
}
|
||||
if newInputCalls != 0 {
|
||||
t.Fatalf("preflight failure invoked suite input factory %d times", newInputCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
inputErr := errors.New("input rejected")
|
||||
newStateCalls, providerCalls := 0, 0
|
||||
suite := Suite[int, struct{}]{
|
||||
NewInput: func() int { return 1 },
|
||||
NewState: func() struct{} { newStateCalls++; return struct{}{} },
|
||||
CheckInput: func(ckconfig.Selection, int) error {
|
||||
return inputErr
|
||||
},
|
||||
SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
providerCalls++
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
|
||||
},
|
||||
}
|
||||
outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeUsage || !errors.Is(outcome.Cause, inputErr) || newStateCalls != 0 || providerCalls != 0 {
|
||||
t.Fatalf("input-check failure reached state/provider: %#v state=%d provider=%d", outcome, newStateCalls, providerCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestCommandAvailabilityCaptureAndHelpLifecycle(t *testing.T) {
|
||||
type input struct{ campaign string }
|
||||
var originalCalls, replacementCalls, inputCalls, stateCalls int
|
||||
var capturedInput input
|
||||
campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString,
|
||||
func(value *input, supplied commandkit.Supplied[string]) { value.campaign = supplied.Value })
|
||||
replacementFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "season"}, (*string)(nil), parseDefinitionString,
|
||||
func(value *input, supplied commandkit.Supplied[string]) {
|
||||
value.campaign = "replacement:" + supplied.Value
|
||||
})
|
||||
selectors := []commandkit.Flag[input]{campaignFlag}
|
||||
suite := Suite[input, int]{
|
||||
NewInput: func() input { inputCalls++; return input{} },
|
||||
NewState: func() int { stateCalls++; return 1 },
|
||||
Selectors: selectors,
|
||||
SmokeTest: func(_ context.Context, _ SmokeRequest, value input, _ int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
originalCalls++
|
||||
capturedInput = value
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil
|
||||
},
|
||||
}
|
||||
declaration := Command(suite)
|
||||
suite.SmokeTest = func(context.Context, SmokeRequest, input, int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
replacementCalls++
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
|
||||
}
|
||||
selectors[0] = replacementFlag
|
||||
app := newPromptTestApp(t, declaration)
|
||||
var help bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 {
|
||||
t.Fatalf("smoke help lifecycle = (%#v, %q), input/state/providers=%d/%d/%d/%d", outcome, help.String(), inputCalls, stateCalls, originalCalls, replacementCalls)
|
||||
}
|
||||
outcome = app.Execute(context.Background(), []string{"version"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 {
|
||||
t.Fatalf("version lifecycle = %#v, input/state/providers=%d/%d/%d/%d", outcome, inputCalls, stateCalls, originalCalls, replacementCalls)
|
||||
}
|
||||
|
||||
outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--campaign=spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || originalCalls != 1 || replacementCalls != 0 || inputCalls != 1 || stateCalls != 1 || capturedInput.campaign != "spring" {
|
||||
t.Fatalf("captured callback/selector = (%#v, %#v), calls=%d/%d input/state=%d/%d", outcome, capturedInput, originalCalls, replacementCalls, inputCalls, stateCalls)
|
||||
}
|
||||
|
||||
otherApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
|
||||
return commandkit.Acquired[[]promptkit.PromptIdentity]{}, nil
|
||||
},
|
||||
}))
|
||||
help.Reset()
|
||||
outcome = otherApp.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || strings.Contains(help.String(), "\n test ") {
|
||||
t.Fatalf("smoke command installed without callback: (%#v, %q)", outcome, help.String())
|
||||
}
|
||||
|
||||
issueSuite := WithSmokeTesting(Suite[struct{}, struct{}]{
|
||||
InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
|
||||
return commandkit.Acquired[*promptkit.RequirementInspection]{}, nil
|
||||
},
|
||||
}, nil, nil)
|
||||
issueSuite.SmokeTest = nil
|
||||
if _, err := commandkit.New(commandkit.Identity{Name: "example"}, Command(issueSuite)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") {
|
||||
t.Fatalf("narrowed suite lost smoke wiring issue: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) {
|
||||
wantKinds := []promptkit.SmokeTestKind{
|
||||
promptkit.SmokeTestReachability,
|
||||
promptkit.SmokeTestAuthentication,
|
||||
promptkit.SmokeTestTextInference,
|
||||
promptkit.SmokeTestStructuredOutput,
|
||||
}
|
||||
var builtKinds []promptkit.SmokeTestKind
|
||||
loadErr := errors.New("engine unavailable")
|
||||
loads := 0
|
||||
suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
||||
loads++
|
||||
return nil, loadErr
|
||||
}, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) {
|
||||
builtKinds = append(builtKinds, request.Request.Kind)
|
||||
return request.Request, nil
|
||||
})
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || loads != 4 || !reflect.DeepEqual(builtKinds, wantKinds) {
|
||||
t.Fatalf("aggregate builder lifecycle = %#v, built=%#v loads=%d", outcome, builtKinds, loads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestHelpAndVersionLeaveWrappedProviderIdle(t *testing.T) {
|
||||
newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls := 0, 0, 0, 0, 0
|
||||
suite := WithSmokeTesting(Suite[int, int]{
|
||||
NewInput: func() int { newInputCalls++; return 1 },
|
||||
NewState: func() int { newStateCalls++; return 2 },
|
||||
CheckInput: func(ckconfig.Selection, int) error {
|
||||
checkCalls++
|
||||
return nil
|
||||
},
|
||||
}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
||||
loadCalls++
|
||||
return nil, errors.New("loader should be idle")
|
||||
}, func(_ context.Context, request SmokeRequest, _ int, _ int) (promptkit.SmokeTestRequest, error) {
|
||||
buildCalls++
|
||||
return request.Request, nil
|
||||
})
|
||||
app := newPromptTestApp(t, Command(suite))
|
||||
for _, args := range [][]string{{"help", "promptkit"}, {"version"}} {
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess {
|
||||
t.Fatalf("%v outcome = %#v", args, outcome)
|
||||
}
|
||||
}
|
||||
if newInputCalls != 0 || checkCalls != 0 || newStateCalls != 0 || buildCalls != 0 || loadCalls != 0 {
|
||||
t.Fatalf("help/version invoked smoke work: input/check/state/build/load=%d/%d/%d/%d/%d", newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestNilFactoriesSupplyZeroValuesForAggregate(t *testing.T) {
|
||||
checkCalls, providerCalls := 0, 0
|
||||
suite := Suite[struct{ value string }, struct{ count int }]{
|
||||
CheckInput: func(_ ckconfig.Selection, value struct{ value string }) error {
|
||||
checkCalls++
|
||||
if value.value != "" {
|
||||
t.Errorf("nil input factory value = %#v", value)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
SmokeTest: func(_ context.Context, _ SmokeRequest, value struct{ value string }, status struct{ count int }) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
providerCalls++
|
||||
if value.value != "" || status.count != 0 {
|
||||
t.Errorf("nil factories yielded input/state (%#v, %#v)", value, status)
|
||||
}
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
|
||||
},
|
||||
}
|
||||
outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || checkCalls != 1 || providerCalls != 4 {
|
||||
t.Fatalf("nil-factory aggregate = %#v, checks/providers=%d/%d", outcome, checkCalls, providerCalls)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user