package ckprompt import ( "bytes" "context" "errors" "io" "reflect" "strings" "testing" commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" promptkit "gitea.maximumdirect.net/eric/promptkit" ) func TestSmokeTestCommandRetainsSingletonValuesAndResetsInvocations(t *testing.T) { type input struct{ campaign string } type received struct { selection ckconfig.Selection request promptkit.SmokeTestRequest input input } campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString, func(target *input, value commandkit.Supplied[string]) { target.campaign = value.Value }) var requests []received suite := Suite[input, struct{}]{ NewInput: func() input { return input{} }, Selectors: []commandkit.Flag[input]{campaignFlag}, SmokeTest: func(_ context.Context, request SmokeRequest, value input, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { requests = append(requests, received{selection: request.Selection, request: request.Request, input: value}) return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil }, } app := newPromptTestApp(t, Command(suite)) first := app.Execute(context.Background(), []string{ "promptkit", "test", "authentication", "--config=old.yml", "--config=final.yml", "--overlay=old-overlay.yml", "--overlay=final-overlay.yml", "--campaign=old", "--campaign=final", "--backend=old-backend", "--backend=final-backend", "--prompt=old-prompt", "--prompt=final-prompt", "--profile=old-profile", "--profile=final-profile", "--model=old-model", "--model=final-model", "--temperature=0", "--transient-retries=0", }, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if first.Kind != commandkit.OutcomeSuccess || len(requests) != 1 { t.Fatalf("first smoke invocation = %#v, requests=%#v", first, requests) } wantSelection := ckconfig.Selection{ Config: commandkit.Supplied[string]{Value: "final.yml", Set: true}, Overlay: commandkit.Supplied[string]{Value: "final-overlay.yml", Set: true}, } got := requests[0] if got.selection != wantSelection || got.input.campaign != "final" || got.request.BackendID != "final-backend" || got.request.PromptID != "final-prompt" || got.request.ProfileID != "final-profile" { t.Fatalf("last supplied values = %#v", got) } if got.request.Execution == nil || got.request.Execution.Model != "final-model" || got.request.Execution.Temperature == nil || *got.request.Execution.Temperature != 0 || got.request.TransientRetries == nil || *got.request.TransientRetries != 0 { t.Fatalf("last execution values = %#v", got.request) } second := app.Execute(context.Background(), []string{"promptkit", "test", "reachability"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if second.Kind != commandkit.OutcomeSuccess || len(requests) != 2 { t.Fatalf("second smoke invocation = %#v, requests=%#v", second, requests) } got = requests[1] if got.selection.Config.Set || got.selection.Overlay.Set || got.input.campaign != "" || got.request.BackendID != "" || got.request.PromptID != "" || got.request.ProfileID != "" || got.request.Execution != nil || got.request.TransientRetries != nil { t.Fatalf("values leaked between invocations: %#v", got) } } func TestSmokeTestCommandUsageAndInputFailuresStopProviderWork(t *testing.T) { for _, test := range []struct { name string args []string }{ {name: "missing kind", args: []string{"promptkit", "test"}}, {name: "extra operand", args: []string{"promptkit", "test", "authentication", "extra"}}, {name: "malformed temperature", args: []string{"promptkit", "test", "authentication", "--temperature=warm"}}, {name: "malformed token count", args: []string{"promptkit", "test", "authentication", "--max-tokens=many"}}, {name: "malformed top-p", args: []string{"promptkit", "test", "authentication", "--top-p=high"}}, {name: "malformed timeout", args: []string{"promptkit", "test", "authentication", "--timeout-seconds=soon"}}, {name: "malformed retries", args: []string{"promptkit", "test", "authentication", "--transient-retries=several"}}, {name: "unsupported format", args: []string{"promptkit", "test", "authentication", "--format=json"}}, {name: "unknown flag", args: []string{"promptkit", "test", "authentication", "--unknown=value"}}, } { t.Run(test.name, func(t *testing.T) { newInputCalls, checkCalls, newStateCalls, providerCalls := 0, 0, 0, 0 suite := Suite[int, struct{}]{ NewInput: func() int { newInputCalls++; return 1 }, NewState: func() struct{} { newStateCalls++; return struct{}{} }, CheckInput: func(ckconfig.Selection, int) error { checkCalls++ return nil }, SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { providerCalls++ return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil }, } outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), test.args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 || checkCalls != 0 || newStateCalls != 0 || providerCalls != 0 { t.Fatalf("preflight failure = %#v, input/check/state/provider=%d/%d/%d/%d", outcome, newInputCalls, checkCalls, newStateCalls, providerCalls) } if newInputCalls != 0 { t.Fatalf("preflight failure invoked suite input factory %d times", newInputCalls) } }) } inputErr := errors.New("input rejected") newStateCalls, providerCalls := 0, 0 suite := Suite[int, struct{}]{ NewInput: func() int { return 1 }, NewState: func() struct{} { newStateCalls++; return struct{}{} }, CheckInput: func(ckconfig.Selection, int) error { return inputErr }, SmokeTest: func(context.Context, SmokeRequest, int, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { providerCalls++ return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil }, } outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeUsage || !errors.Is(outcome.Cause, inputErr) || newStateCalls != 0 || providerCalls != 0 { t.Fatalf("input-check failure reached state/provider: %#v state=%d provider=%d", outcome, newStateCalls, providerCalls) } } func TestSmokeTestCommandAvailabilityCaptureAndHelpLifecycle(t *testing.T) { type input struct{ campaign string } var originalCalls, replacementCalls, inputCalls, stateCalls int var capturedInput input campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString, func(value *input, supplied commandkit.Supplied[string]) { value.campaign = supplied.Value }) replacementFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "season"}, (*string)(nil), parseDefinitionString, func(value *input, supplied commandkit.Supplied[string]) { value.campaign = "replacement:" + supplied.Value }) selectors := []commandkit.Flag[input]{campaignFlag} suite := Suite[input, int]{ NewInput: func() input { inputCalls++; return input{} }, NewState: func() int { stateCalls++; return 1 }, Selectors: selectors, SmokeTest: func(_ context.Context, _ SmokeRequest, value input, _ int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { originalCalls++ capturedInput = value return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil }, } declaration := Command(suite) suite.SmokeTest = func(context.Context, SmokeRequest, input, int) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { replacementCalls++ return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil } selectors[0] = replacementFlag app := newPromptTestApp(t, declaration) var help bytes.Buffer outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 { t.Fatalf("smoke help lifecycle = (%#v, %q), input/state/providers=%d/%d/%d/%d", outcome, help.String(), inputCalls, stateCalls, originalCalls, replacementCalls) } outcome = app.Execute(context.Background(), []string{"version"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || inputCalls != 0 || stateCalls != 0 || originalCalls != 0 || replacementCalls != 0 { t.Fatalf("version lifecycle = %#v, input/state/providers=%d/%d/%d/%d", outcome, inputCalls, stateCalls, originalCalls, replacementCalls) } outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--campaign=spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || originalCalls != 1 || replacementCalls != 0 || inputCalls != 1 || stateCalls != 1 || capturedInput.campaign != "spring" { t.Fatalf("captured callback/selector = (%#v, %#v), calls=%d/%d input/state=%d/%d", outcome, capturedInput, originalCalls, replacementCalls, inputCalls, stateCalls) } otherApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) { return commandkit.Acquired[[]promptkit.PromptIdentity]{}, nil }, })) help.Reset() outcome = otherApp.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || strings.Contains(help.String(), "\n test ") { t.Fatalf("smoke command installed without callback: (%#v, %q)", outcome, help.String()) } issueSuite := WithSmokeTesting(Suite[struct{}, struct{}]{ InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) { return commandkit.Acquired[*promptkit.RequirementInspection]{}, nil }, }, nil, nil) issueSuite.SmokeTest = nil if _, err := commandkit.New(commandkit.Identity{Name: "example"}, Command(issueSuite)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") { t.Fatalf("narrowed suite lost smoke wiring issue: %v", err) } } func TestSmokeTestBuilderReceivesConcreteAggregateKinds(t *testing.T) { wantKinds := []promptkit.SmokeTestKind{ promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput, } var builtKinds []promptkit.SmokeTestKind loadErr := errors.New("engine unavailable") suite := WithSmokeTesting(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { return nil, loadErr }, func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (promptkit.SmokeTestRequest, error) { builtKinds = append(builtKinds, request.Request.Kind) return request.Request, nil }) app := newPromptTestApp(t, Command(suite)) outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, loadErr) || !reflect.DeepEqual(builtKinds, wantKinds) { t.Fatalf("aggregate builder lifecycle = %#v, built=%#v", outcome, builtKinds) } } func TestSmokeTestHelpAndVersionLeaveWrappedProviderIdle(t *testing.T) { newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls := 0, 0, 0, 0, 0 suite := WithSmokeTesting(Suite[int, int]{ NewInput: func() int { newInputCalls++; return 1 }, NewState: func() int { newStateCalls++; return 2 }, CheckInput: func(ckconfig.Selection, int) error { checkCalls++ return nil }, }, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { loadCalls++ return nil, errors.New("loader should be idle") }, func(_ context.Context, request SmokeRequest, _ int, _ int) (promptkit.SmokeTestRequest, error) { buildCalls++ return request.Request, nil }) app := newPromptTestApp(t, Command(suite)) for _, args := range [][]string{{"help", "promptkit"}, {"version"}} { outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("%v outcome = %#v", args, outcome) } } if newInputCalls != 0 || checkCalls != 0 || newStateCalls != 0 || buildCalls != 0 || loadCalls != 0 { t.Fatalf("help/version invoked smoke work: input/check/state/build/load=%d/%d/%d/%d/%d", newInputCalls, checkCalls, newStateCalls, buildCalls, loadCalls) } } func TestSmokeTestNilFactoriesSupplyZeroValuesForAggregate(t *testing.T) { checkCalls, providerCalls := 0, 0 suite := Suite[struct{ value string }, struct{ count int }]{ CheckInput: func(_ ckconfig.Selection, value struct{ value string }) error { checkCalls++ if value.value != "" { t.Errorf("nil input factory value = %#v", value) } return nil }, SmokeTest: func(_ context.Context, _ SmokeRequest, value struct{ value string }, status struct{ count int }) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { providerCalls++ if value.value != "" || status.count != 0 { t.Errorf("nil factories yielded input/state (%#v, %#v)", value, status) } return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil }, } outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || checkCalls != 1 || providerCalls != 4 { t.Fatalf("nil-factory aggregate = %#v, checks/providers=%d/%d", outcome, checkCalls, providerCalls) } }