package ckprompt import ( "bytes" "context" "errors" "io" "strings" "testing" commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" promptkit "gitea.maximumdirect.net/eric/promptkit" "go.yaml.in/yaml/v4" ) func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) { type input struct{ campaign string } type state struct{ initialized bool } campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Campaign selector."}, (*string)(nil), parseDefinitionString, func(value *input, campaign commandkit.Supplied[string]) { value.campaign = campaign.Value }) var gotSelection ckconfig.Selection var gotRequest promptkit.SmokeTestRequest var gotInput input var gotState state providerCalls, checkedCalls := 0, 0 suite := Suite[input, state]{ NewInput: func() input { return input{} }, NewState: func() state { return state{initialized: true} }, Selectors: []commandkit.Flag[input]{campaignFlag}, CheckInput: func(selection ckconfig.Selection, value input) error { checkedCalls++ if checkedCalls == 1 && (!selection.Config.Set || selection.Config.Value != "config.yml" || selection.Overlay.Value != "overlay.yml" || value.campaign != "spring") { t.Errorf("checked selection and input = (%#v, %#v)", selection, value) } return nil }, SmokeTest: func(_ context.Context, request SmokeRequest, value input, status state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { providerCalls++ gotSelection, gotRequest, gotInput, gotState = request.Selection, request.Request, value, status return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Supported: true, Completed: true}}, nil }, } app := newPromptTestApp(t, Command(suite)) var help bytes.Buffer outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") { t.Fatalf("promptkit help = (%#v, %q)", outcome, help.String()) } var output bytes.Buffer outcome = app.Execute(context.Background(), []string{ "promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring", "--backend=local", "--prompt=briefing", "--profile=editorial", "--endpoint=https://example.invalid", "--model=fixture", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0", "--service-tier=", "--reasoning-effort=", "--api-key-env=TOKEN", "--transient-retries=0", }, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 { t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls) } if gotSelection.Config.Value != "config.yml" || gotSelection.Overlay.Value != "overlay.yml" || gotInput.campaign != "spring" || !gotState.initialized { t.Errorf("provider selection/input/state = (%#v, %#v, %#v)", gotSelection, gotInput, gotState) } if gotRequest.Kind != promptkit.SmokeTestAuthentication || gotRequest.BackendID != "local" || gotRequest.PromptID != "briefing" || gotRequest.ProfileID != "editorial" { t.Errorf("smoke request identity = %#v", gotRequest) } if gotRequest.Execution == nil || gotRequest.Execution.Endpoint != "https://example.invalid" || gotRequest.Execution.Model != "fixture" || gotRequest.Execution.Temperature == nil || *gotRequest.Execution.Temperature != 0 || gotRequest.Execution.MaxTokens == nil || *gotRequest.Execution.MaxTokens != 0 || gotRequest.Execution.TopP == nil || *gotRequest.Execution.TopP != 0 || gotRequest.Execution.TimeoutSeconds == nil || *gotRequest.Execution.TimeoutSeconds != 0 || gotRequest.Execution.ServiceTier != "" || gotRequest.Execution.ReasoningEffort == nil || *gotRequest.Execution.ReasoningEffort != "" || gotRequest.Execution.APIKeyEnv != "TOKEN" || gotRequest.TransientRetries == nil || *gotRequest.TransientRetries != 0 { t.Errorf("smoke execution overrides did not preserve supplied values: %#v", gotRequest) } var report map[string]any if err := yaml.Unmarshal(output.Bytes(), &report); err != nil { t.Fatalf("decode smoke report %q: %v", output.String(), err) } if report["kind"] != "authentication" || report["completed"] != true { t.Errorf("smoke report = %#v", report) } var omittedRequest promptkit.SmokeTestRequest omittedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { omittedRequest = request.Request return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil }, })) output.Reset() outcome = omittedApp.Execute(context.Background(), []string{"promptkit", "test", "text-inference"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || omittedRequest.Execution != nil || omittedRequest.TransientRetries != nil { t.Fatalf("omitted execution overrides = (%#v, %#v), execution nil=%t retries nil=%t", outcome, omittedRequest, omittedRequest.Execution == nil, omittedRequest.TransientRetries == nil) } providerCalls = 0 output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard}) var aggregate struct { Tests []struct { Kind string `yaml:"kind"` } `yaml:"tests"` } if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 4 || yaml.Unmarshal(output.Bytes(), &aggregate) != nil || len(aggregate.Tests) != 4 { t.Fatalf("aggregate smoke result = (%#v, %q), calls=%d", outcome, output.String(), providerCalls) } } func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) { sentinel := errors.New("native smoke operation failed") app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}}, sentinel }, })) var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 { t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String()) } app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { return commandkit.Acquired[*promptkit.SmokeTestReport]{}, nil }, })) output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || output.Len() != 0 { t.Fatalf("empty acquisition outcome = (%#v, %q)", outcome, output.String()) } } type smokeShortWriter struct{} func (smokeShortWriter) Write(value []byte) (int, error) { if len(value) == 0 { return 0, nil } return len(value) - 1, nil } func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) { app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil }, })) outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, io.ErrShortWrite) || !errors.Is(outcome.DeliveryError, io.ErrShortWrite) || !outcome.ReportIncomplete { t.Fatalf("short report delivery = %#v", outcome) } }