Add PromptKit single smoke test command

integrations/ckprompt/smoke_command_test.go

@@ -0,0 +1,145 @@
+package ckprompt
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+	"go.yaml.in/yaml/v4"
+)
+
+func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) {
+	type input struct{ campaign string }
+	type state struct{ initialized bool }
+	campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Campaign selector."}, (*string)(nil), parseDefinitionString,
+		func(value *input, campaign commandkit.Supplied[string]) { value.campaign = campaign.Value })
+
+	var gotSelection ckconfig.Selection
+	var gotRequest promptkit.SmokeTestRequest
+	var gotInput input
+	var gotState state
+	providerCalls, checkedCalls := 0, 0
+	suite := Suite[input, state]{
+		NewInput:  func() input { return input{} },
+		NewState:  func() state { return state{initialized: true} },
+		Selectors: []commandkit.Flag[input]{campaignFlag},
+		CheckInput: func(selection ckconfig.Selection, value input) error {
+			checkedCalls++
+			if checkedCalls == 1 && (!selection.Config.Set || selection.Config.Value != "config.yml" || selection.Overlay.Value != "overlay.yml" || value.campaign != "spring") {
+				t.Errorf("checked selection and input = (%#v, %#v)", selection, value)
+			}
+			return nil
+		},
+		SmokeTest: func(_ context.Context, request SmokeRequest, value input, status state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			providerCalls++
+			gotSelection, gotRequest, gotInput, gotState = request.Selection, request.Request, value, status
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Supported: true, Completed: true}}, nil
+		},
+	}
+	app := newPromptTestApp(t, Command(suite))
+
+	var help bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") {
+		t.Fatalf("promptkit help = (%#v, %q)", outcome, help.String())
+	}
+
+	var output bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{
+		"promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring",
+		"--backend=local", "--prompt=briefing", "--profile=editorial", "--endpoint=https://example.invalid",
+		"--model=fixture", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0",
+		"--service-tier=", "--reasoning-effort=", "--api-key-env=TOKEN", "--transient-retries=0",
+	}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 {
+		t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
+	}
+	if gotSelection.Config.Value != "config.yml" || gotSelection.Overlay.Value != "overlay.yml" || gotInput.campaign != "spring" || !gotState.initialized {
+		t.Errorf("provider selection/input/state = (%#v, %#v, %#v)", gotSelection, gotInput, gotState)
+	}
+	if gotRequest.Kind != promptkit.SmokeTestAuthentication || gotRequest.BackendID != "local" || gotRequest.PromptID != "briefing" || gotRequest.ProfileID != "editorial" {
+		t.Errorf("smoke request identity = %#v", gotRequest)
+	}
+	if gotRequest.Execution == nil || gotRequest.Execution.Endpoint != "https://example.invalid" || gotRequest.Execution.Model != "fixture" || gotRequest.Execution.Temperature == nil || *gotRequest.Execution.Temperature != 0 || gotRequest.Execution.MaxTokens == nil || *gotRequest.Execution.MaxTokens != 0 || gotRequest.Execution.TopP == nil || *gotRequest.Execution.TopP != 0 || gotRequest.Execution.TimeoutSeconds == nil || *gotRequest.Execution.TimeoutSeconds != 0 || gotRequest.Execution.ServiceTier != "" || gotRequest.Execution.ReasoningEffort == nil || *gotRequest.Execution.ReasoningEffort != "" || gotRequest.Execution.APIKeyEnv != "TOKEN" || gotRequest.TransientRetries == nil || *gotRequest.TransientRetries != 0 {
+		t.Errorf("smoke execution overrides did not preserve supplied values: %#v", gotRequest)
+	}
+	var report map[string]any
+	if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
+		t.Fatalf("decode smoke report %q: %v", output.String(), err)
+	}
+	if report["kind"] != "authentication" || report["completed"] != true {
+		t.Errorf("smoke report = %#v", report)
+	}
+
+	var omittedRequest promptkit.SmokeTestRequest
+	omittedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			omittedRequest = request.Request
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
+		},
+	}))
+	output.Reset()
+	outcome = omittedApp.Execute(context.Background(), []string{"promptkit", "test", "text-inference"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || omittedRequest.Execution != nil || omittedRequest.TransientRetries != nil {
+		t.Fatalf("omitted execution overrides = (%#v, %#v), execution nil=%t retries nil=%t", outcome, omittedRequest, omittedRequest.Execution == nil, omittedRequest.TransientRetries == nil)
+	}
+
+	providerCalls = 0
+	output.Reset()
+	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeUsage || providerCalls != 0 || output.Len() != 0 {
+		t.Fatalf("unexpanded aggregate token = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
+	}
+}
+
+func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) {
+	sentinel := errors.New("native smoke operation failed")
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}}, sentinel
+		},
+	}))
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 {
+		t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String())
+	}
+
+	app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{}, nil
+		},
+	}))
+	output.Reset()
+	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || output.Len() != 0 {
+		t.Fatalf("empty acquisition outcome = (%#v, %q)", outcome, output.String())
+	}
+}
+
+type smokeShortWriter struct{}
+
+func (smokeShortWriter) Write(value []byte) (int, error) {
+	if len(value) == 0 {
+		return 0, nil
+	}
+	return len(value) - 1, nil
+}
+
+func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) {
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil
+		},
+	}))
+	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, io.ErrShortWrite) || !errors.Is(outcome.DeliveryError, io.ErrShortWrite) || !outcome.ReportIncomplete {
+		t.Fatalf("short report delivery = %#v", outcome)
+	}
+}