Add PromptKit single smoke test command

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Dependency upgrades, discovery, and smoke testing implementation plan
 
-Status: Stages 1–10 complete; Stage 11 is next.
+Status: Stages 1–11 complete; Stage 12 is next.
 
 ## Authority and baseline
 
@@ -577,6 +577,15 @@ Run focused smoke command tests.
 **Complete when:** all single-kind bindings invoke the optional provider and
 faithfully report its result and errors.
 
+**Completed:** 2026-10-17. Added the optional `promptkit test KIND` command with
+all accepted scalar bindings, YAML preflight, prepared input/state, and native
+report delivery. Explicit zero/empty overrides remain present, omitted
+execution/retry overrides remain absent, and `all` is rejected before native
+request construction pending aggregate coordination. Assembled tests cover
+flag forwarding, findings, report-plus-error, nil-result contract failure, and
+short-write delivery. `GOTOOLCHAIN=local go test ./... -run
+'TestSmokeTestCommand' -count=1` passed in ckprompt.
+
 ## Stage 12: Coordinate aggregate smoke execution
 
 **Primary files:** smoke command runner, aggregate projection/outcome helpers,

integrations/ckprompt/command.go

@@ -53,8 +53,8 @@ type inputAssignment struct {
 }
 
 // Command captures the optional PromptKit catalogs, named inspection,
-// requirements discovery, and concrete request validation callbacks and adds
+// requirements discovery, concrete request validation, and smoke-test
-// only supplied operations. Discovery and validation share the
+// callbacks and adds only supplied operations. Discovery and validation share the
 // `requirements` group and remain independently optional. An absent suite
 // claims no namespace; attached construction issues remain reportable. It adds
 // no configuration family; callers compose ConfigScope independently.
@@ -67,7 +67,7 @@ type inputAssignment struct {
 // render YAML with `yaml` as the only format and default. Named-show formats are
 // unchanged.
 func Command[I, S any](suite Suite[I, S]) commandkit.Command {
-	if suite.InspectPrompt == nil && suite.InspectProfile == nil && suite.InspectRequirements == nil && suite.ValidateRequest == nil && suite.ListPrompts == nil && suite.ListProfiles == nil && suite.ListBackends == nil && suite.ListNamespaces == nil && suite.ListCollections == nil {
+	if suite.InspectPrompt == nil && suite.InspectProfile == nil && suite.InspectRequirements == nil && suite.ValidateRequest == nil && suite.SmokeTest == nil && suite.ListPrompts == nil && suite.ListProfiles == nil && suite.ListBackends == nil && suite.ListNamespaces == nil && suite.ListCollections == nil {
 		command := commandkit.NoCommand()
 		for _, issue := range suite.issues {
 			command = commandkit.WithError(command, issue)
@@ -116,7 +116,10 @@ func Command[I, S any](suite Suite[I, S]) commandkit.Command {
 	if len(requirementsCommands) != 0 {
 		rootCommands = append(rootCommands, commandkit.Group("requirements", "Inspect and validate prompt requirements.", requirementsCommands...))
 	}
-	command := commandkit.RootGroup("promptkit", "Inspect, validate, and list PromptKit definitions and requirements.", rootCommands...)
+	if suite.SmokeTest != nil {
+		rootCommands = append(rootCommands, newSmokeTestCommand(suite))
+	}
+	command := commandkit.RootGroup("promptkit", "Inspect, list, validate requests, and run PromptKit smoke tests.", rootCommands...)
 	for _, issue := range suite.issues {
 		command = commandkit.WithError(command, issue)
 	}

integrations/ckprompt/smoke_command.go

@@ -0,0 +1,246 @@
+package ckprompt
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+)
+
+type smokeCommandInput[I any] struct {
+	selection ckconfig.Selection
+	kind      string
+	request   promptkit.SmokeTestRequest
+	format    string
+	captured  commandkit.FlagValues[I]
+	selected  I
+}
+
+func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
+	flags := commandkit.CaptureFlags(suite.Selectors, func(input **smokeCommandInput[I]) *commandkit.FlagValues[I] {
+		if input == nil || *input == nil {
+			return nil
+		}
+		return &(*input).captured
+	})
+	flags = append(flags,
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Configuration input path."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				(*input).selection.Config = value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay path."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				(*input).selection.Overlay = value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "format", Usage: "Smoke report format: yaml."}, stringDefault("yaml"), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).format = value.Value }),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "backend", Usage: "PromptKit backend ID."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				(*input).request.BackendID = value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "prompt", Usage: "PromptKit prompt ID."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				(*input).request.PromptID = value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "profile", Usage: "PromptKit profile ID."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				(*input).request.ProfileID = value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "endpoint", Usage: "Execution endpoint override."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.Endpoint = value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "model", Usage: "Execution model override."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.Model = value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "temperature", Usage: "Execution temperature override."}, (*float64)(nil), parseSmokeFloat,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.Temperature = &value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "max-tokens", Usage: "Execution maximum tokens override."}, (*int)(nil), strconv.Atoi,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[int]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.MaxTokens = &value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "top-p", Usage: "Execution top-p override."}, (*float64)(nil), parseSmokeFloat,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[float64]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.TopP = &value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "timeout-seconds", Usage: "Execution timeout override."}, (*int)(nil), strconv.Atoi,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[int]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.TimeoutSeconds = &value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "service-tier", Usage: "Execution service tier."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.ServiceTier = value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "reasoning-effort", Usage: "Execution reasoning effort override."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.ReasoningEffort = &value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "api-key-env", Usage: "Execution API key environment variable."}, (*string)(nil), parseDefinitionString,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[string]) {
+				if !value.Set {
+					return
+				}
+				execution := smokeExecution(input)
+				execution.APIKeyEnv = value.Value
+			}),
+		commandkit.ValueFlag(commandkit.FlagMeta{Name: "transient-retries", Usage: "Inference transient retry count."}, (*int)(nil), strconv.Atoi,
+			func(input **smokeCommandInput[I], value commandkit.Supplied[int]) {
+				if value.Set {
+					(*input).request.TransientRetries = &value.Value
+				}
+			}),
+	)
+
+	return commandkit.NewCommand(commandkit.CommandSpec[*smokeCommandInput[I]]{
+		Name:    "test",
+		Summary: "Run one PromptKit smoke test.",
+		NewInput: func() *smokeCommandInput[I] {
+			return &smokeCommandInput[I]{format: "yaml"}
+		},
+		Flags: flags,
+		Arguments: []commandkit.Argument[*smokeCommandInput[I]]{
+			commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "kind", Usage: "Native smoke-test kind."}, true, parseDefinitionString,
+				func(input **smokeCommandInput[I], value commandkit.Supplied[string]) { (*input).kind = value.Value }),
+		},
+		CheckInput: func(input *smokeCommandInput[I]) error {
+			if input == nil {
+				return errors.New("PromptKit smoke-test input must not be nil")
+			}
+			if err := checkSmokeFormat(input.format); err != nil {
+				return err
+			}
+			if input.kind == "all" {
+				return errors.New("PromptKit aggregate smoke tests are not available")
+			}
+			input.request.Kind = promptkit.SmokeTestKind(input.kind)
+			selected, err := prepareSuiteInput(suite, input.selection, input.captured)
+			if err != nil {
+				return err
+			}
+			input.selected = selected
+			return nil
+		},
+		Run: func(invocation commandkit.Invocation, input *smokeCommandInput[I]) commandkit.Outcome {
+			return runSmokeTest(invocation, suite, input)
+		},
+	})
+}
+
+func smokeExecution[I any](input **smokeCommandInput[I]) *promptkit.ExecutionTargetOverride {
+	if (*input).request.Execution == nil {
+		(*input).request.Execution = &promptkit.ExecutionTargetOverride{}
+	}
+	return (*input).request.Execution
+}
+
+func parseSmokeFloat(value string) (float64, error) {
+	return strconv.ParseFloat(value, 64)
+}
+
+func checkSmokeFormat(format string) error {
+	if format != "yaml" {
+		return fmt.Errorf("unsupported smoke report format %q; expected yaml", format)
+	}
+	return nil
+}
+
+func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
+	if err := promptKitContextError(invocation.Context); err != nil {
+		return definitionFailure(err, nil)
+	}
+	var state S
+	if suite.NewState != nil {
+		state = suite.NewState()
+	}
+	if err := promptKitContextError(invocation.Context); err != nil {
+		return definitionFailure(err, nil)
+	}
+
+	request := SmokeRequest{Selection: input.selection, Request: input.request}
+	acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state)
+	if acquired.Value == nil && operationErr == nil {
+		operationErr = errors.New("PromptKit smoke test returned neither a report nor an error")
+	}
+	if err := promptKitContextError(invocation.Context); err != nil && !errors.Is(operationErr, err) {
+		if operationErr == nil {
+			operationErr = err
+		} else {
+			operationErr = errors.Join(operationErr, err)
+		}
+	}
+	if acquired.Value == nil {
+		return definitionFailure(operationErr, acquired.Diagnostics)
+	}
+
+	output, encodeErr := encodeSmokeReport(acquired.Value)
+	if encodeErr != nil {
+		return smokeDeliveryFailure(operationErr, encodeErr, acquired.Diagnostics, false)
+	}
+	return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr)
+}
+
+func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome {
+	written, deliveryErr := invocation.Streams.Out.Write(output)
+	if deliveryErr == nil && written != len(output) {
+		deliveryErr = io.ErrShortWrite
+	}
+	if deliveryErr != nil {
+		return smokeDeliveryFailure(operationErr, deliveryErr, diagnostics, true)
+	}
+	if operationErr != nil {
+		return definitionFailure(operationErr, diagnostics)
+	}
+	outcome := commandkit.Success()
+	outcome.Diagnostics = diagnostics
+	return outcome
+}
+
+func smokeDeliveryFailure(operationErr, deliveryErr error, diagnostics []commandkit.Diagnostic, reportStarted bool) commandkit.Outcome {
+	cause := operationErr
+	if cause == nil {
+		cause = deliveryErr
+	}
+	outcome := definitionFailure(cause, diagnostics)
+	outcome.DeliveryError = deliveryErr
+	outcome.ReportIncomplete = reportStarted
+	return outcome
+}

integrations/ckprompt/smoke_command_test.go

@@ -0,0 +1,145 @@
+package ckprompt
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+	"go.yaml.in/yaml/v4"
+)
+
+func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) {
+	type input struct{ campaign string }
+	type state struct{ initialized bool }
+	campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Campaign selector."}, (*string)(nil), parseDefinitionString,
+		func(value *input, campaign commandkit.Supplied[string]) { value.campaign = campaign.Value })
+
+	var gotSelection ckconfig.Selection
+	var gotRequest promptkit.SmokeTestRequest
+	var gotInput input
+	var gotState state
+	providerCalls, checkedCalls := 0, 0
+	suite := Suite[input, state]{
+		NewInput:  func() input { return input{} },
+		NewState:  func() state { return state{initialized: true} },
+		Selectors: []commandkit.Flag[input]{campaignFlag},
+		CheckInput: func(selection ckconfig.Selection, value input) error {
+			checkedCalls++
+			if checkedCalls == 1 && (!selection.Config.Set || selection.Config.Value != "config.yml" || selection.Overlay.Value != "overlay.yml" || value.campaign != "spring") {
+				t.Errorf("checked selection and input = (%#v, %#v)", selection, value)
+			}
+			return nil
+		},
+		SmokeTest: func(_ context.Context, request SmokeRequest, value input, status state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			providerCalls++
+			gotSelection, gotRequest, gotInput, gotState = request.Selection, request.Request, value, status
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Supported: true, Completed: true}}, nil
+		},
+	}
+	app := newPromptTestApp(t, Command(suite))
+
+	var help bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"help", "promptkit"}, commandkit.Streams{Out: &help, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "test") {
+		t.Fatalf("promptkit help = (%#v, %q)", outcome, help.String())
+	}
+
+	var output bytes.Buffer
+	outcome = app.Execute(context.Background(), []string{
+		"promptkit", "test", "authentication", "--config=config.yml", "--overlay=overlay.yml", "--campaign=spring",
+		"--backend=local", "--prompt=briefing", "--profile=editorial", "--endpoint=https://example.invalid",
+		"--model=fixture", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0",
+		"--service-tier=", "--reasoning-effort=", "--api-key-env=TOKEN", "--transient-retries=0",
+	}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || providerCalls != 1 {
+		t.Fatalf("smoke outcome = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
+	}
+	if gotSelection.Config.Value != "config.yml" || gotSelection.Overlay.Value != "overlay.yml" || gotInput.campaign != "spring" || !gotState.initialized {
+		t.Errorf("provider selection/input/state = (%#v, %#v, %#v)", gotSelection, gotInput, gotState)
+	}
+	if gotRequest.Kind != promptkit.SmokeTestAuthentication || gotRequest.BackendID != "local" || gotRequest.PromptID != "briefing" || gotRequest.ProfileID != "editorial" {
+		t.Errorf("smoke request identity = %#v", gotRequest)
+	}
+	if gotRequest.Execution == nil || gotRequest.Execution.Endpoint != "https://example.invalid" || gotRequest.Execution.Model != "fixture" || gotRequest.Execution.Temperature == nil || *gotRequest.Execution.Temperature != 0 || gotRequest.Execution.MaxTokens == nil || *gotRequest.Execution.MaxTokens != 0 || gotRequest.Execution.TopP == nil || *gotRequest.Execution.TopP != 0 || gotRequest.Execution.TimeoutSeconds == nil || *gotRequest.Execution.TimeoutSeconds != 0 || gotRequest.Execution.ServiceTier != "" || gotRequest.Execution.ReasoningEffort == nil || *gotRequest.Execution.ReasoningEffort != "" || gotRequest.Execution.APIKeyEnv != "TOKEN" || gotRequest.TransientRetries == nil || *gotRequest.TransientRetries != 0 {
+		t.Errorf("smoke execution overrides did not preserve supplied values: %#v", gotRequest)
+	}
+	var report map[string]any
+	if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
+		t.Fatalf("decode smoke report %q: %v", output.String(), err)
+	}
+	if report["kind"] != "authentication" || report["completed"] != true {
+		t.Errorf("smoke report = %#v", report)
+	}
+
+	var omittedRequest promptkit.SmokeTestRequest
+	omittedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			omittedRequest = request.Request
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
+		},
+	}))
+	output.Reset()
+	outcome = omittedApp.Execute(context.Background(), []string{"promptkit", "test", "text-inference"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || omittedRequest.Execution != nil || omittedRequest.TransientRetries != nil {
+		t.Fatalf("omitted execution overrides = (%#v, %#v), execution nil=%t retries nil=%t", outcome, omittedRequest, omittedRequest.Execution == nil, omittedRequest.TransientRetries == nil)
+	}
+
+	providerCalls = 0
+	output.Reset()
+	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeUsage || providerCalls != 0 || output.Len() != 0 {
+		t.Fatalf("unexpanded aggregate token = (%#v, %q), calls=%d", outcome, output.String(), providerCalls)
+	}
+}
+
+func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) {
+	sentinel := errors.New("native smoke operation failed")
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}}, sentinel
+		},
+	}))
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 {
+		t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String())
+	}
+
+	app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{}, nil
+		},
+	}))
+	output.Reset()
+	outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || output.Len() != 0 {
+		t.Fatalf("empty acquisition outcome = (%#v, %q)", outcome, output.String())
+	}
+}
+
+type smokeShortWriter struct{}
+
+func (smokeShortWriter) Write(value []byte) (int, error) {
+	if len(value) == 0 {
+		return 0, nil
+	}
+	return len(value) - 1, nil
+}
+
+func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) {
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil
+		},
+	}))
+	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, io.ErrShortWrite) || !errors.Is(outcome.DeliveryError, io.ErrShortWrite) || !outcome.ReportIncomplete {
+		t.Fatalf("short report delivery = %#v", outcome)
+	}
+}