Coordinate aggregate PromptKit smoke tests

integrations/ckprompt/smoke_command.go

@@ -148,10 +148,9 @@ func newSmokeTestCommand[I, S any](suite Suite[I, S]) commandkit.Command {
 			if err := checkSmokeFormat(input.format); err != nil {
 				return err
 			}
-			if input.kind == "all" {
-				return errors.New("PromptKit aggregate smoke tests are not available")
+			if input.kind != "all" {
+				input.request.Kind = promptkit.SmokeTestKind(input.kind)
 			}
-			input.request.Kind = promptkit.SmokeTestKind(input.kind)
 			selected, err := prepareSuiteInput(suite, input.selection, input.captured)
 			if err != nil {
 				return err
@@ -194,6 +193,9 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
 	if err := promptKitContextError(invocation.Context); err != nil {
 		return definitionFailure(err, nil)
 	}
+	if input.kind == "all" {
+		return runSmokeAggregate(invocation, suite, input, state)
+	}
 
 	request := SmokeRequest{Selection: input.selection, Request: input.request}
 	acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state)
@@ -218,6 +220,99 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
 	return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr)
 }
 
+func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
+	kinds := [...]promptkit.SmokeTestKind{
+		promptkit.SmokeTestReachability,
+		promptkit.SmokeTestAuthentication,
+		promptkit.SmokeTestTextInference,
+		promptkit.SmokeTestStructuredOutput,
+	}
+	results := make([]smokeResult, 0, len(kinds))
+	var diagnostics []commandkit.Diagnostic
+	var operationErrors []error
+
+	for _, kind := range kinds {
+		if err := promptKitContextError(invocation.Context); err != nil {
+			if len(results) == 0 {
+				return definitionFailure(err, diagnostics)
+			}
+			operationErrors = append(operationErrors, err)
+			break
+		}
+
+		seed := smokeRequestForKind(input.request, kind)
+		acquired, operationErr := suite.SmokeTest(invocation.Context, SmokeRequest{Selection: input.selection, Request: seed}, input.selected, state)
+		if acquired.Value == nil && operationErr == nil {
+			operationErr = errors.New("PromptKit smoke test returned neither a report nor an error")
+		}
+		if err := promptKitContextError(invocation.Context); err != nil && !errors.Is(operationErr, err) {
+			if operationErr == nil {
+				operationErr = err
+			} else {
+				operationErr = errors.Join(operationErr, err)
+			}
+		}
+
+		results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr})
+		diagnostics = append(diagnostics, acquired.Diagnostics...)
+		if operationErr != nil {
+			operationErrors = append(operationErrors, operationErr)
+		}
+		if err := promptKitContextError(invocation.Context); err != nil {
+			break
+		}
+	}
+
+	if len(results) == 0 {
+		return definitionFailure(errors.Join(operationErrors...), diagnostics)
+	}
+	output, encodeErr := encodeSmokeResults(results)
+	if encodeErr != nil {
+		return smokeDeliveryFailure(errors.Join(operationErrors...), encodeErr, diagnostics, false)
+	}
+	return writeSmokeReport(invocation, output, diagnostics, errors.Join(operationErrors...))
+}
+
+func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest {
+	request := seed
+	request.Kind = kind
+	if seed.Execution != nil {
+		execution := *seed.Execution
+		execution.Temperature = copyFloat64(seed.Execution.Temperature)
+		execution.MaxTokens = copyInt(seed.Execution.MaxTokens)
+		execution.TopP = copyFloat64(seed.Execution.TopP)
+		execution.TimeoutSeconds = copyInt(seed.Execution.TimeoutSeconds)
+		execution.ReasoningEffort = copyString(seed.Execution.ReasoningEffort)
+		request.Execution = &execution
+	}
+	request.TransientRetries = copyInt(seed.TransientRetries)
+	return request
+}
+
+func copyFloat64(value *float64) *float64 {
+	if value == nil {
+		return nil
+	}
+	copy := *value
+	return &copy
+}
+
+func copyInt(value *int) *int {
+	if value == nil {
+		return nil
+	}
+	copy := *value
+	return &copy
+}
+
+func copyString(value *string) *string {
+	if value == nil {
+		return nil
+	}
+	copy := *value
+	return &copy
+}
+
 func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome {
 	written, deliveryErr := invocation.Streams.Out.Write(output)
 	if deliveryErr == nil && written != len(output) {