Verify PromptKit smoke outcomes and delivery

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Dependency upgrades, discovery, and smoke testing implementation plan
 
-Status: Stages 1–13 complete; Stage 14 is next.
+Status: Stages 1–14 complete; Stage 15 is next.
 
 ## Authority and baseline
 
@@ -694,6 +694,14 @@ provider smoke-test correctness belongs upstream.
 **Complete when:** realistic smoke failures preserve their reports, native error
 identity, and delivery outcomes without a parallel diagnostic test suite.
 
+**Completed:** 2026-10-17. Added outcome and delivery coverage for false/unknown
+observations, error-only and report-plus-error results, cancellation before and
+after acquisition, and retained aggregate results across cancellation. Tests
+also cover error and short report writes, encoding failure, diagnostic writer
+failure, operation plus delivery errors, ordered diagnostics, and joined
+`errors.Is`/`errors.As` identity. The full ckprompt suite passed with
+`GOTOOLCHAIN=local go test -count=1 ./...`.
+
 ## Stage 15: Demonstrate the new interfaces in the maintained consumer
 
 **Primary files:** `integrations/ckprompt/examples/composed`, its README,

integrations/ckprompt/smoke_outcome_test.go

@@ -0,0 +1,175 @@
+package ckprompt
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	commandkit "gitea.maximumdirect.net/eric/CommandKit"
+	promptkit "gitea.maximumdirect.net/eric/promptkit"
+	"go.yaml.in/yaml/v4"
+)
+
+func TestSmokeTestFindingsDoNotChangeSuccessfulOutcome(t *testing.T) {
+	falseValue := false
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{
+				Kind: promptkit.SmokeTestReachability, Supported: false, Applicable: nil, Completed: false,
+				Facts: promptkit.SmokeTestFacts{EndpointReachable: &falseValue},
+			}}, nil
+		},
+	}))
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "reachability"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || outcome.Cause != nil {
+		t.Fatalf("false and unknown findings changed outcome: %#v", outcome)
+	}
+	var report struct {
+		Supported  bool  `yaml:"supported"`
+		Applicable *bool `yaml:"applicable"`
+		Completed  bool  `yaml:"completed"`
+		Facts      struct {
+			EndpointReachable *bool `yaml:"endpoint_reachable"`
+		} `yaml:"facts"`
+	}
+	if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
+		t.Fatalf("decode smoke findings %q: %v", output.String(), err)
+	}
+	if report.Supported || report.Applicable != nil || report.Completed || report.Facts.EndpointReachable == nil || *report.Facts.EndpointReachable {
+		t.Fatalf("native findings were not retained: %#v", report)
+	}
+}
+
+func TestSmokeTestCancellationAfterAcquisitionStillDeliversReport(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	calls := 0
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			calls++
+			cancel()
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil
+		},
+	}))
+	var output bytes.Buffer
+	outcome := app.Execute(ctx, []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 1 || output.Len() == 0 {
+		t.Fatalf("post-acquisition cancellation = (%#v, %q), calls=%d", outcome, output.String(), calls)
+	}
+}
+
+func TestSmokeTestCancellationBeforeAcquisitionSkipsProvider(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	calls, stateCalls := 0, 0
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		NewState: func() struct{} { stateCalls++; return struct{}{} },
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			calls++
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
+		},
+	}))
+	var output bytes.Buffer
+	outcome := app.Execute(ctx, []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 0 || stateCalls != 0 || output.Len() != 0 {
+		t.Fatalf("pre-acquisition cancellation = (%#v, %q), calls/state=%d/%d", outcome, output.String(), calls, stateCalls)
+	}
+}
+
+func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
+	operationErr := errors.New("native operation failed")
+	diagnostic := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
+		_, err := io.WriteString(writer, "smoke warning\n")
+		return err
+	}}
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{
+				Value:       &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication},
+				Diagnostics: []commandkit.Diagnostic{diagnostic},
+			}, operationErr
+		},
+	}))
+	writeErr := errors.New("smoke report writer failed")
+	for _, test := range []struct {
+		name   string
+		writer io.Writer
+		want   error
+	}{
+		{name: "writer error", writer: partialReportWriter{err: writeErr}, want: writeErr},
+		{name: "short write", writer: partialReportWriter{}, want: io.ErrShortWrite},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var diagnosticOutput bytes.Buffer
+			outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
+			warningIndex := strings.Index(diagnosticOutput.String(), "smoke warning")
+			operationIndex := strings.Index(diagnosticOutput.String(), operationErr.Error())
+			if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, test.want) || !outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || warningIndex < 0 || operationIndex < warningIndex {
+				t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
+			}
+		})
+	}
+
+	diagnosticErr := errors.New("smoke diagnostic writer failed")
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete || output.Len() == 0 {
+		t.Fatalf("diagnostic delivery = (%#v, %q)", outcome, output.String())
+	}
+
+	encodeApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{
+				Target: promptkit.SmokeTestTarget{EffectiveSettings: promptkit.ExecutionTarget{ExtraParams: map[string]any{"unsupported": func() {}}}},
+			}}, nil
+		},
+	}))
+	output.Reset()
+	outcome = encodeApp.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || !errors.Is(outcome.DeliveryError, outcome.Cause) || outcome.ReportIncomplete || output.Len() != 0 {
+		t.Fatalf("encoding failure = (%#v, %q)", outcome, output.String())
+	}
+}
+
+type smokeTypedError struct{ text string }
+
+func (err *smokeTypedError) Error() string { return err.text }
+
+func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) {
+	firstErr := &smokeTypedError{text: "reachability failed"}
+	secondErr := &smokeTypedError{text: "authentication failed"}
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
+		SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
+			var err error
+			switch request.Request.Kind {
+			case promptkit.SmokeTestReachability:
+				err = firstErr
+			case promptkit.SmokeTestAuthentication:
+				err = secondErr
+			}
+			return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, err
+		},
+	}))
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	var retained *smokeTypedError
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, firstErr) || !errors.Is(outcome.Cause, secondErr) || !errors.As(outcome.Cause, &retained) || retained != firstErr {
+		t.Fatalf("joined aggregate errors lost identity: %#v, retained=%#v", outcome, retained)
+	}
+	var document struct {
+		Tests []struct {
+			Kind  string `yaml:"kind"`
+			Error string `yaml:"error"`
+		} `yaml:"tests"`
+	}
+	if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
+		t.Fatalf("decode aggregate errors %q: %v", output.String(), err)
+	}
+	if len(document.Tests) != 4 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[0].Error != firstErr.Error() || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) || document.Tests[1].Error != secondErr.Error() || document.Tests[2].Error != "" || document.Tests[3].Error != "" {
+		t.Fatalf("aggregate error entries = %#v", document.Tests)
+	}
+}