Verify validation outcomes and report delivery

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # PromptKit request validation implementation plan
 
-Status: Stages 1–5 complete; implementation is ongoing.
+Status: Stages 1–6 complete; implementation is ongoing.
 
 ## Authority, scope, and baseline
 
@@ -356,6 +356,16 @@ its maintained composed consumer tests.
 **Complete when:** assembled validation follows native semantics and existing
 outcome/report delivery contracts under representative successes and failures.
 
+**Completed:** 2026-10-12. Exercised the composed command with real PromptKit
+success, missing required input, and invalid profile selection; success emits
+only `prompt_id` and `valid`, and the fail-on-call model client remains unused.
+Command tests cover builder, ConfigKit loader, custom provider, and native
+failures; warning diagnostics on success and error; cancellation before and
+after acquisition; callback-error precedence; report writer errors, short
+writes, and diagnostic writer errors with the established outcome fields.
+Focused outcome tests and `GOTOOLCHAIN=local go test -count=1 ./...` passed from
+`integrations/ckprompt`.
+
 ## Stage 7: Extend the maintained composed consumer
 
 **Primary files:** `integrations/ckprompt/examples/composed/main.go`, its tests,

integrations/ckprompt/validation_command_test.go

@@ -11,6 +11,7 @@ import (
 
 	commandkit "gitea.maximumdirect.net/eric/CommandKit"
 	"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
+	configkit "gitea.maximumdirect.net/eric/ConfigKit"
 	promptkit "gitea.maximumdirect.net/eric/promptkit"
 	"go.yaml.in/yaml/v4"
 )
@@ -314,3 +315,196 @@ func TestValidationCommandSuppliesZeroValueFactoriesIncludingNilPointers(t *test
 		t.Fatalf("omitted factories = outcome %#v, input=%v state=%v", outcome, gotInput, gotState)
 	}
 }
+
+func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T) {
+	client := &validationFailOnGenerate{}
+	engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client))
+	loadCalls := 0
+	suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
+		loadCalls++
+		return engine, nil
+	}, nil)
+	app := newPromptTestApp(t, Command(suite))
+
+	var output bytes.Buffer
+	outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
+		t.Fatalf("native request validation = %#v, output=%q", outcome, output.String())
+	}
+	var report map[string]any
+	if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
+		t.Fatalf("decode native confirmation %q: %v", output.String(), err)
+	}
+	if len(report) != 2 || report["prompt_id"] != "empty" || report["valid"] != true {
+		t.Fatalf("native confirmation contains unexpected data: %#v", report)
+	}
+
+	for _, test := range []struct {
+		name string
+		args []string
+		want error
+	}{
+		{name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "default"}, want: promptkit.ErrPromptRender},
+		{name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "missing", "--input", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			output.Reset()
+			outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
+			if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, test.want) || output.Len() != 0 {
+				t.Fatalf("native failure = %#v, output=%q, want %v", outcome, output.String(), test.want)
+			}
+		})
+	}
+	if client.calls != 0 || loadCalls != 3 {
+		t.Fatalf("model calls=%d, engine loads=%d", client.calls, loadCalls)
+	}
+}
+
+func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) {
+	warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
+		_, err := io.WriteString(writer, "validation warning\n")
+		return err
+	}}
+	withWarning := func(diagnostics []commandkit.Diagnostic) bool {
+		for _, diagnostic := range diagnostics {
+			if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
+				var output bytes.Buffer
+				if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
+					return true
+				}
+			}
+		}
+		return false
+	}
+	var diagnostics bytes.Buffer
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
+		return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
+	}}))
+	outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
+	if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" {
+		t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String())
+	}
+
+	providerErr := errors.New("validation provider failed")
+	failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
+		return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
+	}}))
+	diagnostics.Reset()
+	outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
+	if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\nvalidation provider failed\n" {
+		t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String())
+	}
+
+	builderErr := errors.New("request builder failed")
+	loaderCalls := 0
+	builderApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
+		loaderCalls++
+		return nil, nil
+	}, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
+		return promptkit.RunRequest{}, builderErr
+	})))
+	outcome = builderApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, builderErr) || loaderCalls != 0 {
+		t.Fatalf("builder failure = %#v, loader calls=%d", outcome, loaderCalls)
+	}
+
+	_, loadErr := configkit.Load[map[string]any](configkit.Bytes("validation command load failure", []byte("broken: [")))
+	if loadErr == nil {
+		t.Fatal("invalid configuration unexpectedly loaded")
+	}
+	loadApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
+		return nil, loadErr
+	}, nil)))
+	outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	var retained *configkit.LoadError
+	if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 {
+		t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
+	}
+}
+
+func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) {
+	warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
+		_, err := io.WriteString(writer, "validation warning\n")
+		return err
+	}}
+	calls := 0
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
+		calls++
+		return commandkit.Acquired[struct{}]{}, nil
+	}}))
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	var output bytes.Buffer
+	outcome := app.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 0 || output.Len() != 0 {
+		t.Fatalf("pre-acquisition cancellation = %#v, calls=%d, output=%q", outcome, calls, output.String())
+	}
+
+	ctx, cancel = context.WithCancel(context.Background())
+	canceledApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
+		calls++
+		cancel()
+		return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
+	}}))
+	output.Reset()
+	outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 {
+		t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String())
+	}
+
+	providerErr := errors.New("provider error before cancellation")
+	ctx, cancel = context.WithCancel(context.Background())
+	errorApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
+		cancel()
+		return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
+	}}))
+	outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
+	if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) {
+		t.Fatalf("provider error after cancellation = %#v", outcome)
+	}
+}
+
+func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testing.T) {
+	warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
+		_, err := io.WriteString(writer, "validation warning\n")
+		return err
+	}}
+	app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
+		return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
+	}}))
+	writeErr := errors.New("validation report writer failed")
+	for _, test := range []struct {
+		name   string
+		writer io.Writer
+		want   error
+	}{
+		{name: "writer error", writer: partialReportWriter{err: writeErr}, want: writeErr},
+		{name: "short write", writer: partialReportWriter{}, want: io.ErrShortWrite},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var diagnosticOutput bytes.Buffer
+			outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
+			if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
+				t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
+			}
+		})
+	}
+
+	diagnosticErr := errors.New("validation diagnostic writer failed")
+	outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
+	if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete {
+		t.Fatalf("diagnostic delivery = %#v", outcome)
+	}
+}
+
+func withValidationWarning(diagnostics []commandkit.Diagnostic) bool {
+	for _, diagnostic := range diagnostics {
+		if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
+			var output bytes.Buffer
+			if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
+				return true
+			}
+		}
+	}
+	return false
+}