diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ba69fdc..264f476 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # PromptKit request validation implementation plan -Status: Stages 1–5 complete; implementation is ongoing. +Status: Stages 1–6 complete; implementation is ongoing. ## Authority, scope, and baseline @@ -356,6 +356,16 @@ its maintained composed consumer tests. **Complete when:** assembled validation follows native semantics and existing outcome/report delivery contracts under representative successes and failures. +**Completed:** 2026-10-12. Exercised the composed command with real PromptKit +success, missing required input, and invalid profile selection; success emits +only `prompt_id` and `valid`, and the fail-on-call model client remains unused. +Command tests cover builder, ConfigKit loader, custom provider, and native +failures; warning diagnostics on success and error; cancellation before and +after acquisition; callback-error precedence; report writer errors, short +writes, and diagnostic writer errors with the established outcome fields. +Focused outcome tests and `GOTOOLCHAIN=local go test -count=1 ./...` passed from +`integrations/ckprompt`. + ## Stage 7: Extend the maintained composed consumer **Primary files:** `integrations/ckprompt/examples/composed/main.go`, its tests, diff --git a/integrations/ckprompt/validation_command_test.go b/integrations/ckprompt/validation_command_test.go index ec94ce0..b6fa886 100644 --- a/integrations/ckprompt/validation_command_test.go +++ b/integrations/ckprompt/validation_command_test.go @@ -11,6 +11,7 @@ import ( commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" + configkit "gitea.maximumdirect.net/eric/ConfigKit" promptkit "gitea.maximumdirect.net/eric/promptkit" "go.yaml.in/yaml/v4" ) @@ -314,3 +315,196 @@ func TestValidationCommandSuppliesZeroValueFactoriesIncludingNilPointers(t *test t.Fatalf("omitted factories = outcome %#v, input=%v state=%v", outcome, gotInput, gotState) } } + +func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T) { + client := &validationFailOnGenerate{} + engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client)) + loadCalls := 0 + suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + loadCalls++ + return engine, nil + }, nil) + app := newPromptTestApp(t, Command(suite)) + + var output bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 { + t.Fatalf("native request validation = %#v, output=%q", outcome, output.String()) + } + var report map[string]any + if err := yaml.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatalf("decode native confirmation %q: %v", output.String(), err) + } + if len(report) != 2 || report["prompt_id"] != "empty" || report["valid"] != true { + t.Fatalf("native confirmation contains unexpected data: %#v", report) + } + + for _, test := range []struct { + name string + args []string + want error + }{ + {name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "default"}, want: promptkit.ErrPromptRender}, + {name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "missing", "--input", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound}, + } { + t.Run(test.name, func(t *testing.T) { + output.Reset() + outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, test.want) || output.Len() != 0 { + t.Fatalf("native failure = %#v, output=%q, want %v", outcome, output.String(), test.want) + } + }) + } + if client.calls != 0 || loadCalls != 3 { + t.Fatalf("model calls=%d, engine loads=%d", client.calls, loadCalls) + } +} + +func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) { + warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { + _, err := io.WriteString(writer, "validation warning\n") + return err + }} + withWarning := func(diagnostics []commandkit.Diagnostic) bool { + for _, diagnostic := range diagnostics { + if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil { + var output bytes.Buffer + if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" { + return true + } + } + } + return false + } + var diagnostics bytes.Buffer + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { + return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil + }})) + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" { + t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String()) + } + + providerErr := errors.New("validation provider failed") + failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { + return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr + }})) + diagnostics.Reset() + outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\nvalidation provider failed\n" { + t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String()) + } + + builderErr := errors.New("request builder failed") + loaderCalls := 0 + builderApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + loaderCalls++ + return nil, nil + }, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) { + return promptkit.RunRequest{}, builderErr + }))) + outcome = builderApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, builderErr) || loaderCalls != 0 { + t.Fatalf("builder failure = %#v, loader calls=%d", outcome, loaderCalls) + } + + _, loadErr := configkit.Load[map[string]any](configkit.Bytes("validation command load failure", []byte("broken: ["))) + if loadErr == nil { + t.Fatal("invalid configuration unexpectedly loaded") + } + loadApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { + return nil, loadErr + }, nil))) + outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + var retained *configkit.LoadError + if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 { + t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome) + } +} + +func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) { + warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { + _, err := io.WriteString(writer, "validation warning\n") + return err + }} + calls := 0 + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { + calls++ + return commandkit.Acquired[struct{}]{}, nil + }})) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var output bytes.Buffer + outcome := app.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 0 || output.Len() != 0 { + t.Fatalf("pre-acquisition cancellation = %#v, calls=%d, output=%q", outcome, calls, output.String()) + } + + ctx, cancel = context.WithCancel(context.Background()) + canceledApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { + calls++ + cancel() + return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil + }})) + output.Reset() + outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 { + t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String()) + } + + providerErr := errors.New("provider error before cancellation") + ctx, cancel = context.WithCancel(context.Background()) + errorApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { + cancel() + return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr + }})) + outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) { + t.Fatalf("provider error after cancellation = %#v", outcome) + } +} + +func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testing.T) { + warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { + _, err := io.WriteString(writer, "validation warning\n") + return err + }} + app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { + return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil + }})) + writeErr := errors.New("validation report writer failed") + for _, test := range []struct { + name string + writer io.Writer + want error + }{ + {name: "writer error", writer: partialReportWriter{err: writeErr}, want: writeErr}, + {name: "short write", writer: partialReportWriter{}, want: io.ErrShortWrite}, + } { + t.Run(test.name, func(t *testing.T) { + var diagnosticOutput bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput}) + if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") { + t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String()) + } + }) + } + + diagnosticErr := errors.New("validation diagnostic writer failed") + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete { + t.Fatalf("diagnostic delivery = %#v", outcome) + } +} + +func withValidationWarning(diagnostics []commandkit.Diagnostic) bool { + for _, diagnostic := range diagnostics { + if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil { + var output bytes.Buffer + if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" { + return true + } + } + } + return false +}