base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Verify validation outcomes and report delivery

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-12 22:20:07 +00:00
parent 346bcaeddf
commit 9e6431be63
2 changed files with 205 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
docs/roadmap/implementation.md
View File

@@ -1,6 +1,6 @@
# PromptKit request validation implementation plan
Status: Stages 1–5 complete; implementation is ongoing.
Status: Stages 1–6 complete; implementation is ongoing.
## Authority, scope, and baseline
@@ -356,6 +356,16 @@ its maintained composed consumer tests.
**Complete when:** assembled validation follows native semantics and existing
outcome/report delivery contracts under representative successes and failures.
**Completed:** 2026-10-12. Exercised the composed command with real PromptKit
success, missing required input, and invalid profile selection; success emits
only `prompt_id` and `valid`, and the fail-on-call model client remains unused.
Command tests cover builder, ConfigKit loader, custom provider, and native
failures; warning diagnostics on success and error; cancellation before and
after acquisition; callback-error precedence; report writer errors, short
writes, and diagnostic writer errors with the established outcome fields.
Focused outcome tests and `GOTOOLCHAIN=local go test -count=1 ./...` passed from
`integrations/ckprompt`.
## Stage 7: Extend the maintained composed consumer
**Primary files:** `integrations/ckprompt/examples/composed/main.go`, its tests,

194
integrations/ckprompt/validation_command_test.go
View File

@@ -11,6 +11,7 @@ import (
commandkit "gitea.maximumdirect.net/eric/CommandKit"
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
promptkit "gitea.maximumdirect.net/eric/promptkit"
"go.yaml.in/yaml/v4"
)
@@ -314,3 +315,196 @@ func TestValidationCommandSuppliesZeroValueFactoriesIncludingNilPointers(t *test
t.Fatalf("omitted factories = outcome %#v, input=%v state=%v", outcome, gotInput, gotState)
}
}
func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T) {
client := &validationFailOnGenerate{}
engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client))
loadCalls := 0
suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
loadCalls++
return engine, nil
}, nil)
app := newPromptTestApp(t, Command(suite))
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
t.Fatalf("native request validation = %#v, output=%q", outcome, output.String())
}
var report map[string]any
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
t.Fatalf("decode native confirmation %q: %v", output.String(), err)
}
if len(report) != 2 || report["prompt_id"] != "empty" || report["valid"] != true {
t.Fatalf("native confirmation contains unexpected data: %#v", report)
}
for _, test := range []struct {
name string
args []string
want error
}{
{name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "default"}, want: promptkit.ErrPromptRender},
{name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "missing", "--input", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound},
} {
t.Run(test.name, func(t *testing.T) {
output.Reset()
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, test.want) || output.Len() != 0 {
t.Fatalf("native failure = %#v, output=%q, want %v", outcome, output.String(), test.want)
}
})
}
if client.calls != 0 || loadCalls != 3 {
t.Fatalf("model calls=%d, engine loads=%d", client.calls, loadCalls)
}
}
func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) {
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "validation warning\n")
return err
}}
withWarning := func(diagnostics []commandkit.Diagnostic) bool {
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
var output bytes.Buffer
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
return true
}
}
}
return false
}
var diagnostics bytes.Buffer
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
}}))
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" {
t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String())
}
providerErr := errors.New("validation provider failed")
failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
}}))
diagnostics.Reset()
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\nvalidation provider failed\n" {
t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String())
}
builderErr := errors.New("request builder failed")
loaderCalls := 0
builderApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
loaderCalls++
return nil, nil
}, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
return promptkit.RunRequest{}, builderErr
})))
outcome = builderApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, builderErr) || loaderCalls != 0 {
t.Fatalf("builder failure = %#v, loader calls=%d", outcome, loaderCalls)
}
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("validation command load failure", []byte("broken: [")))
if loadErr == nil {
t.Fatal("invalid configuration unexpectedly loaded")
}
loadApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return nil, loadErr
}, nil)))
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
var retained *configkit.LoadError
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 {
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
}
}
func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) {
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "validation warning\n")
return err
}}
calls := 0
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
calls++
return commandkit.Acquired[struct{}]{}, nil
}}))
ctx, cancel := context.WithCancel(context.Background())
cancel()
var output bytes.Buffer
outcome := app.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 0 || output.Len() != 0 {
t.Fatalf("pre-acquisition cancellation = %#v, calls=%d, output=%q", outcome, calls, output.String())
}
ctx, cancel = context.WithCancel(context.Background())
canceledApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
calls++
cancel()
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
}}))
output.Reset()
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 {
t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String())
}
providerErr := errors.New("provider error before cancellation")
ctx, cancel = context.WithCancel(context.Background())
errorApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
cancel()
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
}}))
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) {
t.Fatalf("provider error after cancellation = %#v", outcome)
}
}
func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testing.T) {
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "validation warning\n")
return err
}}
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
}}))
writeErr := errors.New("validation report writer failed")
for _, test := range []struct {
name string
writer io.Writer
want error
}{
{name: "writer error", writer: partialReportWriter{err: writeErr}, want: writeErr},
{name: "short write", writer: partialReportWriter{}, want: io.ErrShortWrite},
} {
t.Run(test.name, func(t *testing.T) {
var diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
}
})
}
diagnosticErr := errors.New("validation diagnostic writer failed")
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete {
t.Fatalf("diagnostic delivery = %#v", outcome)
}
}
func withValidationWarning(diagnostics []commandkit.Diagnostic) bool {
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
var output bytes.Buffer
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
return true
}
}
}
return false
}