511 lines
28 KiB
Go
511 lines
28 KiB
Go
package ckprompt
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
|
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
|
"go.yaml.in/yaml/v4"
|
|
)
|
|
|
|
func TestValidationCommandComposesWithRequirementsShow(t *testing.T) {
|
|
type input struct{ campaign commandkit.Supplied[string] }
|
|
type state struct{ sequence int }
|
|
campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Usage: "Campaign selector."}, (*string)(nil), parseDefinitionString,
|
|
func(target *input, value commandkit.Supplied[string]) { target.campaign = value })
|
|
wantSelection := ckconfig.Selection{
|
|
Config: commandkit.Supplied[string]{Value: "config.yml", Set: true},
|
|
Overlay: commandkit.Supplied[string]{Value: "overlay.yml", Set: true},
|
|
}
|
|
wantInput := input{campaign: commandkit.Supplied[string]{Value: "spring", Set: true}}
|
|
var received ValidationRequest
|
|
var receivedInput input
|
|
validateCalls := 0
|
|
suite := Suite[input, state]{
|
|
NewInput: func() input { return input{} },
|
|
NewState: func() state { return state{sequence: 1} },
|
|
Selectors: []commandkit.Flag[input]{campaignFlag},
|
|
CheckInput: func(selection ckconfig.Selection, value input) error {
|
|
if selection != wantSelection || !reflect.DeepEqual(value, wantInput) {
|
|
t.Errorf("checked input = (%#v, %#v)", selection, value)
|
|
}
|
|
return nil
|
|
},
|
|
InspectRequirements: func(context.Context, RequirementsRequest, input, state) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
|
|
return commandkit.Acquired[*promptkit.RequirementInspection]{Value: &promptkit.RequirementInspection{PromptID: "briefing"}}, nil
|
|
},
|
|
ValidateRequest: func(_ context.Context, request ValidationRequest, value input, gotState state) (commandkit.Acquired[struct{}], error) {
|
|
validateCalls++
|
|
received = request
|
|
receivedInput = value
|
|
if gotState.sequence != 1 {
|
|
t.Errorf("state = %#v", gotState)
|
|
}
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
},
|
|
}
|
|
app := newPromptTestApp(t, Command(suite))
|
|
|
|
var help bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"help", "promptkit", "requirements"}, commandkit.Streams{Out: &help, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || !bytes.Contains(help.Bytes(), []byte("show")) || !bytes.Contains(help.Bytes(), []byte("validate")) {
|
|
t.Fatalf("requirements help = (%#v, %q)", outcome, help.String())
|
|
}
|
|
|
|
var output bytes.Buffer
|
|
outcome = app.Execute(context.Background(), []string{
|
|
"promptkit", "requirements", "validate", "briefing",
|
|
"--config", "config.yml", "--overlay", "overlay.yml", "--campaign", "spring",
|
|
"--version", "v2", "--profile", "editorial", "--session-id", "session-1",
|
|
"--var", "topic=roadmap", "--input", "notes=fixtures/notes=final.txt", "--format", "yaml",
|
|
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || validateCalls != 1 {
|
|
t.Fatalf("validation outcome = (%#v, %q), calls=%d", outcome, output.String(), validateCalls)
|
|
}
|
|
if received.Selection != wantSelection || received.Request.PromptID != "briefing" || received.Request.PromptVersion != "v2" || received.Request.ProfileID != "editorial" || received.Request.SessionID != "session-1" {
|
|
t.Errorf("validation request = %#v", received)
|
|
}
|
|
if received.Request.Vars["topic"] != "roadmap" || len(received.Request.Vars) != 1 {
|
|
t.Errorf("request vars = %#v", received.Request.Vars)
|
|
}
|
|
if ref, ok := received.Request.Inputs["notes"]; !ok || ref != promptkit.File("fixtures/notes=final.txt") || len(received.Request.Inputs) != 1 {
|
|
t.Errorf("request inputs = %#v", received.Request.Inputs)
|
|
}
|
|
if !reflect.DeepEqual(receivedInput, wantInput) {
|
|
t.Errorf("provider input = %#v, want %#v", receivedInput, wantInput)
|
|
}
|
|
|
|
var report map[string]any
|
|
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
|
|
t.Fatalf("decode validation report %q: %v", output.String(), err)
|
|
}
|
|
if len(report) != 2 || report["prompt_id"] != "briefing" || report["valid"] != true {
|
|
t.Fatalf("validation confirmation = %#v", report)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandIsAvailableIndependentlyAndOwnsItsFamily(t *testing.T) {
|
|
validationOnly := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
}}))
|
|
var help bytes.Buffer
|
|
outcome := validationOnly.Execute(context.Background(), []string{"help", "promptkit", "requirements"}, commandkit.Streams{Out: &help, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "show") {
|
|
t.Fatalf("validation-only availability = (%#v, %q)", outcome, help.String())
|
|
}
|
|
|
|
declaration := Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
}})
|
|
other := commandkit.Group("promptkit", "An application command tree.", commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "child", Summary: "A child."}))
|
|
app := newPromptTestApp(t, other)
|
|
if err := app.Register(declaration); err == nil {
|
|
t.Fatal("validation-only command installed over an existing PromptKit namespace")
|
|
}
|
|
if err := newPromptTestApp(t, declaration).Register(other); err == nil {
|
|
t.Fatal("later application command installed over validation-only PromptKit namespace")
|
|
}
|
|
if _, err := commandkit.New(commandkit.Identity{Name: "nested"}, commandkit.Group("outer", "Outer group.", declaration)); err == nil {
|
|
t.Fatal("validation-only root family was accepted beneath another group")
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandConstructionAndCaptureLifecycle(t *testing.T) {
|
|
type input struct{ campaign commandkit.Supplied[string] }
|
|
type state struct{}
|
|
newInputCalls, newStateCalls, checkCalls := 0, 0, 0
|
|
buildCalls, loadCalls := 0, 0
|
|
engine := adapterEngine(t)
|
|
oldSelector := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Required: true}, (*string)(nil), parseDefinitionString,
|
|
func(target *input, value commandkit.Supplied[string]) { target.campaign = value })
|
|
newSelector := commandkit.ValueFlag(commandkit.FlagMeta{Name: "replacement"}, (*string)(nil), parseDefinitionString,
|
|
func(*input, commandkit.Supplied[string]) {})
|
|
suite := WithRequestValidation(Suite[input, state]{
|
|
NewInput: func() input { newInputCalls++; return input{} },
|
|
NewState: func() state { newStateCalls++; return state{} },
|
|
Selectors: []commandkit.Flag[input]{oldSelector},
|
|
CheckInput: func(_ ckconfig.Selection, value input) error {
|
|
checkCalls++
|
|
if !value.campaign.Set || value.campaign.Value != "spring" {
|
|
return errors.New("captured campaign selector was not applied")
|
|
}
|
|
return nil
|
|
},
|
|
}, func(context.Context, ckconfig.Selection, input, state) (*promptkit.Engine, error) {
|
|
loadCalls++
|
|
return engine, nil
|
|
}, func(_ context.Context, request ValidationRequest, _ input, _ state) (promptkit.RunRequest, error) {
|
|
buildCalls++
|
|
return request.Request, nil
|
|
})
|
|
declaration := Command(suite)
|
|
if newInputCalls != 0 || newStateCalls != 0 || checkCalls != 0 || buildCalls != 0 || loadCalls != 0 {
|
|
t.Fatal("command construction invoked application or adapter callbacks")
|
|
}
|
|
|
|
// Command captured the original callback and binding metadata.
|
|
suite.ValidateRequest = func(context.Context, ValidationRequest, input, state) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{}, errors.New("replacement callback was captured")
|
|
}
|
|
suite.Selectors[0] = newSelector
|
|
app := newPromptTestApp(t, declaration)
|
|
for _, args := range [][]string{{"help", "promptkit", "requirements", "validate"}, {"version"}} {
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Fatalf("construction lifecycle command %v = %#v", args, outcome)
|
|
}
|
|
}
|
|
if newInputCalls != 0 || newStateCalls != 0 || checkCalls != 0 || buildCalls != 0 || loadCalls != 0 {
|
|
t.Fatal("installation, help, or version invoked application or adapter callbacks")
|
|
}
|
|
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || newInputCalls != 1 || newStateCalls != 1 || checkCalls != 1 || buildCalls != 1 || loadCalls != 1 {
|
|
t.Fatalf("captured command = %#v, callback counts input/state/check/build/load=%d/%d/%d/%d/%d", outcome, newInputCalls, newStateCalls, checkCalls, buildCalls, loadCalls)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandPreservesPresenceAndRepeatedAssignmentsPerInvocation(t *testing.T) {
|
|
type input struct{ campaign commandkit.Supplied[string] }
|
|
type state struct{ sequence int }
|
|
campaignFlag := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign"}, (*string)(nil), parseDefinitionString,
|
|
func(target *input, value commandkit.Supplied[string]) { target.campaign = value })
|
|
var requests []ValidationRequest
|
|
var selectedInputs []input
|
|
newInputCalls, checkCalls, newStateCalls := 0, 0, 0
|
|
suite := Suite[input, state]{
|
|
NewInput: func() input { newInputCalls++; return input{} },
|
|
NewState: func() state { newStateCalls++; return state{sequence: newStateCalls} },
|
|
Selectors: []commandkit.Flag[input]{campaignFlag},
|
|
CheckInput: func(ckconfig.Selection, input) error { checkCalls++; return nil },
|
|
ValidateRequest: func(_ context.Context, request ValidationRequest, value input, _ state) (commandkit.Acquired[struct{}], error) {
|
|
requests = append(requests, request)
|
|
selectedInputs = append(selectedInputs, value)
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
},
|
|
}
|
|
app := newPromptTestApp(t, Command(suite))
|
|
|
|
firstArgs := []string{
|
|
"promptkit", "requirements", "validate", "briefing", "--config", "", "--overlay", "",
|
|
"--version", "", "--profile", "", "--session-id", "", "--campaign", "spring",
|
|
"--var", "=", "--var", "phrase= two words, three ", "--var", "duplicate=first", "--var", "duplicate=last=value",
|
|
"--input", "=", "--input", "file= path, with commas=and=equals ", "--input", "duplicate=first.txt", "--input", "duplicate=last=path.txt",
|
|
}
|
|
outcome := app.Execute(context.Background(), firstArgs, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || len(requests) != 1 {
|
|
t.Fatalf("first repeated assignment invocation = %#v, requests=%d", outcome, len(requests))
|
|
}
|
|
first := requests[0]
|
|
if first.Selection.Config.Value != "" || !first.Selection.Config.Set || first.Selection.Overlay.Value != "" || !first.Selection.Overlay.Set {
|
|
t.Errorf("explicit empty selection = %#v", first.Selection)
|
|
}
|
|
if first.Request.PromptVersion != "" || first.Request.ProfileID != "" || first.Request.SessionID != "" {
|
|
t.Errorf("empty scalar values = %#v", first.Request)
|
|
}
|
|
wantVars := map[string]string{"": "", "phrase": " two words, three ", "duplicate": "last=value"}
|
|
if !reflect.DeepEqual(first.Request.Vars, wantVars) {
|
|
t.Errorf("request vars = %#v, want %#v", first.Request.Vars, wantVars)
|
|
}
|
|
wantInputs := map[string]promptkit.ArtifactRef{
|
|
"": promptkit.File(""),
|
|
"file": promptkit.File(" path, with commas=and=equals "),
|
|
"duplicate": promptkit.File("last=path.txt"),
|
|
}
|
|
if !reflect.DeepEqual(first.Request.Inputs, wantInputs) {
|
|
t.Errorf("request inputs = %#v, want %#v", first.Request.Inputs, wantInputs)
|
|
}
|
|
if !selectedInputs[0].campaign.Set || selectedInputs[0].campaign.Value != "spring" {
|
|
t.Errorf("first selector input = %#v", selectedInputs[0])
|
|
}
|
|
|
|
outcome = app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || len(requests) != 2 {
|
|
t.Fatalf("second invocation = %#v, requests=%d", outcome, len(requests))
|
|
}
|
|
second := requests[1]
|
|
if second.Selection.Config.Set || second.Selection.Overlay.Set || second.Request.PromptVersion != "" || second.Request.ProfileID != "" || second.Request.SessionID != "" || len(second.Request.Vars) != 0 || len(second.Request.Inputs) != 0 {
|
|
t.Errorf("first invocation values leaked into second request: %#v", second)
|
|
}
|
|
if selectedInputs[1].campaign.Set || selectedInputs[1].campaign.Value != "" {
|
|
t.Errorf("first selector leaked into second input: %#v", selectedInputs[1])
|
|
}
|
|
if newInputCalls != 2 || checkCalls != 2 || newStateCalls != 2 {
|
|
t.Errorf("per-invocation counts input/check/state = %d/%d/%d", newInputCalls, checkCalls, newStateCalls)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandUsageAndInputFailuresStopProviderWork(t *testing.T) {
|
|
type input struct{ campaign commandkit.Supplied[string] }
|
|
selector := commandkit.ValueFlag(commandkit.FlagMeta{Name: "campaign", Required: true}, (*string)(nil), parseDefinitionString,
|
|
func(target *input, value commandkit.Supplied[string]) { target.campaign = value })
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
}{
|
|
{name: "missing operand", args: []string{"promptkit", "requirements", "validate"}},
|
|
{name: "extra operand", args: []string{"promptkit", "requirements", "validate", "briefing", "extra", "--campaign", "spring"}},
|
|
{name: "unsupported format", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--format", "text"}},
|
|
{name: "unknown flag", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--unknown", "value"}},
|
|
{name: "malformed variable before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--var", "broken", "--var", "mode=full"}},
|
|
{name: "malformed input before valid assignment", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "broken", "--input", "notes=file.txt"}},
|
|
{name: "missing required selector", args: []string{"promptkit", "requirements", "validate", "briefing"}},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
newInputCalls, checkCalls, newStateCalls, providerCalls := 0, 0, 0, 0
|
|
suite := Suite[input, struct{}]{
|
|
NewInput: func() input { newInputCalls++; return input{} },
|
|
NewState: func() struct{} { newStateCalls++; return struct{}{} },
|
|
Selectors: []commandkit.Flag[input]{selector},
|
|
CheckInput: func(ckconfig.Selection, input) error { checkCalls++; return nil },
|
|
ValidateRequest: func(context.Context, ValidationRequest, input, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
providerCalls++
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
},
|
|
}
|
|
outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), test.args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
wantInputCalls := 0
|
|
if test.name == "missing required selector" {
|
|
// prepareSuiteInput initializes the caller input before core validates
|
|
// captured required bindings; CheckInput and provider work still stop.
|
|
wantInputCalls = 1
|
|
}
|
|
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 || newInputCalls != wantInputCalls || checkCalls != 0 || newStateCalls != 0 || providerCalls != 0 {
|
|
t.Fatalf("preflight failure = %#v, input/check/state/provider=%d/%d/%d/%d", outcome, newInputCalls, checkCalls, newStateCalls, providerCalls)
|
|
}
|
|
})
|
|
}
|
|
|
|
newStateCalls, providerCalls := 0, 0
|
|
suite := Suite[input, struct{}]{
|
|
NewInput: func() input { return input{} },
|
|
NewState: func() struct{} { newStateCalls++; return struct{}{} },
|
|
Selectors: []commandkit.Flag[input]{selector},
|
|
CheckInput: func(ckconfig.Selection, input) error { return errors.New("input rejected") },
|
|
ValidateRequest: func(context.Context, ValidationRequest, input, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
providerCalls++
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
},
|
|
}
|
|
outcome := newPromptTestApp(t, Command(suite)).Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeUsage || newStateCalls != 0 || providerCalls != 0 {
|
|
t.Fatalf("input check failure reached state/provider: %#v state=%d provider=%d", outcome, newStateCalls, providerCalls)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandSuppliesZeroValueFactoriesIncludingNilPointers(t *testing.T) {
|
|
var gotInput *int
|
|
var gotState *int
|
|
suite := Suite[*int, *int]{ValidateRequest: func(_ context.Context, _ ValidationRequest, input, state *int) (commandkit.Acquired[struct{}], error) {
|
|
gotInput, gotState = input, state
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
}}
|
|
app := newPromptTestApp(t, Command(suite))
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || gotInput != nil || gotState != nil {
|
|
t.Fatalf("omitted factories = outcome %#v, input=%v state=%v", outcome, gotInput, gotState)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T) {
|
|
client := &validationFailOnGenerate{}
|
|
engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client))
|
|
loadCalls := 0
|
|
suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
loadCalls++
|
|
return engine, nil
|
|
}, nil)
|
|
app := newPromptTestApp(t, Command(suite))
|
|
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "empty", "--profile", "default"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
|
|
t.Fatalf("native request validation = %#v, output=%q", outcome, output.String())
|
|
}
|
|
var report map[string]any
|
|
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
|
|
t.Fatalf("decode native confirmation %q: %v", output.String(), err)
|
|
}
|
|
if len(report) != 2 || report["prompt_id"] != "empty" || report["valid"] != true {
|
|
t.Fatalf("native confirmation contains unexpected data: %#v", report)
|
|
}
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
want error
|
|
}{
|
|
{name: "missing required input", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "default"}, want: promptkit.ErrPromptRender},
|
|
{name: "invalid profile selection", args: []string{"promptkit", "requirements", "validate", "briefing", "--profile", "missing", "--input", "topic=unused.txt"}, want: promptkit.ErrProfileNotFound},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
output.Reset()
|
|
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, test.want) || output.Len() != 0 {
|
|
t.Fatalf("native failure = %#v, output=%q, want %v", outcome, output.String(), test.want)
|
|
}
|
|
})
|
|
}
|
|
if client.calls != 0 || loadCalls != 3 {
|
|
t.Fatalf("model calls=%d, engine loads=%d", client.calls, loadCalls)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) {
|
|
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
|
|
_, err := io.WriteString(writer, "validation warning\n")
|
|
return err
|
|
}}
|
|
withWarning := func(diagnostics []commandkit.Diagnostic) bool {
|
|
for _, diagnostic := range diagnostics {
|
|
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
|
|
var output bytes.Buffer
|
|
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
var diagnostics bytes.Buffer
|
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
|
|
}}))
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" {
|
|
t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String())
|
|
}
|
|
|
|
providerErr := errors.New("validation provider failed")
|
|
failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
|
|
}}))
|
|
diagnostics.Reset()
|
|
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
|
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\nvalidation provider failed\n" {
|
|
t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String())
|
|
}
|
|
|
|
builderErr := errors.New("request builder failed")
|
|
loaderCalls := 0
|
|
builderApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
return nil, nil
|
|
}, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
|
|
return promptkit.RunRequest{}, builderErr
|
|
})))
|
|
outcome = builderApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, builderErr) || loaderCalls != 0 {
|
|
t.Fatalf("builder failure = %#v, loader calls=%d", outcome, loaderCalls)
|
|
}
|
|
|
|
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("validation command load failure", []byte("broken: [")))
|
|
if loadErr == nil {
|
|
t.Fatal("invalid configuration unexpectedly loaded")
|
|
}
|
|
loadApp := newPromptTestApp(t, Command(WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
return nil, loadErr
|
|
}, nil)))
|
|
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
var retained *configkit.LoadError
|
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 {
|
|
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) {
|
|
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
|
|
_, err := io.WriteString(writer, "validation warning\n")
|
|
return err
|
|
}}
|
|
calls := 0
|
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
calls++
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
}}))
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 0 || output.Len() != 0 {
|
|
t.Fatalf("pre-acquisition cancellation = %#v, calls=%d, output=%q", outcome, calls, output.String())
|
|
}
|
|
|
|
ctx, cancel = context.WithCancel(context.Background())
|
|
canceledApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
calls++
|
|
cancel()
|
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
|
|
}}))
|
|
output.Reset()
|
|
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 {
|
|
t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String())
|
|
}
|
|
|
|
providerErr := errors.New("provider error before cancellation")
|
|
ctx, cancel = context.WithCancel(context.Background())
|
|
errorApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
cancel()
|
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
|
|
}}))
|
|
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) {
|
|
t.Fatalf("provider error after cancellation = %#v", outcome)
|
|
}
|
|
}
|
|
|
|
func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testing.T) {
|
|
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
|
|
_, err := io.WriteString(writer, "validation warning\n")
|
|
return err
|
|
}}
|
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
|
|
}}))
|
|
writeErr := errors.New("validation report writer failed")
|
|
for _, test := range []struct {
|
|
name string
|
|
writer io.Writer
|
|
want error
|
|
}{
|
|
{name: "writer error", writer: partialReportWriter{err: writeErr}, want: writeErr},
|
|
{name: "short write", writer: partialReportWriter{}, want: io.ErrShortWrite},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var diagnosticOutput bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
|
|
if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
|
|
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
diagnosticErr := errors.New("validation diagnostic writer failed")
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
|
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete {
|
|
t.Fatalf("diagnostic delivery = %#v", outcome)
|
|
}
|
|
}
|
|
|
|
func withValidationWarning(diagnostics []commandkit.Diagnostic) bool {
|
|
for _, diagnostic := range diagnostics {
|
|
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
|
|
var output bytes.Buffer
|
|
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|