334 lines
16 KiB
Go
334 lines
16 KiB
Go
package ckprompt
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"testing/fstest"
|
|
|
|
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
|
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
type validationFailOnGenerate struct{ calls int }
|
|
|
|
func (client *validationFailOnGenerate) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
client.calls++
|
|
return nil, errors.New("request validation must not generate")
|
|
}
|
|
|
|
type validationCancelArtifactReader struct{ cancel context.CancelFunc }
|
|
|
|
func (reader validationCancelArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
|
reader.cancel()
|
|
return nil, context.Canceled
|
|
}
|
|
|
|
func validationTestEngine(t *testing.T, configuration string, options ...promptkit.Option) *promptkit.Engine {
|
|
t.Helper()
|
|
inputs, err := promptkit.FilesystemInputs("request validation fixture", fstest.MapFS{
|
|
"config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)},
|
|
}, "config")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
Underlays: []configkit.Underlay{{Inputs: inputs}},
|
|
}}, options...)
|
|
if err != nil {
|
|
t.Fatalf("construct PromptKit validation fixture: %v", err)
|
|
}
|
|
return engine
|
|
}
|
|
|
|
func TestWithRequestValidationPreservesSuiteAndValidatesSeedWithoutBuilder(t *testing.T) {
|
|
type contextKey struct{}
|
|
type input struct{ value string }
|
|
type state struct{ value string }
|
|
|
|
ctx := context.WithValue(context.Background(), contextKey{}, "invocation")
|
|
engine := adapterEngine(t)
|
|
selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "config.yml", Set: true}}
|
|
inspect := func(context.Context, RequirementsRequest, input, state) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
|
|
return commandkit.Acquired[*promptkit.RequirementInspection]{}, nil
|
|
}
|
|
newInput := func() input { return input{value: "factory"} }
|
|
suite := Suite[input, state]{InspectRequirements: inspect, NewInput: newInput}
|
|
loaderCalls := 0
|
|
suite = WithRequestValidation(suite, func(gotContext context.Context, gotSelection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
if gotContext != ctx || gotContext.Value(contextKey{}) != "invocation" || gotSelection != selection || gotInput.value != "input" || gotState.value != "state" {
|
|
t.Errorf("loader args = (%v, %#v, %#v, %#v)", gotContext, gotSelection, gotInput, gotState)
|
|
}
|
|
return engine, nil
|
|
}, nil)
|
|
if suite.InspectRequirements == nil || suite.NewInput == nil || suite.ValidateRequest == nil {
|
|
t.Fatalf("composed suite lost a member or omitted validation: %#v", suite)
|
|
}
|
|
|
|
acquired, err := suite.ValidateRequest(ctx, ValidationRequest{
|
|
Selection: selection,
|
|
Request: promptkit.RunRequest{PromptID: "empty", ProfileID: "default"},
|
|
}, input{value: "input"}, state{value: "state"})
|
|
if err != nil {
|
|
t.Fatalf("validate seed request: %v", err)
|
|
}
|
|
if acquired.Value != (struct{}{}) || len(acquired.Diagnostics) != 0 || loaderCalls != 1 {
|
|
t.Fatalf("acquisition = %#v, loader calls = %d", acquired, loaderCalls)
|
|
}
|
|
}
|
|
|
|
func TestWithRequestValidationBuilderFailurePrecedesLoading(t *testing.T) {
|
|
type input struct{ value string }
|
|
type state struct{ value string }
|
|
buildErr := errors.New("cannot build request")
|
|
loadCalls, buildCalls := 0, 0
|
|
suite := WithRequestValidation(
|
|
Suite[input, state]{},
|
|
func(context.Context, ckconfig.Selection, input, state) (*promptkit.Engine, error) {
|
|
loadCalls++
|
|
return nil, errors.New("loader should not run")
|
|
},
|
|
func(ctx context.Context, request ValidationRequest, gotInput input, gotState state) (promptkit.RunRequest, error) {
|
|
buildCalls++
|
|
if ctx == nil || request.Request.PromptID != "seed" || gotInput.value != "input" || gotState.value != "state" {
|
|
t.Errorf("builder args = (%v, %#v, %#v, %#v)", ctx, request, gotInput, gotState)
|
|
}
|
|
return promptkit.RunRequest{}, buildErr
|
|
},
|
|
)
|
|
|
|
_, err := suite.ValidateRequest(context.Background(), ValidationRequest{Request: promptkit.RunRequest{PromptID: "seed"}}, input{value: "input"}, state{value: "state"})
|
|
if !errors.Is(err, buildErr) || buildCalls != 1 || loadCalls != 0 {
|
|
t.Fatalf("validation = (%v), builder calls = %d, loader calls = %d", err, buildCalls, loadCalls)
|
|
}
|
|
}
|
|
|
|
func TestWithRequestValidationRetainsNilLoaderIssueIndependently(t *testing.T) {
|
|
firstIssue := errors.New("existing adapter issue")
|
|
backing := make([]error, 1, 3)
|
|
backing[0] = firstIssue
|
|
existingCallback := func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{}, nil
|
|
}
|
|
suite := Suite[struct{}, struct{}]{ValidateRequest: existingCallback, issues: backing[:1]}
|
|
composed := WithRequestValidation(suite, nil, nil)
|
|
if composed.ValidateRequest == nil || len(composed.issues) != 2 {
|
|
t.Fatalf("nil loader composition = %#v", composed)
|
|
}
|
|
if _, err := composed.ValidateRequest(context.Background(), ValidationRequest{}, struct{}{}, struct{}{}); err != nil {
|
|
t.Fatalf("nil loader replaced the existing validation callback: %v", err)
|
|
}
|
|
if !errors.Is(composed.issues[0], firstIssue) || composed.issues[1] == nil || backing[:cap(backing)][1] != nil {
|
|
t.Fatalf("issues = %#v, original backing = %#v", composed.issues, backing[:cap(backing)])
|
|
}
|
|
|
|
composed.ValidateRequest = nil
|
|
if err := newPromptTestApp(t).Register(Command(composed)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") {
|
|
t.Fatalf("installation did not report retained issue after capabilities were cleared: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestWithRequestValidationDelegatesNativeOutcomesWithoutModelCalls(t *testing.T) {
|
|
client := &validationFailOnGenerate{}
|
|
engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client))
|
|
selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "selected.yml", Set: true}}
|
|
suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(_ context.Context, gotSelection ckconfig.Selection, _, _ struct{}) (*promptkit.Engine, error) {
|
|
if gotSelection != selection {
|
|
t.Errorf("loader selection = %#v, want %#v", gotSelection, selection)
|
|
}
|
|
return engine, nil
|
|
}, nil)
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
request promptkit.RunRequest
|
|
want error
|
|
}{
|
|
{name: "success", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}},
|
|
{name: "missing required input", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default"}, want: promptkit.ErrPromptRender},
|
|
{name: "invalid profile selection", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "missing", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}, want: promptkit.ErrProfileNotFound},
|
|
{name: "unloadable artifact", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("missing-topic.txt")}}, want: promptkit.ErrArtifactLoad},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
nativeErr := engine.ValidateRequest(context.Background(), test.request)
|
|
acquired, adapterErr := suite.ValidateRequest(context.Background(), ValidationRequest{Selection: selection, Request: test.request}, struct{}{}, struct{}{})
|
|
if test.want == nil {
|
|
if nativeErr != nil || adapterErr != nil {
|
|
t.Fatalf("native error = %v, adapter error = %v", nativeErr, adapterErr)
|
|
}
|
|
if acquired.Value != (struct{}{}) {
|
|
t.Fatalf("successful acquisition = %#v", acquired)
|
|
}
|
|
return
|
|
}
|
|
if !errors.Is(nativeErr, test.want) || !errors.Is(adapterErr, test.want) {
|
|
t.Fatalf("native error = %v, adapter error = %v, want both to match %v", nativeErr, adapterErr, test.want)
|
|
}
|
|
})
|
|
}
|
|
if client.calls != 0 {
|
|
t.Fatalf("request validation made %d model calls", client.calls)
|
|
}
|
|
}
|
|
|
|
func TestWithRequestValidationBuilderOutputReachesNativeValidation(t *testing.T) {
|
|
type contextKey struct{}
|
|
type input struct{ campaign string }
|
|
type state struct{ token string }
|
|
|
|
client := &validationFailOnGenerate{}
|
|
engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client))
|
|
ctx := context.WithValue(context.Background(), contextKey{}, "validation invocation")
|
|
selection := ckconfig.Selection{Overlay: commandkit.Supplied[string]{Value: "campaign.yml", Set: true}}
|
|
seed := promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Vars: map[string]string{"topic": "seed topic"}}
|
|
wantInput, wantState := input{campaign: "spring"}, state{token: "shared"}
|
|
builderCalls, loaderCalls := 0, 0
|
|
suite := WithRequestValidation(Suite[input, state]{}, func(gotContext context.Context, gotSelection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || gotSelection != selection || gotInput != wantInput || gotState != wantState {
|
|
t.Errorf("loader args = (%v, %#v, %#v, %#v)", gotContext, gotSelection, gotInput, gotState)
|
|
}
|
|
return engine, nil
|
|
}, func(gotContext context.Context, request ValidationRequest, gotInput input, gotState state) (promptkit.RunRequest, error) {
|
|
builderCalls++
|
|
if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || request.Selection != selection || request.Request.PromptID != seed.PromptID || request.Request.Vars["topic"] != "seed topic" || gotInput != wantInput || gotState != wantState {
|
|
t.Errorf("builder args = (%v, %#v, %#v, %#v)", gotContext, request, gotInput, gotState)
|
|
}
|
|
built := request.Request
|
|
built.Inputs = map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("required artifact from builder")}
|
|
built.SessionID = "builder-session"
|
|
return built, nil
|
|
})
|
|
|
|
acquired, err := suite.ValidateRequest(ctx, ValidationRequest{Selection: selection, Request: seed}, wantInput, wantState)
|
|
if err != nil || acquired.Value != (struct{}{}) || builderCalls != 1 || loaderCalls != 1 {
|
|
t.Fatalf("built request validation = (%#v, %v), builder calls = %d, loader calls = %d", acquired, err, builderCalls, loaderCalls)
|
|
}
|
|
if client.calls != 0 {
|
|
t.Fatalf("request validation made %d model calls", client.calls)
|
|
}
|
|
}
|
|
|
|
func TestWithRequestValidationRetainsNilEngineAndLoadExplanation(t *testing.T) {
|
|
request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "empty", ProfileID: "default"}}
|
|
nilEngineSuite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
return nil, nil
|
|
}, nil)
|
|
acquired, err := nilEngineSuite.ValidateRequest(context.Background(), request, struct{}{}, struct{}{})
|
|
if err == nil || !strings.Contains(err.Error(), "nil engine") || len(acquired.Diagnostics) != 0 {
|
|
t.Fatalf("nil engine result = (%#v, %v)", acquired, err)
|
|
}
|
|
|
|
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid validation configuration", []byte("broken: [")))
|
|
if loadErr == nil {
|
|
t.Fatal("invalid configuration unexpectedly loaded")
|
|
}
|
|
loadFailure := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
return nil, loadErr
|
|
}, nil)
|
|
acquired, err = loadFailure.ValidateRequest(context.Background(), request, struct{}{}, struct{}{})
|
|
var retained *configkit.LoadError
|
|
if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Explanation.Details) == 0 {
|
|
t.Fatalf("load failure = (%#v, %v), want original ConfigKit error and terminal context", acquired, err)
|
|
}
|
|
}
|
|
|
|
func TestWithRequestValidationCancellationBoundaries(t *testing.T) {
|
|
engine := adapterEngine(t)
|
|
request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "missing", ProfileID: "default"}}
|
|
|
|
t.Run("before builder", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
builderCalls, loaderCalls := 0, 0
|
|
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
return engine, nil
|
|
}, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) {
|
|
builderCalls++
|
|
return request.Request, nil
|
|
})
|
|
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
|
|
if !errors.Is(err, context.Canceled) || builderCalls != 0 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 {
|
|
t.Fatalf("pre-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("after builder", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
builderCalls, loaderCalls := 0, 0
|
|
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
return engine, nil
|
|
}, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) {
|
|
builderCalls++
|
|
cancel()
|
|
return request.Request, nil
|
|
})
|
|
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
|
|
if !errors.Is(err, context.Canceled) || builderCalls != 1 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 {
|
|
t.Fatalf("post-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("during load retains load failure", func(t *testing.T) {
|
|
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("cancelled validation configuration", []byte("broken: [")))
|
|
if loadErr == nil {
|
|
t.Fatal("invalid configuration unexpectedly loaded")
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
loaderCalls := 0
|
|
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
cancel()
|
|
return nil, loadErr
|
|
}, nil)
|
|
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
|
|
var retained *configkit.LoadError
|
|
if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Explanation.Details) == 0 || loaderCalls != 1 {
|
|
t.Fatalf("cancelled load = (%#v, %v), loader calls=%d", acquired, err, loaderCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("after successful load skips native validation", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
loaderCalls := 0
|
|
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
cancel()
|
|
return engine, nil
|
|
}, nil)
|
|
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
|
|
if !errors.Is(err, context.Canceled) || loaderCalls != 1 || len(acquired.Diagnostics) != 0 {
|
|
t.Fatalf("post-load cancellation = (%#v, %v), loader calls=%d", acquired, err, loaderCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("native artifact cancellation retains PromptKit category", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
client := &validationFailOnGenerate{}
|
|
engine := validationTestEngine(t, adapterPrompt, promptkit.WithArtifactReader(validationCancelArtifactReader{cancel: cancel}), promptkit.WithLLMClient(client))
|
|
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
|
|
return engine, nil
|
|
}, nil)
|
|
request := ValidationRequest{Request: promptkit.RunRequest{
|
|
PromptID: "briefing",
|
|
ProfileID: "default",
|
|
Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("cancel-topic.txt")},
|
|
}}
|
|
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
|
|
if !errors.Is(err, promptkit.ErrArtifactLoad) || len(acquired.Diagnostics) != 0 {
|
|
t.Fatalf("native artifact cancellation = (%#v, %v), want PromptKit artifact-load category", acquired, err)
|
|
}
|
|
if client.calls != 0 {
|
|
t.Fatalf("request validation made %d model calls", client.calls)
|
|
}
|
|
})
|
|
}
|