package ckprompt import ( "context" "errors" "strings" "testing" "testing/fstest" commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" configkit "gitea.maximumdirect.net/eric/ConfigKit" promptkit "gitea.maximumdirect.net/eric/promptkit" ) type validationFailOnGenerate struct{ calls int } func (client *validationFailOnGenerate) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.calls++ return nil, errors.New("request validation must not generate") } type validationCancelArtifactReader struct{ cancel context.CancelFunc } func (reader validationCancelArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { reader.cancel() return nil, context.Canceled } func validationTestEngine(t *testing.T, configuration string, options ...promptkit.Option) *promptkit.Engine { t.Helper() inputs, err := promptkit.FilesystemInputs("request validation fixture", fstest.MapFS{ "config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)}, }, "config") if err != nil { t.Fatal(err) } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: inputs}}, }}, options...) if err != nil { t.Fatalf("construct PromptKit validation fixture: %v", err) } return engine } func TestWithRequestValidationPreservesSuiteAndValidatesSeedWithoutBuilder(t *testing.T) { type contextKey struct{} type input struct{ value string } type state struct{ value string } ctx := context.WithValue(context.Background(), contextKey{}, "invocation") engine := adapterEngine(t) selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "config.yml", Set: true}} inspect := func(context.Context, RequirementsRequest, input, state) (commandkit.Acquired[*promptkit.RequirementInspection], error) { return commandkit.Acquired[*promptkit.RequirementInspection]{}, nil } newInput := func() input { return input{value: "factory"} } suite := Suite[input, state]{InspectRequirements: inspect, NewInput: newInput} loaderCalls := 0 suite = WithRequestValidation(suite, func(gotContext context.Context, gotSelection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) { loaderCalls++ if gotContext != ctx || gotContext.Value(contextKey{}) != "invocation" || gotSelection != selection || gotInput.value != "input" || gotState.value != "state" { t.Errorf("loader args = (%v, %#v, %#v, %#v)", gotContext, gotSelection, gotInput, gotState) } return engine, nil }, nil) if suite.InspectRequirements == nil || suite.NewInput == nil || suite.ValidateRequest == nil { t.Fatalf("composed suite lost a member or omitted validation: %#v", suite) } acquired, err := suite.ValidateRequest(ctx, ValidationRequest{ Selection: selection, Request: promptkit.RunRequest{PromptID: "empty", ProfileID: "default"}, }, input{value: "input"}, state{value: "state"}) if err != nil { t.Fatalf("validate seed request: %v", err) } if acquired.Value != (struct{}{}) || len(acquired.Diagnostics) != 0 || loaderCalls != 1 { t.Fatalf("acquisition = %#v, loader calls = %d", acquired, loaderCalls) } } func TestWithRequestValidationBuilderFailurePrecedesLoading(t *testing.T) { type input struct{ value string } type state struct{ value string } buildErr := errors.New("cannot build request") loadCalls, buildCalls := 0, 0 suite := WithRequestValidation( Suite[input, state]{}, func(context.Context, ckconfig.Selection, input, state) (*promptkit.Engine, error) { loadCalls++ return nil, errors.New("loader should not run") }, func(ctx context.Context, request ValidationRequest, gotInput input, gotState state) (promptkit.RunRequest, error) { buildCalls++ if ctx == nil || request.Request.PromptID != "seed" || gotInput.value != "input" || gotState.value != "state" { t.Errorf("builder args = (%v, %#v, %#v, %#v)", ctx, request, gotInput, gotState) } return promptkit.RunRequest{}, buildErr }, ) _, err := suite.ValidateRequest(context.Background(), ValidationRequest{Request: promptkit.RunRequest{PromptID: "seed"}}, input{value: "input"}, state{value: "state"}) if !errors.Is(err, buildErr) || buildCalls != 1 || loadCalls != 0 { t.Fatalf("validation = (%v), builder calls = %d, loader calls = %d", err, buildCalls, loadCalls) } } func TestWithRequestValidationRetainsNilLoaderIssueIndependently(t *testing.T) { firstIssue := errors.New("existing adapter issue") backing := make([]error, 1, 3) backing[0] = firstIssue existingCallback := func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { return commandkit.Acquired[struct{}]{}, nil } suite := Suite[struct{}, struct{}]{ValidateRequest: existingCallback, issues: backing[:1]} composed := WithRequestValidation(suite, nil, nil) if composed.ValidateRequest == nil || len(composed.issues) != 2 { t.Fatalf("nil loader composition = %#v", composed) } if _, err := composed.ValidateRequest(context.Background(), ValidationRequest{}, struct{}{}, struct{}{}); err != nil { t.Fatalf("nil loader replaced the existing validation callback: %v", err) } if !errors.Is(composed.issues[0], firstIssue) || composed.issues[1] == nil || backing[:cap(backing)][1] != nil { t.Fatalf("issues = %#v, original backing = %#v", composed.issues, backing[:cap(backing)]) } composed.ValidateRequest = nil if err := newPromptTestApp(t).Register(Command(composed)); err == nil || !strings.Contains(err.Error(), "PromptKit engine loader is nil") { t.Fatalf("installation did not report retained issue after capabilities were cleared: %v", err) } } func TestWithRequestValidationDelegatesNativeOutcomesWithoutModelCalls(t *testing.T) { client := &validationFailOnGenerate{} engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client)) selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "selected.yml", Set: true}} suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(_ context.Context, gotSelection ckconfig.Selection, _, _ struct{}) (*promptkit.Engine, error) { if gotSelection != selection { t.Errorf("loader selection = %#v, want %#v", gotSelection, selection) } return engine, nil }, nil) for _, test := range []struct { name string request promptkit.RunRequest want error }{ {name: "success", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}}, {name: "missing required input", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default"}, want: promptkit.ErrPromptRender}, {name: "invalid profile selection", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "missing", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}, want: promptkit.ErrProfileNotFound}, {name: "unloadable artifact", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("missing-topic.txt")}}, want: promptkit.ErrArtifactLoad}, } { t.Run(test.name, func(t *testing.T) { nativeErr := engine.ValidateRequest(context.Background(), test.request) acquired, adapterErr := suite.ValidateRequest(context.Background(), ValidationRequest{Selection: selection, Request: test.request}, struct{}{}, struct{}{}) if test.want == nil { if nativeErr != nil || adapterErr != nil { t.Fatalf("native error = %v, adapter error = %v", nativeErr, adapterErr) } if acquired.Value != (struct{}{}) { t.Fatalf("successful acquisition = %#v", acquired) } return } if !errors.Is(nativeErr, test.want) || !errors.Is(adapterErr, test.want) { t.Fatalf("native error = %v, adapter error = %v, want both to match %v", nativeErr, adapterErr, test.want) } }) } if client.calls != 0 { t.Fatalf("request validation made %d model calls", client.calls) } } func TestWithRequestValidationBuilderOutputReachesNativeValidation(t *testing.T) { type contextKey struct{} type input struct{ campaign string } type state struct{ token string } client := &validationFailOnGenerate{} engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client)) ctx := context.WithValue(context.Background(), contextKey{}, "validation invocation") selection := ckconfig.Selection{Overlay: commandkit.Supplied[string]{Value: "campaign.yml", Set: true}} seed := promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Vars: map[string]string{"topic": "seed topic"}} wantInput, wantState := input{campaign: "spring"}, state{token: "shared"} builderCalls, loaderCalls := 0, 0 suite := WithRequestValidation(Suite[input, state]{}, func(gotContext context.Context, gotSelection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) { loaderCalls++ if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || gotSelection != selection || gotInput != wantInput || gotState != wantState { t.Errorf("loader args = (%v, %#v, %#v, %#v)", gotContext, gotSelection, gotInput, gotState) } return engine, nil }, func(gotContext context.Context, request ValidationRequest, gotInput input, gotState state) (promptkit.RunRequest, error) { builderCalls++ if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || request.Selection != selection || request.Request.PromptID != seed.PromptID || request.Request.Vars["topic"] != "seed topic" || gotInput != wantInput || gotState != wantState { t.Errorf("builder args = (%v, %#v, %#v, %#v)", gotContext, request, gotInput, gotState) } built := request.Request built.Inputs = map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("required artifact from builder")} built.SessionID = "builder-session" return built, nil }) acquired, err := suite.ValidateRequest(ctx, ValidationRequest{Selection: selection, Request: seed}, wantInput, wantState) if err != nil || acquired.Value != (struct{}{}) || builderCalls != 1 || loaderCalls != 1 { t.Fatalf("built request validation = (%#v, %v), builder calls = %d, loader calls = %d", acquired, err, builderCalls, loaderCalls) } if client.calls != 0 { t.Fatalf("request validation made %d model calls", client.calls) } } func TestWithRequestValidationRetainsNilEngineAndLoadDiagnostics(t *testing.T) { request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "empty", ProfileID: "default"}} nilEngineSuite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { return nil, nil }, nil) acquired, err := nilEngineSuite.ValidateRequest(context.Background(), request, struct{}{}, struct{}{}) if err == nil || !strings.Contains(err.Error(), "nil engine") || len(acquired.Diagnostics) != 0 { t.Fatalf("nil engine result = (%#v, %v)", acquired, err) } _, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid validation configuration", []byte("broken: ["))) if loadErr == nil { t.Fatal("invalid configuration unexpectedly loaded") } loadFailure := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) { return nil, loadErr }, nil) acquired, err = loadFailure.ValidateRequest(context.Background(), request, struct{}{}, struct{}{}) var retained *configkit.LoadError if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 { t.Fatalf("load failure = (%#v, %v), want original ConfigKit error and diagnostics", acquired, err) } } func TestWithRequestValidationCancellationBoundaries(t *testing.T) { engine := adapterEngine(t) request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "missing", ProfileID: "default"}} t.Run("before builder", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() builderCalls, loaderCalls := 0, 0 suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { loaderCalls++ return engine, nil }, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) { builderCalls++ return request.Request, nil }) acquired, err := suite.ValidateRequest(ctx, request, 0, 0) if !errors.Is(err, context.Canceled) || builderCalls != 0 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 { t.Fatalf("pre-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls) } }) t.Run("after builder", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) builderCalls, loaderCalls := 0, 0 suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { loaderCalls++ return engine, nil }, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) { builderCalls++ cancel() return request.Request, nil }) acquired, err := suite.ValidateRequest(ctx, request, 0, 0) if !errors.Is(err, context.Canceled) || builderCalls != 1 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 { t.Fatalf("post-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls) } }) t.Run("during load retains load failure", func(t *testing.T) { _, loadErr := configkit.Load[map[string]any](configkit.Bytes("cancelled validation configuration", []byte("broken: ["))) if loadErr == nil { t.Fatal("invalid configuration unexpectedly loaded") } ctx, cancel := context.WithCancel(context.Background()) loaderCalls := 0 suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { loaderCalls++ cancel() return nil, loadErr }, nil) acquired, err := suite.ValidateRequest(ctx, request, 0, 0) var retained *configkit.LoadError if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 || loaderCalls != 1 { t.Fatalf("cancelled load = (%#v, %v), loader calls=%d", acquired, err, loaderCalls) } }) t.Run("after successful load skips native validation", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() loaderCalls := 0 suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { loaderCalls++ cancel() return engine, nil }, nil) acquired, err := suite.ValidateRequest(ctx, request, 0, 0) if !errors.Is(err, context.Canceled) || loaderCalls != 1 || len(acquired.Diagnostics) != 0 { t.Fatalf("post-load cancellation = (%#v, %v), loader calls=%d", acquired, err, loaderCalls) } }) t.Run("native artifact cancellation retains PromptKit category", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := &validationFailOnGenerate{} engine := validationTestEngine(t, adapterPrompt, promptkit.WithArtifactReader(validationCancelArtifactReader{cancel: cancel}), promptkit.WithLLMClient(client)) suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) { return engine, nil }, nil) request := ValidationRequest{Request: promptkit.RunRequest{ PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("cancel-topic.txt")}, }} acquired, err := suite.ValidateRequest(ctx, request, 0, 0) if !errors.Is(err, promptkit.ErrArtifactLoad) || len(acquired.Diagnostics) != 0 { t.Fatalf("native artifact cancellation = (%#v, %v), want PromptKit artifact-load category", acquired, err) } if client.calls != 0 { t.Fatalf("request validation made %d model calls", client.calls) } }) }