promptkit/validate_request_contract_test.go

package promptkit_test

import (
	"context"
	"errors"
	"path/filepath"
	"sync/atomic"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/promptkit"
)

func TestValidateRequestMatchesPrepareContract(t *testing.T) {
	var modelCalls atomic.Int64
	profiles := map[string]any{
		"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
	}
	newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine {
		t.Helper()
		return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles,
			promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
	}

	t.Run("successful request", func(t *testing.T) {
		engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
		assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, nil)
	})

	t.Run("nil engine", func(t *testing.T) {
		var engine *promptkit.Engine
		request := promptkit.RunRequest{PromptID: "synthetic"}
		assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidConfig)
	})

	t.Run("invalid appended message", func(t *testing.T) {
		engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
		request := promptkit.RunRequest{
			PromptID: "synthetic",
			AppendedMessages: []promptkit.RenderedMessage{{
				Role: "unsupported-role", Content: "not sent to a model",
			}},
		}
		assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidRequest)
	})

	t.Run("missing required input", func(t *testing.T) {
		definition := contractPrompt("offline", `{{input "required"}}`, []any{
			map[string]any{"name": "required", "required": true},
		}, "", noValidationOutput)
		engine := newEngine(t, definition)
		assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, promptkit.ErrPromptRender)
	})

	if got := modelCalls.Load(); got != 0 {
		t.Fatalf("preparation operations made %d model calls, want none", got)
	}
}

func assertValidateRequestParity(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) *promptkit.PreparedRun {
	t.Helper()
	prepared, prepareErr := engine.Prepare(context.Background(), request)
	validateErr := engine.ValidateRequest(context.Background(), request)
	if want == nil {
		if prepareErr != nil || prepared == nil {
			t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr)
		}
		if validateErr != nil {
			t.Fatalf("ValidateRequest() error = %v, want nil", validateErr)
		}
		return prepared
	}
	if prepared != nil || !errors.Is(prepareErr, want) {
		t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want)
	}
	if !errors.Is(validateErr, want) {
		t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want)
	}
	return nil
}

func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testing.T) {
	var modelCalls atomic.Int64
	profiles := map[string]any{
		"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
	}
	branchContent := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
	optionalDefinition := contractPrompt("offline", branchContent, []any{
		map[string]any{"name": "details", "required": false},
	}, "", noValidationOutput)
	requiredDefinition := contractPrompt("offline", branchContent, []any{
		map[string]any{"name": "details", "required": true},
	}, "", noValidationOutput)
	engine := requirementsEngine(t, map[string]any{
		"optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"],
		"required": oneVersionPrompt("required", "v1", requiredDefinition)["required"],
		"direct":   oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
		"index":    oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
	}, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))

	cases := []struct {
		name    string
		request promptkit.RunRequest
		want    error
	}{
		{name: "brief branch omits optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "brief"}}},
		{name: "detailed branch needs optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}}, want: promptkit.ErrPromptRender},
		{name: "detailed branch accepts inline input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.Inline("Long form details")}}},
		{name: "declared required input is needed on brief branch", request: promptkit.RunRequest{PromptID: "required", Vars: map[string]string{"mode": "brief"}}, want: promptkit.ErrPromptRender},
		{name: "evaluated variable needs a supplied key", request: promptkit.RunRequest{PromptID: "direct"}, want: promptkit.ErrPromptRender},
		{name: "empty value and unrelated variable are accepted", request: promptkit.RunRequest{PromptID: "direct", Vars: map[string]string{"audience": "", "unrelated": "accepted"}}},
		{name: "missing index lookup remains optional", request: promptkit.RunRequest{PromptID: "index"}},
	}
	for _, test := range cases {
		t.Run(test.name, func(t *testing.T) {
			assertValidateRequestParity(t, engine, test.request, test.want)
		})
	}
	if got := modelCalls.Load(); got != 0 {
		t.Fatalf("preparation operations made %d model calls, want none", got)
	}
}

type canceledValidationArtifactReader struct{}

func (canceledValidationArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
	return nil, context.Canceled
}

func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
	var modelCalls atomic.Int64
	profiles := map[string]any{
		"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
	}
	branch := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
	definition := contractPrompt("offline", branch, []any{
		map[string]any{"name": "details", "required": false},
	}, "", noValidationOutput)
	profilePrompt := contractPrompt("offline", "hello", nil, "", noValidationOutput)
	schemaPrompt := contractPrompt("offline", "{}", nil, "", map[string]any{
		"format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json",
	})
	prompts := map[string]any{
		"branch":  oneVersionPrompt("branch", "v1", definition)["branch"],
		"profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"],
		"schema":  oneVersionPrompt("schema", "v1", schemaPrompt)["schema"],
	}
	engine := requirementsEngine(t, prompts, profiles,
		promptkit.WithSchemaFS(fstest.MapFS{}, "."),
		promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
	)

	t.Run("unselected supplied input is still loaded", func(t *testing.T) {
		missingPath := filepath.Join(t.TempDir(), "missing-input.txt")
		request := promptkit.RunRequest{
			PromptID: "branch",
			Vars:     map[string]string{"mode": "brief"},
			Inputs:   map[string]promptkit.ArtifactRef{"details": promptkit.File(missingPath)},
		}
		assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
	})
	t.Run("missing selected profile", func(t *testing.T) {
		request := promptkit.RunRequest{PromptID: "profile", ProfileID: "not-configured"}
		assertValidateRequestParity(t, engine, request, promptkit.ErrProfileNotFound)
	})
	t.Run("unavailable selected schema", func(t *testing.T) {
		request := promptkit.RunRequest{PromptID: "schema"}
		assertValidateRequestParity(t, engine, request, promptkit.ErrValidation)
	})
	if got := modelCalls.Load(); got != 0 {
		t.Fatalf("preparation operations made %d model calls, want none", got)
	}
}

func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
	var modelCalls atomic.Int64
	definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{
		map[string]any{"name": "details", "required": false},
	}, "", noValidationOutput)
	engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
		"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
	}, promptkit.WithArtifactReader(canceledValidationArtifactReader{}),
		promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
	request := promptkit.RunRequest{
		PromptID: "synthetic",
		Vars:     map[string]string{"mode": "brief"},
		Inputs:   map[string]promptkit.ArtifactRef{"details": promptkit.File("controlled-canceled-loader")},
	}
	assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
	if got := modelCalls.Load(); got != 0 {
		t.Fatalf("preparation operations made %d model calls, want none", got)
	}
}

func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) {
	var modelCalls atomic.Int64
	definition := contractPrompt("partial", "base message", nil, "", noValidationOutput)
	engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
		"partial": map[string]any{"model": "partial-model"},
	}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
	request := promptkit.RunRequest{
		PromptID:  "synthetic",
		SessionID: "request-session",
		Execution: &promptkit.ExecutionTargetOverride{Endpoint: "http://override.example/v1"},
		AppendedMessages: []promptkit.RenderedMessage{{
			Role: promptkit.RoleAssistant, Content: "caller suffix",
		}},
	}
	prepared := assertValidateRequestParity(t, engine, request, nil)
	if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Model != "partial-model" ||
		prepared.EffectiveModelParams.Endpoint != "http://override.example/v1" || prepared.SessionID != "request-session" ||
		len(prepared.Messages) != 2 || prepared.Messages[1].Content != "caller suffix" {
		t.Fatalf("preparation fixture did not exercise profile and request settings: %+v", prepared)
	}
	if got := modelCalls.Load(); got != 0 {
		t.Fatalf("preparation operations made %d model calls, want none", got)
	}
}