promptkit/internal/usecase/smoke_inference_case_test.go

package usecase

import (
	"context"
	"errors"
	"reflect"
	"strings"
	"testing"

	"go.maximumdirect.net/promptkit/internal/domain"
)

func TestSmokeTextInferenceCaseBuildsRequestAndEvaluatesExactContent(t *testing.T) {
	target := domain.SmokeTestTarget{
		EffectiveSettings: domain.ExecutionTarget{
			BackendID:      "selected-backend",
			Endpoint:       "https://example.invalid/v1",
			Model:          "selected-model",
			TimeoutSeconds: 17,
		},
		SettingsPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
	}
	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestTextInference, target, 3)
	if err != nil {
		t.Fatal(err)
	}
	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
		t.Fatalf("private text request = %+v", caseValue.request.Prompt)
	}
	if caseValue.request.StructuredOutput != nil || caseValue.request.TransientRetries != 3 ||
		caseValue.request.Generation != 1 || !reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
		t.Fatalf("generation request = %+v", caseValue.request)
	}

	for _, tc := range []struct {
		name             string
		content          string
		wantTextReturned bool
		wantMatched      bool
	}{
		{name: "exact token", content: caseValue.expectedContent, wantTextReturned: true, wantMatched: true},
		{name: "surrounding whitespace", content: " \n" + caseValue.expectedContent + "\t", wantTextReturned: true, wantMatched: true},
		{name: "case differs", content: strings.ToLower(caseValue.expectedContent), wantTextReturned: true},
		{name: "unexpected text", content: "another answer", wantTextReturned: true},
		{name: "whitespace only", content: " \n\t"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
			if err != nil {
				t.Fatal(err)
			}
			if facts.TextReturned == nil || *facts.TextReturned != tc.wantTextReturned ||
				facts.ExpectedTextMatched == nil || *facts.ExpectedTextMatched != tc.wantMatched {
				t.Fatalf("facts = %+v", facts)
			}
			if facts.ValidJSON != nil || facts.SchemaMatched != nil {
				t.Fatalf("text evaluation populated structured facts: %+v", facts)
			}
		})
	}

	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
	if err != nil {
		t.Fatal(err)
	}
	if noResponseFacts.TextReturned != nil || noResponseFacts.ExpectedTextMatched != nil {
		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
	}
}

func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
	target := domain.SmokeTestTarget{
		EffectiveSettings: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "selected-model"},
		SettingsPresence:  domain.ExecutionTargetPresence{TimeoutSeconds: true},
	}
	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, target, 2)
	if err != nil {
		t.Fatal(err)
	}
	if !caseValue.structuredOutputRequested() || caseValue.request.StructuredOutput.JSONSchema == nil ||
		caseValue.request.StructuredOutput.JSONSchema.Name != "promptkit_smoke" ||
		!caseValue.request.StructuredOutput.JSONSchema.Strict {
		t.Fatalf("structured request = %+v", caseValue.request.StructuredOutput)
	}
	if len(caseValue.request.Prompt.Messages) != 1 || caseValue.request.Prompt.Messages[0].Role != "user" ||
		!strings.Contains(caseValue.request.Prompt.Messages[0].Content, caseValue.expectedContent) {
		t.Fatalf("private structured request = %+v", caseValue.request.Prompt)
	}
	if caseValue.request.TransientRetries != 2 || caseValue.request.Generation != 1 ||
		!reflect.DeepEqual(caseValue.request.Target, target.EffectiveSettings) ||
		!reflect.DeepEqual(caseValue.request.TargetPresence, target.SettingsPresence) {
		t.Fatalf("generation request = %+v", caseValue.request)
	}

	additionalPropertyContent := strings.TrimSuffix(caseValue.expectedContent, "}") + `,"extra":true}`
	for _, tc := range []struct {
		name              string
		content           string
		wantValidJSON     bool
		wantSchemaMatched *bool
	}{
		{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
		{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
		{name: "scalar JSON misses schema", content: `42`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
		{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
		{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
		{name: "empty content", content: "", wantSchemaMatched: nil},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts, err := caseValue.evaluate(context.Background(), &domain.GenerateResponse{Content: tc.content})
			if err != nil {
				t.Fatal(err)
			}
			if facts.ValidJSON == nil || *facts.ValidJSON != tc.wantValidJSON || !sameBool(facts.SchemaMatched, tc.wantSchemaMatched) {
				t.Fatalf("facts = %+v", facts)
			}
		})
	}

	noResponseFacts, err := caseValue.evaluate(context.Background(), nil)
	if err != nil {
		t.Fatal(err)
	}
	if noResponseFacts.ValidJSON != nil || noResponseFacts.SchemaMatched != nil {
		t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
	}
}

func TestSmokeStructuredInferenceCaseHonorsCancellation(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	if _, err := prepareSmokeInferenceCase(ctx, domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0); !errors.Is(err, context.Canceled) {
		t.Fatalf("prepare error = %v, want cancellation", err)
	}
	caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0)
	if err != nil {
		t.Fatal(err)
	}
	facts, err := caseValue.evaluate(ctx, &domain.GenerateResponse{Content: `{"status":"ok"}`})
	if !errors.Is(err, context.Canceled) || facts.ValidJSON != nil || facts.SchemaMatched != nil {
		t.Fatalf("evaluation facts = %+v, error = %v", facts, err)
	}
}