scriptorium/internal/usecase/integration_test.go

package usecase

import (
	"context"
	"path/filepath"
	"testing"

	"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
	"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
	"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
	"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
	"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
	"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
)

type integrationLLM struct{}

func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
	lastIntegrationRequest = req
	return &domain.GenerateResponse{
		Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`,
		Usage: domain.TokenUsage{
			PromptTokens:     42,
			CompletionTokens: 36,
			TotalTokens:      78,
		},
	}, nil
}

var lastIntegrationRequest domain.GenerateRequest

func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) {
	root, err := filepath.Abs(filepath.Join("..", ".."))
	if err != nil {
		t.Fatalf("failed to resolve repo root: %v", err)
	}

	promptsDir := filepath.Join(root, "examples", "prompts")
	profilesDir := filepath.Join(root, "examples", "profiles")
	schemasDir := filepath.Join(root, "examples", "schemas")
	fixturesDir := filepath.Join(root, "examples", "fixtures")
	t.Setenv("SCRIPTORIUM_API_KEY", "test-key")

	runner := NewRunner(
		promptdef.NewFilesystemRepository(promptsDir),
		profile.NewFilesystemRepository(profilesDir),
		artifact.NewCompositeReader(),
		prompt.NewGoRenderer(),
		&integrationLLM{},
		validate.NewStandardValidator(schemasDir),
	)

	res, err := runner.Run(context.Background(), domain.RunRequest{
		PromptID: "generic.structured_events",
		Inputs: map[string]domain.ArtifactRef{
			"transcript": {
				Type: domain.ArtifactRefFile,
				URI:  filepath.Join(fixturesDir, "transcript.md"),
			},
			"glossary": {
				Type: domain.ArtifactRefFile,
				URI:  filepath.Join(fixturesDir, "glossary.yml"),
			},
		},
	})
	if err != nil {
		t.Fatalf("expected no error, got %v", err)
	}

	if res.PromptID != "generic.structured_events" {
		t.Fatalf("unexpected prompt id: %q", res.PromptID)
	}
	if res.SelectedProfileID != "local-quality" {
		t.Fatalf("expected selected profile local-quality from prompt default, got %q", res.SelectedProfileID)
	}
	if res.RunID == "" {
		t.Fatal("expected run id")
	}
	if res.PromptHash == "" {
		t.Fatal("expected prompt hash")
	}
	if res.PromptVersion != "1.0.0" {
		t.Fatalf("unexpected prompt version: %q", res.PromptVersion)
	}
	if res.Validation.Status != domain.ValidationPassed {
		t.Fatalf("expected passed validation, got %q", res.Validation.Status)
	}
	if res.Validation.Mode != domain.ValidationJSONSchema {
		t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode)
	}
	if lastIntegrationRequest.StructuredOutput == nil {
		t.Fatal("expected provider-level structured output request for json_schema prompt")
	}
	if lastIntegrationRequest.StructuredOutput.Type != domain.StructuredOutputJSONSchema {
		t.Fatalf("expected structured output type json_schema, got %q", lastIntegrationRequest.StructuredOutput.Type)
	}
	if lastIntegrationRequest.StructuredOutput.JSONSchema == nil || lastIntegrationRequest.StructuredOutput.JSONSchema.Schema == nil {
		t.Fatalf("expected structured output json_schema payload, got %+v", lastIntegrationRequest.StructuredOutput.JSONSchema)
	}
	if res.Artifact.ContentType != "application/json" {
		t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType)
	}
	if len(res.RawOutput) == 0 {
		t.Fatal("expected raw output to be preserved")
	}
	if res.PromptHash == "" {
		t.Fatal("expected non-empty prompt hash")
	}
	if len(res.InputHashes) != 2 {
		t.Fatalf("expected two input hashes, got %d", len(res.InputHashes))
	}
	if res.InputHashes["transcript"] == "" || res.InputHashes["glossary"] == "" {
		t.Fatalf("expected both input hashes to be set, got %#v", res.InputHashes)
	}
	if res.Usage.TotalTokens != 78 {
		t.Fatalf("expected usage from fake llm, got %+v", res.Usage)
	}
	if res.StartTime.IsZero() || res.EndTime.IsZero() {
		t.Fatal("expected start/end timestamps")
	}
	if res.EndTime.Before(res.StartTime) {
		t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
	}
	if res.Duration < 0 {
		t.Fatalf("expected non-negative duration, got %s", res.Duration)
	}
}