Migrate LLM fixtures to keyed PromptKit sources

This commit is contained in:
2026-10-04 12:15:26 +00:00
parent 30509f9d33
commit 8a8ca6d83c
7 changed files with 277 additions and 299 deletions

View File

@@ -2,12 +2,14 @@ package llm
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"testing/fstest"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -18,7 +20,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
}, "prompts")
}, ".")
mustRegisterSchemaFS(t, registry, fstest.MapFS{
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
}, "root")
@@ -42,7 +44,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
}
}
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
func TestAssetRegistryConstructionFailsForMissingPromptAsset(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
@@ -51,16 +53,21 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
"out.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"ok":true}`),
},
})
if err == nil || !strings.Contains(err.Error(), "content_file") {
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
sources, options, err := registry.PromptKitConfigurationSources()
if err != nil {
t.Fatal(err)
}
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
"asset-test-profile": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
Model: promptKitTestString("asset-test-model"),
},
}),
}}})
_, err = promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "messages/user.tmpl") {
t.Fatalf("NewEngine() error = %v, want invalid configuration for missing prompt content", err)
}
}
@@ -75,8 +82,23 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
"present.json": {Data: []byte(`{"type":"object"}`)},
}, ".")
engine := newPromptKitAssetTestEngine(t, registry)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
sources, options, err := registry.PromptKitConfigurationSources()
if err != nil {
t.Fatal(err)
}
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
"asset-test-profile": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
Model: promptKitTestString("asset-test-model"),
},
}),
}}})
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "asset.test",
ProfileID: "asset-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
@@ -84,14 +106,14 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
},
})
if err == nil || !strings.Contains(err.Error(), "missing.json") {
t.Fatalf("Prepare() error = %v, want missing schema error", err)
t.Fatalf("Prepare() error = %v, want missing schema asset failure", err)
}
}
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
registry := NewAssetRegistry()
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("prompts: {}")}}, "one")
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("prompts: {}")}}, "two")
_, err := registry.PromptFS()
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
@@ -102,12 +124,12 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
"first/profiles/one.yaml": {Data: []byte("profiles:\n one:\n model: first\n")},
}, "first/profiles"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
"second/two.yaml": {Data: []byte("profiles:\n two:\n model: second\n")},
}, "second"); err != nil {
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
}
@@ -144,10 +166,10 @@ func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
registry := NewAssetRegistry()
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("profiles:\n first: {}\n")}}, "first"); err != nil {
t.Fatal(err)
}
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("profiles:\n second: {}\n")}}, "second"); err != nil {
t.Fatal(err)
}
_, err := registry.FallbackProfileFS()
@@ -172,8 +194,8 @@ func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *test
return digest
}
first := digestFor("id: fallback\nmodel: model-one\n")
second := digestFor("id: fallback\nmodel: model-two\n")
first := digestFor("profiles:\n fallback:\n model: model-one\n")
second := digestFor("profiles:\n fallback:\n model: model-two\n")
if first == second {
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
}
@@ -231,16 +253,19 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
t.Helper()
options, err := registry.PromptKitOptions()
sources, options, err := registry.PromptKitConfigurationSources()
if err != nil {
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
t.Fatalf("PromptKitConfigurationSources() error = %v, want nil", err)
}
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "asset-test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "asset-test-model",
})))
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
"asset-test-profile": {
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
Model: promptKitTestString("asset-test-model"),
},
}),
}}})
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources, Timeout: time.Second}, options...)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
@@ -262,22 +287,23 @@ func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.Map
}
func validPromptYAML(schemaPath string) string {
return `id: asset.test
version: "v1"
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content_file: ./messages/user.tmpl
- role: user
content_file: ./messages/task.tmpl
output:
format: json
validation_mode: json_schema
schema_path: ` + schemaPath + `
repair_attempts: 0
return `prompts:
asset.test:
v1:
inputs:
- name: transcript
required: true
content_type: application/json
messages:
- role: user
content_file: ./messages/user.tmpl
- role: user
content_file: ./messages/task.tmpl
output:
format: json
validation_mode: json_schema
schema_path: ` + schemaPath + `
repair_attempts: 0
`
}