Migrate LLM fixtures to keyed PromptKit sources
This commit is contained in:
@@ -2,12 +2,14 @@ package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -18,7 +20,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "prompts")
|
||||
}, ".")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "root")
|
||||
@@ -42,7 +44,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
||||
func TestAssetRegistryConstructionFailsForMissingPromptAsset(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
||||
@@ -51,16 +53,21 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
||||
"out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newPromptKitAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "content_file") {
|
||||
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
|
||||
sources, options, err := registry.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||
"asset-test-profile": {
|
||||
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||
Model: promptKitTestString("asset-test-model"),
|
||||
},
|
||||
}),
|
||||
}}})
|
||||
_, err = promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "messages/user.tmpl") {
|
||||
t.Fatalf("NewEngine() error = %v, want invalid configuration for missing prompt content", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,8 +82,23 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
||||
"present.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newPromptKitAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
sources, options, err := registry.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||
"asset-test-profile": {
|
||||
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||
Model: promptKitTestString("asset-test-model"),
|
||||
},
|
||||
}),
|
||||
}}})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
@@ -84,14 +106,14 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
||||
t.Fatalf("Prepare() error = %v, want missing schema error", err)
|
||||
t.Fatalf("Prepare() error = %v, want missing schema asset failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("prompts: {}")}}, "one")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("prompts: {}")}}, "two")
|
||||
|
||||
_, err := registry.PromptFS()
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
||||
@@ -102,12 +124,12 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
|
||||
"first/profiles/one.yaml": {Data: []byte("profiles:\n one:\n model: first\n")},
|
||||
}, "first/profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
|
||||
"second/two.yaml": {Data: []byte("profiles:\n two:\n model: second\n")},
|
||||
}, "second"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
@@ -144,10 +166,10 @@ func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
|
||||
|
||||
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("profiles:\n first: {}\n")}}, "first"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
|
||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("profiles:\n second: {}\n")}}, "second"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := registry.FallbackProfileFS()
|
||||
@@ -172,8 +194,8 @@ func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *test
|
||||
return digest
|
||||
}
|
||||
|
||||
first := digestFor("id: fallback\nmodel: model-one\n")
|
||||
second := digestFor("id: fallback\nmodel: model-two\n")
|
||||
first := digestFor("profiles:\n fallback:\n model: model-one\n")
|
||||
second := digestFor("profiles:\n fallback:\n model: model-two\n")
|
||||
if first == second {
|
||||
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
|
||||
}
|
||||
@@ -231,16 +253,19 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
|
||||
|
||||
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.PromptKitOptions()
|
||||
sources, options, err := registry.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
||||
t.Fatalf("PromptKitConfigurationSources() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "asset-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "asset-test-model",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||
"asset-test-profile": {
|
||||
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||
Model: promptKitTestString("asset-test-model"),
|
||||
},
|
||||
}),
|
||||
}}})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources, Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
@@ -262,22 +287,23 @@ func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.Map
|
||||
}
|
||||
|
||||
func validPromptYAML(schemaPath string) string {
|
||||
return `id: asset.test
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./messages/user.tmpl
|
||||
- role: user
|
||||
content_file: ./messages/task.tmpl
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: ` + schemaPath + `
|
||||
repair_attempts: 0
|
||||
return `prompts:
|
||||
asset.test:
|
||||
v1:
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./messages/user.tmpl
|
||||
- role: user
|
||||
content_file: ./messages/task.tmpl
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: ` + schemaPath + `
|
||||
repair_attempts: 0
|
||||
`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user