Migrate LLM fixtures to keyed PromptKit sources
This commit is contained in:
@@ -292,6 +292,10 @@ Prompt size: constructor/error call sites only.
|
|||||||
|
|
||||||
## Stage 4 — Migrate LLM Fixtures And Restore The Adapter Suite
|
## Stage 4 — Migrate LLM Fixtures And Restore The Adapter Suite
|
||||||
|
|
||||||
|
> Completion: complete. `GOWORK=off go test ./internal/framework/llm` passes
|
||||||
|
> offline; legacy keyed-format and removed-API fixtures are cleared from the
|
||||||
|
> LLM adapter suite. CLI and registered-module fixtures remain for Stage 5.
|
||||||
|
|
||||||
Goal: exercise released configuration through existing adapter coverage.
|
Goal: exercise released configuration through existing adapter coverage.
|
||||||
|
|
||||||
Inspect `promptkit_client_test.go`, `promptkit_observation_test.go`,
|
Inspect `promptkit_client_test.go`, `promptkit_observation_test.go`,
|
||||||
|
|||||||
@@ -2,12 +2,14 @@ package llm
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,7 +20,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
|||||||
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||||
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||||
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, "prompts")
|
}, ".")
|
||||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||||
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, "root")
|
}, "root")
|
||||||
@@ -42,7 +44,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
func TestAssetRegistryConstructionFailsForMissingPromptAsset(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||||
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
||||||
@@ -51,16 +53,21 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
|||||||
"out.json": {Data: []byte(`{"type":"object"}`)},
|
"out.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, ".")
|
}, ".")
|
||||||
|
|
||||||
engine := newPromptKitAssetTestEngine(t, registry)
|
sources, options, err := registry.PromptKitConfigurationSources()
|
||||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
if err != nil {
|
||||||
PromptID: "asset.test",
|
t.Fatal(err)
|
||||||
ProfileID: "asset-test-profile",
|
}
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
"transcript": promptkit.Inline(`{"ok":true}`),
|
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||||
},
|
"asset-test-profile": {
|
||||||
})
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
if err == nil || !strings.Contains(err.Error(), "content_file") {
|
Model: promptKitTestString("asset-test-model"),
|
||||||
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
|
},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
_, err = promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||||
|
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "messages/user.tmpl") {
|
||||||
|
t.Fatalf("NewEngine() error = %v, want invalid configuration for missing prompt content", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,8 +82,23 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
|||||||
"present.json": {Data: []byte(`{"type":"object"}`)},
|
"present.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}, ".")
|
}, ".")
|
||||||
|
|
||||||
engine := newPromptKitAssetTestEngine(t, registry)
|
sources, options, err := registry.PromptKitConfigurationSources()
|
||||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
|
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"asset-test-profile": {
|
||||||
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
|
Model: promptKitTestString("asset-test-model"),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "asset.test",
|
PromptID: "asset.test",
|
||||||
ProfileID: "asset-test-profile",
|
ProfileID: "asset-test-profile",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
@@ -84,14 +106,14 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
||||||
t.Fatalf("Prepare() error = %v, want missing schema error", err)
|
t.Fatalf("Prepare() error = %v, want missing schema asset failure", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
|
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("prompts: {}")}}, "one")
|
||||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
|
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("prompts: {}")}}, "two")
|
||||||
|
|
||||||
_, err := registry.PromptFS()
|
_, err := registry.PromptFS()
|
||||||
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
||||||
@@ -102,12 +124,12 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
|||||||
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
|
func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||||
"first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")},
|
"first/profiles/one.yaml": {Data: []byte("profiles:\n one:\n model: first\n")},
|
||||||
}, "first/profiles"); err != nil {
|
}, "first/profiles"); err != nil {
|
||||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{
|
||||||
"second/two.yaml": {Data: []byte("id: two\nmodel: second\n")},
|
"second/two.yaml": {Data: []byte("profiles:\n two:\n model: second\n")},
|
||||||
}, "second"); err != nil {
|
}, "second"); err != nil {
|
||||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -144,10 +166,10 @@ func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) {
|
|||||||
|
|
||||||
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
|
func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil {
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("profiles:\n first: {}\n")}}, "first"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil {
|
if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("profiles:\n second: {}\n")}}, "second"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, err := registry.FallbackProfileFS()
|
_, err := registry.FallbackProfileFS()
|
||||||
@@ -172,8 +194,8 @@ func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *test
|
|||||||
return digest
|
return digest
|
||||||
}
|
}
|
||||||
|
|
||||||
first := digestFor("id: fallback\nmodel: model-one\n")
|
first := digestFor("profiles:\n fallback:\n model: model-one\n")
|
||||||
second := digestFor("id: fallback\nmodel: model-two\n")
|
second := digestFor("profiles:\n fallback:\n model: model-two\n")
|
||||||
if first == second {
|
if first == second {
|
||||||
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
|
t.Fatalf("fallback profile digests = %q and %q, want content change", first, second)
|
||||||
}
|
}
|
||||||
@@ -231,16 +253,19 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
|
|||||||
|
|
||||||
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
|
func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
options, err := registry.PromptKitOptions()
|
sources, options, err := registry.PromptKitConfigurationSources()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("PromptKitOptions() error = %v, want nil", err)
|
t.Fatalf("PromptKitConfigurationSources() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
ID: "asset-test-profile",
|
Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
"asset-test-profile": {
|
||||||
Model: "asset-test-model",
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
})))
|
Model: promptKitTestString("asset-test-model"),
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
|
},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources, Timeout: time.Second}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -262,22 +287,23 @@ func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.Map
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validPromptYAML(schemaPath string) string {
|
func validPromptYAML(schemaPath string) string {
|
||||||
return `id: asset.test
|
return `prompts:
|
||||||
version: "v1"
|
asset.test:
|
||||||
inputs:
|
v1:
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: user
|
messages:
|
||||||
content_file: ./messages/user.tmpl
|
- role: user
|
||||||
- role: user
|
content_file: ./messages/user.tmpl
|
||||||
content_file: ./messages/task.tmpl
|
- role: user
|
||||||
output:
|
content_file: ./messages/task.tmpl
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: ` + schemaPath + `
|
validation_mode: json_schema
|
||||||
repair_attempts: 0
|
schema_path: ` + schemaPath + `
|
||||||
|
repair_attempts: 0
|
||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ type PromptKitLocalBackendConfig struct {
|
|||||||
type PromptKitClientConfig struct {
|
type PromptKitClientConfig struct {
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
ProfileFile string
|
ProfileFile string
|
||||||
|
Sources promptkit.ConfigurationSources
|
||||||
TransientRetries *int
|
TransientRetries *int
|
||||||
LocalBackend *PromptKitLocalBackendConfig
|
LocalBackend *PromptKitLocalBackendConfig
|
||||||
Assets *AssetRegistry
|
Assets *AssetRegistry
|
||||||
@@ -76,7 +77,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
|||||||
ProfileDir: cfg.ProfileDir,
|
ProfileDir: cfg.ProfileDir,
|
||||||
ProfileFile: cfg.ProfileFile,
|
ProfileFile: cfg.ProfileFile,
|
||||||
LocalBackend: cfg.LocalBackend,
|
LocalBackend: cfg.LocalBackend,
|
||||||
}, cfg.Assets)
|
}, cfg.Assets, cfg.Sources)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &promptKitClientConfigurationError{err: err}
|
return nil, &promptKitClientConfigurationError{err: err}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -190,94 +190,74 @@ func TestPromptKitClientRejectsInvalidCorrectionsBeforePromptPreparation(t *test
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
|
func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) {
|
||||||
const initialPrompt = `id: snapshot.test
|
const initialPrompt = `prompts:
|
||||||
version: "v1"
|
snapshot.test:
|
||||||
default_profile: snapshot-profile
|
v1:
|
||||||
inputs:
|
default_profile: snapshot-profile
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: user
|
messages:
|
||||||
content: "Snapshot A: {{ input \"transcript\" }}"
|
- role: user
|
||||||
output:
|
content: "Snapshot A: {{ input \"transcript\" }}"
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: adapter.schema.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 0
|
schema_path: adapter.schema.json
|
||||||
|
repair_attempts: 0
|
||||||
`
|
`
|
||||||
const updatedPrompt = `id: snapshot.test
|
const updatedPrompt = `prompts:
|
||||||
version: "v1"
|
snapshot.test:
|
||||||
default_profile: snapshot-profile
|
v1:
|
||||||
inputs:
|
default_profile: snapshot-profile
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: user
|
messages:
|
||||||
content: "Snapshot B: {{ input \"transcript\" }}"
|
- role: user
|
||||||
output:
|
content: "Snapshot B: {{ input \"transcript\" }}"
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: adapter.schema.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 0
|
schema_path: adapter.schema.json
|
||||||
|
repair_attempts: 0
|
||||||
`
|
`
|
||||||
source := &switchingPromptFS{files: fstest.MapFS{
|
source := fstest.MapFS{
|
||||||
"snapshot.test.yaml": {Data: []byte(initialPrompt)},
|
"snapshot.test.yaml": {Data: []byte(initialPrompt)},
|
||||||
}}
|
}
|
||||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
EngineOptions: []promptkit.Option{
|
Sources: promptKitTestSources(
|
||||||
promptkit.WithPromptFS(source, "."),
|
map[string]promptkit.BackendConfig{"snapshot-backend": {Endpoint: promptKitTestString("http://promptkit.test/v1")}},
|
||||||
promptkit.WithBackend(promptkit.Backend{ID: "snapshot-backend", Endpoint: "http://promptkit.test/v1"}),
|
map[string]promptkit.ProfileConfig{"snapshot-profile": {Backend: promptKitTestString("snapshot-backend"), Model: promptKitTestString("snapshot-model")}},
|
||||||
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
promptKitTestInputs(t, "snapshot-prompt", source, ".")...,
|
||||||
ID: "snapshot-profile",
|
),
|
||||||
BackendID: "snapshot-backend",
|
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
|
||||||
Model: "snapshot-model",
|
|
||||||
})),
|
|
||||||
promptkit.WithLLMClient(fake),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
opened := source.holdNextPromptRead()
|
source["snapshot.test.yaml"] = &fstest.MapFile{Data: []byte(updatedPrompt)}
|
||||||
defer source.resumePromptRead()
|
var out struct {
|
||||||
type completion struct {
|
OK bool `json:"ok"`
|
||||||
response contracts.StructuredCompletionResponse
|
|
||||||
err error
|
|
||||||
}
|
}
|
||||||
completed := make(chan completion, 1)
|
result, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
go func() {
|
PromptID: "snapshot.test",
|
||||||
var out struct {
|
Inputs: contracts.LLMInputSet{
|
||||||
OK bool `json:"ok"`
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||||
}
|
},
|
||||||
response, callErr := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
}, &out)
|
||||||
PromptID: "snapshot.test",
|
if err != nil {
|
||||||
Inputs: contracts.LLMInputSet{
|
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
||||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
||||||
},
|
|
||||||
}, &out)
|
|
||||||
completed <- completion{response: response, err: callErr}
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-opened:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("PromptKit did not read the prompt source")
|
|
||||||
}
|
}
|
||||||
source.replacePrompt([]byte(updatedPrompt))
|
if result.Debug == nil || result.Debug.Prompt == nil || len(result.Debug.Prompt.Messages) != 1 {
|
||||||
source.resumePromptRead()
|
t.Fatalf("debug prompt = %#v, want one prepared message", result.Debug)
|
||||||
|
|
||||||
result := <-completed
|
|
||||||
if result.err != nil {
|
|
||||||
t.Fatalf("CompleteStructured() error = %v, want nil", result.err)
|
|
||||||
}
|
}
|
||||||
if result.response.Debug == nil || result.response.Debug.Prompt == nil || len(result.response.Debug.Prompt.Messages) != 1 {
|
debugContent := result.Debug.Prompt.Messages[0].Content
|
||||||
t.Fatalf("debug prompt = %#v, want one prepared message", result.response.Debug)
|
|
||||||
}
|
|
||||||
debugContent := result.response.Debug.Prompt.Messages[0].Content
|
|
||||||
generatedContent := fake.lastRequest().Prompt.Messages[0].Content
|
generatedContent := fake.lastRequest().Prompt.Messages[0].Content
|
||||||
if debugContent != generatedContent {
|
if debugContent != generatedContent {
|
||||||
t.Fatalf("debug content = %q, generation content = %q, want one snapshot", debugContent, generatedContent)
|
t.Fatalf("debug content = %q, generation content = %q, want one snapshot", debugContent, generatedContent)
|
||||||
@@ -399,26 +379,22 @@ func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
|
func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) {
|
||||||
t.Run("assets", func(t *testing.T) {
|
t.Run("asset read", func(t *testing.T) {
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
for _, root := range []string{"one", "two"} {
|
if err := registry.RegisterPromptFS(unreadableAssetFS{}, "."); err != nil {
|
||||||
if err := registry.RegisterPromptFS(fstest.MapFS{
|
t.Fatalf("RegisterPromptFS() error = %v", err)
|
||||||
root + "/prompt.yaml": {Data: []byte("id: duplicate")},
|
|
||||||
}, root); err != nil {
|
|
||||||
t.Fatalf("RegisterPromptFS() error = %v", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
|
if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil ||
|
||||||
!strings.Contains(err.Error(), "duplicate asset path") {
|
!errors.Is(err, fs.ErrPermission) || strings.Contains(err.Error(), "permission denied") {
|
||||||
t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err)
|
t.Fatalf("NewPromptKitClient() error = %v, want safe asset read failure", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("engine", func(t *testing.T) {
|
t.Run("engine", func(t *testing.T) {
|
||||||
if _, err := NewPromptKitClient(PromptKitClientConfig{
|
if _, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")},
|
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(nil)},
|
||||||
}); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") {
|
}); err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || strings.Contains(err.Error(), "nil client") {
|
||||||
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
|
t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -429,7 +405,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
|
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
|
||||||
writeProfile := func(model string) {
|
writeProfile := func(model string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
content := "id: checkpoint-profile\nendpoint: http://promptkit.test/v1\nmodel: " + model + "\napi_key_env: " + credentialEnvironment + "\n"
|
content := "profiles:\n checkpoint-profile:\n endpoint: http://promptkit.test/v1\n model: " + model + "\n api_key_env: " + credentialEnvironment + "\n"
|
||||||
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
|
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -508,7 +484,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
firstPath := filepath.Join(profileDir, "first-profile.yaml")
|
firstPath := filepath.Join(profileDir, "first-profile.yaml")
|
||||||
secondPath := filepath.Join(profileDir, "second-profile.yaml")
|
secondPath := filepath.Join(profileDir, "second-profile.yaml")
|
||||||
content := []byte("id: directory-profile\nendpoint: http://promptkit.test/v1\nmodel: directory-model\n")
|
content := []byte("profiles:\n directory-profile:\n endpoint: http://promptkit.test/v1\n model: directory-model\n")
|
||||||
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
|
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -535,17 +511,17 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
parentPath := filepath.Join(profileDir, "parent.yaml")
|
parentPath := filepath.Join(profileDir, "parent.yaml")
|
||||||
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
||||||
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil {
|
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-one\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil {
|
if err := os.WriteFile(leafPath, []byte("profiles:\n leaf:\n extends: parent\n model: leaf-model\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil {
|
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-two\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||||
@@ -590,7 +566,7 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
|
|||||||
assets := newTestPromptKitAssets(t)
|
assets := newTestPromptKitAssets(t)
|
||||||
const profileID = "fallback-profile"
|
const profileID = "fallback-profile"
|
||||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||||
"profiles/fallback.yaml": {Data: []byte("id: " + profileID + "\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
"profiles/fallback.yaml": {Data: []byte("profiles:\n " + profileID + ":\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")},
|
||||||
}, "profiles"); err != nil {
|
}, "profiles"); err != nil {
|
||||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||||
}
|
}
|
||||||
@@ -633,10 +609,10 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing
|
|||||||
|
|
||||||
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
|
func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) {
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil {
|
if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("profiles:\n base:\n endpoint: http://promptkit.test/v1\n model: base-model\n reasoning_effort: medium\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil {
|
if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("profiles:\n inherited-profile:\n extends: base\n model: leaf-model\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
@@ -690,7 +666,7 @@ func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T)
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
|
t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "")
|
||||||
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
|
profilePath := filepath.Join(t.TempDir(), "optional.yaml")
|
||||||
if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
|
if err := os.WriteFile(profilePath, []byte("profiles:\n optional-profile:\n endpoint: "+server.URL+"/v1\n model: optional-model\n api_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||||
@@ -713,7 +689,7 @@ func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing.
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
|
t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential)
|
||||||
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
|
profilePath := filepath.Join(t.TempDir(), "profile.yaml")
|
||||||
if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
|
if err := os.WriteFile(profilePath, []byte("profiles:\n generation-profile:\n endpoint: "+server.URL+"/v1\n model: test\n api_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath})
|
||||||
@@ -778,8 +754,8 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
|||||||
return fingerprints[0]
|
return fingerprints[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
first := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-one\n")
|
first := fingerprintFor("profiles:\n fallback:\n endpoint: http://promptkit.test/v1\n model: model-one\n")
|
||||||
second := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-two\n")
|
second := fingerprintFor("profiles:\n fallback:\n endpoint: http://promptkit.test/v1\n model: model-two\n")
|
||||||
if first == second {
|
if first == second {
|
||||||
t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second)
|
t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second)
|
||||||
}
|
}
|
||||||
@@ -904,9 +880,10 @@ func TestPromptKitClientUsesConfiguredLocalBackend(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||||
if err := os.WriteFile(profilePath, []byte(`id: local-profile
|
if err := os.WriteFile(profilePath, []byte(`profiles:
|
||||||
backend: local
|
local-profile:
|
||||||
model: local-model
|
backend: local
|
||||||
|
model: local-model
|
||||||
`), 0o600); err != nil {
|
`), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -956,19 +933,11 @@ model: local-model
|
|||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
ProfileFile: profilePath,
|
ProfileFile: profilePath,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if clientWithoutRegistration != nil || err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
t.Fatalf("NewPromptKitClient() without local registration error = %v, want nil", err)
|
t.Fatalf("NewPromptKitClient() without local registration = (%v, %v), want invalid configuration", clientWithoutRegistration, err)
|
||||||
}
|
}
|
||||||
_, err = clientWithoutRegistration.CompleteStructured(context.Background(), request, &out)
|
if strings.Contains(err.Error(), "local-profile") || strings.Contains(err.Error(), profilePath) {
|
||||||
if err == nil ||
|
t.Fatalf("configuration error exposes profile details: %v", err)
|
||||||
!strings.Contains(err.Error(), "prepare PromptKit prompt") ||
|
|
||||||
!strings.Contains(err.Error(), promptkit.BackendLocal) {
|
|
||||||
t.Fatalf("CompleteStructured() without registration error = %v, want preparation failure with local backend context", err)
|
|
||||||
}
|
|
||||||
for _, sentinel := range []error{promptkit.ErrProfileLoad, promptkit.ErrInvalidRequest, promptkit.ErrPromptNotFound} {
|
|
||||||
if errors.Is(err, sentinel) {
|
|
||||||
t.Fatalf("preparation failure exposes PromptKit sentinel %v: %v", sentinel, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if providerCalls.Load() != 1 {
|
if providerCalls.Load() != 1 {
|
||||||
t.Fatalf("provider calls after missing-registration failure = %d, want 1", providerCalls.Load())
|
t.Fatalf("provider calls after missing-registration failure = %d, want 1", providerCalls.Load())
|
||||||
@@ -1009,7 +978,7 @@ func TestPromptKitClientAppliesConfiguredBuiltInTransientRetries(t *testing.T) {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
profilePath := filepath.Join(t.TempDir(), "profile.yml")
|
profilePath := filepath.Join(t.TempDir(), "profile.yml")
|
||||||
if err := os.WriteFile(profilePath, []byte("id: retry-profile\nendpoint: "+server.URL+"/v1\nmodel: retry-model\n"), 0o600); err != nil {
|
if err := os.WriteFile(profilePath, []byte("profiles:\n retry-profile:\n endpoint: "+server.URL+"/v1\n model: retry-model\n"), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
@@ -1232,20 +1201,16 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
EngineOptions: []promptkit.Option{
|
Sources: promptKitTestSources(
|
||||||
promptkit.WithBackend(promptkit.Backend{
|
map[string]promptkit.BackendConfig{"limited-backend": {
|
||||||
ID: "limited-backend",
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
ConcurrencyLimit: promptKitTestInt(1), QueueCapacity: &queueCapacity,
|
||||||
ConcurrencyLimit: 1,
|
}},
|
||||||
QueueCapacity: &queueCapacity,
|
map[string]promptkit.ProfileConfig{"limited-profile": {
|
||||||
}),
|
Backend: promptKitTestString("limited-backend"), Model: promptKitTestString("limited-model"),
|
||||||
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
}},
|
||||||
ID: "limited-profile",
|
),
|
||||||
BackendID: "limited-backend",
|
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
|
||||||
Model: "limited-model",
|
|
||||||
})),
|
|
||||||
promptkit.WithLLMClient(fake),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
||||||
@@ -1364,13 +1329,9 @@ func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
|
|||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
Timeout: configuredTimeout,
|
Timeout: configuredTimeout,
|
||||||
HTTPClient: &http.Client{Transport: transport},
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
EngineOptions: []promptkit.Option{
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
|
||||||
promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
Endpoint: promptKitTestString("http://promptkit.test/v1"), Model: promptKitTestString("default-model"),
|
||||||
ID: "default-profile",
|
}}),
|
||||||
Endpoint: "http://promptkit.test/v1",
|
|
||||||
Model: "default-model",
|
|
||||||
})),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||||
@@ -1415,9 +1376,9 @@ func TestPromptKitClientClassifiesBuiltInTransportAndMalformedResponses(t *testi
|
|||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)},
|
HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)},
|
||||||
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
|
||||||
ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model",
|
Endpoint: promptKitTestString("http://promptkit.test/v1"), Model: promptKitTestString("default-model"),
|
||||||
}))},
|
}}),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -1571,27 +1532,20 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r
|
|||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: registry,
|
Assets: registry,
|
||||||
ReasoningEffort: reasoningEffort,
|
ReasoningEffort: reasoningEffort,
|
||||||
EngineOptions: []promptkit.Option{
|
Sources: promptKitTestSources(
|
||||||
promptkit.WithBackend(promptkit.Backend{
|
map[string]promptkit.BackendConfig{"test-backend": {Endpoint: promptKitTestString("http://127.0.0.1:1/v1")}},
|
||||||
ID: "test-backend",
|
map[string]promptkit.ProfileConfig{
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
"default-profile": {
|
||||||
}),
|
Endpoint: promptKitTestString("http://127.0.0.1:1/v1"),
|
||||||
promptkit.WithProfiles(
|
Model: promptKitTestString("default-model"), ReasoningEffort: promptKitTestString("profile-reasoning"),
|
||||||
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
},
|
||||||
ID: "default-profile",
|
"explicit-profile": {
|
||||||
Endpoint: "http://127.0.0.1:1/v1",
|
Backend: promptKitTestString("test-backend"),
|
||||||
Model: "default-model",
|
Model: promptKitTestString("explicit-model"), ReasoningEffort: promptKitTestString("profile-reasoning"),
|
||||||
ReasoningEffort: "profile-reasoning",
|
},
|
||||||
}),
|
},
|
||||||
promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
),
|
||||||
ID: "explicit-profile",
|
EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)},
|
||||||
BackendID: "test-backend",
|
|
||||||
Model: "explicit-model",
|
|
||||||
ReasoningEffort: "profile-reasoning",
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
promptkit.WithLLMClient(fake),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
t.Fatalf("NewPromptKitClient() error = %v, want nil", err)
|
||||||
@@ -1603,38 +1557,40 @@ func newTestPromptKitAssets(t *testing.T) *AssetRegistry {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
registry := NewAssetRegistry()
|
registry := NewAssetRegistry()
|
||||||
if err := registry.RegisterPromptFS(fstest.MapFS{
|
if err := registry.RegisterPromptFS(fstest.MapFS{
|
||||||
"adapter.test.yaml": {Data: []byte(`id: adapter.test
|
"adapter.test.yaml": {Data: []byte(`prompts:
|
||||||
version: "v1"
|
adapter.test:
|
||||||
default_profile: default-profile
|
v1:
|
||||||
session_id: "{{ .session_id }}"
|
default_profile: default-profile
|
||||||
inputs:
|
session_id: "{{ .session_id }}"
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: user
|
messages:
|
||||||
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}"
|
- role: user
|
||||||
output:
|
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}"
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: adapter.schema.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 0
|
schema_path: adapter.schema.json
|
||||||
|
repair_attempts: 0
|
||||||
`)},
|
`)},
|
||||||
"adapter.direct-session.yaml": {Data: []byte(`id: adapter.direct-session
|
"adapter.direct-session.yaml": {Data: []byte(`prompts:
|
||||||
version: "v1"
|
adapter.direct-session:
|
||||||
default_profile: default-profile
|
v1:
|
||||||
inputs:
|
default_profile: default-profile
|
||||||
- name: transcript
|
inputs:
|
||||||
required: true
|
- name: transcript
|
||||||
content_type: application/json
|
required: true
|
||||||
messages:
|
content_type: application/json
|
||||||
- role: user
|
messages:
|
||||||
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
|
- role: user
|
||||||
output:
|
content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}"
|
||||||
format: json
|
output:
|
||||||
validation_mode: json_schema
|
format: json
|
||||||
schema_path: adapter.schema.json
|
validation_mode: json_schema
|
||||||
repair_attempts: 0
|
schema_path: adapter.schema.json
|
||||||
|
repair_attempts: 0
|
||||||
`)},
|
`)},
|
||||||
}, "."); err != nil {
|
}, "."); err != nil {
|
||||||
t.Fatalf("RegisterPromptFS() error = %v", err)
|
t.Fatalf("RegisterPromptFS() error = %v", err)
|
||||||
@@ -1647,61 +1603,6 @@ output:
|
|||||||
return registry
|
return registry
|
||||||
}
|
}
|
||||||
|
|
||||||
type switchingPromptFS struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
files fstest.MapFS
|
|
||||||
pauseNextOpen bool
|
|
||||||
opened chan struct{}
|
|
||||||
resume chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *switchingPromptFS) Open(name string) (fs.File, error) {
|
|
||||||
f.mu.Lock()
|
|
||||||
file, err := f.files.Open(name)
|
|
||||||
pause := f.pauseNextOpen && name == "snapshot.test.yaml"
|
|
||||||
resume := f.resume
|
|
||||||
if pause {
|
|
||||||
f.pauseNextOpen = false
|
|
||||||
close(f.opened)
|
|
||||||
}
|
|
||||||
f.mu.Unlock()
|
|
||||||
if pause {
|
|
||||||
<-resume
|
|
||||||
}
|
|
||||||
return file, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *switchingPromptFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
return f.files.ReadDir(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *switchingPromptFS) holdNextPromptRead() <-chan struct{} {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
f.pauseNextOpen = true
|
|
||||||
f.opened = make(chan struct{})
|
|
||||||
f.resume = make(chan struct{})
|
|
||||||
return f.opened
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *switchingPromptFS) replacePrompt(content []byte) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
f.files["snapshot.test.yaml"] = &fstest.MapFile{Data: append([]byte(nil), content...)}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *switchingPromptFS) resumePromptRead() {
|
|
||||||
f.mu.Lock()
|
|
||||||
resume := f.resume
|
|
||||||
f.resume = nil
|
|
||||||
f.mu.Unlock()
|
|
||||||
if resume != nil {
|
|
||||||
close(resume)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type fakePromptKitLLM struct {
|
type fakePromptKitLLM struct {
|
||||||
content string
|
content string
|
||||||
responses []promptkit.GenerateResponse
|
responses []promptkit.GenerateResponse
|
||||||
|
|||||||
@@ -23,9 +23,9 @@ func newObservedHTTPClient(t *testing.T, handler http.HandlerFunc, retries int,
|
|||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t), TransientRetries: &retries,
|
Assets: newTestPromptKitAssets(t), TransientRetries: &retries,
|
||||||
HTTPClient: &http.Client{Transport: transport},
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": {
|
||||||
ID: "default-profile", Endpoint: server.URL + "/v1", Model: "test-model",
|
Endpoint: promptKitTestString(server.URL + "/v1"), Model: promptKitTestString("test-model"),
|
||||||
}))},
|
}}),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -17,8 +17,9 @@ type PromptKitProfileSourceConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PromptKitProfileInspectorConfig struct {
|
type PromptKitProfileInspectorConfig struct {
|
||||||
Source PromptKitProfileSourceConfig
|
Source PromptKitProfileSourceConfig
|
||||||
Assets *AssetRegistry
|
Assets *AssetRegistry
|
||||||
|
Sources promptkit.ConfigurationSources
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c PromptKitProfileSourceConfig) localEndpoint() string {
|
func (c PromptKitProfileSourceConfig) localEndpoint() string {
|
||||||
@@ -75,7 +76,7 @@ func (e *promptKitProfileConfigurationError) Unwrap() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
|
func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
|
||||||
_, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets)
|
_, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets, cfg.Sources)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &promptKitProfileConfigurationError{err: err}
|
return nil, &promptKitProfileConfigurationError{err: err}
|
||||||
}
|
}
|
||||||
@@ -112,7 +113,7 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) {
|
func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry, configuredSources promptkit.ConfigurationSources) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) {
|
||||||
profileSource, profileSources, err := promptKitProfileConfigurationSources(cfg)
|
profileSource, profileSources, err := promptKitProfileConfigurationSources(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err
|
return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err
|
||||||
@@ -127,7 +128,9 @@ func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *Asse
|
|||||||
sources = maintainedSources
|
sources = maintainedSources
|
||||||
options = append(options, maintainedOptions...)
|
options = append(options, maintainedOptions...)
|
||||||
}
|
}
|
||||||
|
sources.Underlays = append(sources.Underlays, configuredSources.Underlays...)
|
||||||
sources.Underlays = append(sources.Underlays, profileSources.Underlays...)
|
sources.Underlays = append(sources.Underlays, profileSources.Underlays...)
|
||||||
|
sources.Main = configuredSources.Main
|
||||||
return profileSource, sources, options, nil
|
return profileSource, sources, options, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
43
internal/framework/llm/promptkit_test_helpers_test.go
Normal file
43
internal/framework/llm/promptkit_test_helpers_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func promptKitTestString(value string) *string { return &value }
|
||||||
|
|
||||||
|
func promptKitTestInt(value int) *int { return &value }
|
||||||
|
|
||||||
|
func promptKitTestFloat(value float64) *float64 { return &value }
|
||||||
|
|
||||||
|
func promptKitTestInputs(t *testing.T, label string, files fstest.MapFS, root string) []configkit.Input {
|
||||||
|
t.Helper()
|
||||||
|
inputs, err := promptkit.FilesystemInputs(label, files, root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FilesystemInputs() error = %v", err)
|
||||||
|
}
|
||||||
|
return inputs
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptKitTestSources(
|
||||||
|
backends map[string]promptkit.BackendConfig,
|
||||||
|
profiles map[string]promptkit.ProfileConfig,
|
||||||
|
otherInputs ...configkit.Input,
|
||||||
|
) promptkit.ConfigurationSources {
|
||||||
|
inputs := make([]configkit.Input, 0, len(otherInputs)+2)
|
||||||
|
if len(backends) != 0 {
|
||||||
|
inputs = append(inputs, configkit.Input{Source: promptkit.BackendSource("test backends", backends)})
|
||||||
|
}
|
||||||
|
if len(profiles) != 0 {
|
||||||
|
inputs = append(inputs, configkit.Input{Source: promptkit.ProfileSource("test profiles", profiles)})
|
||||||
|
}
|
||||||
|
inputs = append(inputs, otherInputs...)
|
||||||
|
if len(inputs) == 0 {
|
||||||
|
return promptkit.ConfigurationSources{}
|
||||||
|
}
|
||||||
|
return promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user