Migrate HTTP and report fixtures to PromptKit sources
This commit is contained in:
@@ -15,6 +15,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -597,64 +598,68 @@ func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "no declared inputs",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
definition: `prompts:
|
||||
p:
|
||||
"1":
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "optional input omitted",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
inputs:
|
||||
- name: note
|
||||
required: false
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
definition: `prompts:
|
||||
p:
|
||||
"1":
|
||||
default_profile: exec
|
||||
inputs:
|
||||
- name: note
|
||||
required: false
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "required input omitted",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
definition: `prompts:
|
||||
p:
|
||||
"1":
|
||||
default_profile: exec
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: "hello"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusBadRequest,
|
||||
wantCode: "prompt_render_failed",
|
||||
},
|
||||
{
|
||||
name: "template input omitted",
|
||||
definition: `id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
definition: `prompts:
|
||||
p:
|
||||
"1":
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
wantStatus: http.StatusBadRequest,
|
||||
wantCode: "prompt_render_failed",
|
||||
@@ -986,24 +991,21 @@ func TestHandlerSessionIDUsesPromptkitResolution(t *testing.T) {
|
||||
profileDir := t.TempDir()
|
||||
sessionLine := ""
|
||||
if tc.definitionSession != "" {
|
||||
sessionLine = "session_id: " + tc.definitionSession + "\n"
|
||||
sessionLine = " session_id: " + tc.definitionSession + "\n"
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("id: p\nversion: \"1\"\ndefault_profile: exec\n"+sessionLine+`messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("prompts:\n p:\n \"1\":\n default_profile: exec\n"+sessionLine+` messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("id: exec\nendpoint: http://127.0.0.1:1/v1\nmodel: test\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n exec:\n endpoint: http://127.0.0.1:1/v1\n model: test\n"), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir}, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||
if err != nil {
|
||||
t.Fatalf("new engine: %v", err)
|
||||
}
|
||||
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, nil, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||
|
||||
body := `{"prompt_id":"p"}`
|
||||
if tc.requestSession != "" {
|
||||
@@ -1035,39 +1037,30 @@ output:
|
||||
func TestHandlerSharesBackendCapacityAcrossConcurrentRequests(t *testing.T) {
|
||||
promptDir := t.TempDir()
|
||||
profileDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`id: p
|
||||
version: "1"
|
||||
default_profile: limited
|
||||
messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
||||
p:
|
||||
"1":
|
||||
default_profile: limited
|
||||
messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: limited
|
||||
backend: limited
|
||||
model: test
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
||||
limited:
|
||||
backend: limited
|
||||
model: test
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
queueCapacity := 0
|
||||
client := &blockingLLMClient{started: make(chan struct{}, 1), release: make(chan struct{})}
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "limited",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
ConcurrencyLimit: 1,
|
||||
QueueCapacity: &queueCapacity,
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("new engine: %v", err)
|
||||
}
|
||||
backend := promptkit.BackendConfig{Endpoint: stringPointer("http://127.0.0.1:1/v1"), ConcurrencyLimit: intPointer(1), QueueCapacity: &queueCapacity}
|
||||
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, &backend, promptkit.WithLLMClient(client))
|
||||
h := NewHandler(engine)
|
||||
first := httptest.NewRecorder()
|
||||
firstDone := make(chan struct{})
|
||||
@@ -1265,16 +1258,24 @@ func newRetryHandler(t *testing.T, endpoint string, retries *int, concurrency in
|
||||
t.Helper()
|
||||
promptDir := t.TempDir()
|
||||
profileDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte("id: p\nversion: \"1\"\ndefault_profile: limited\nmessages:\n - role: user\n content: hi\noutput:\n format: text\n validation_mode: none\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
||||
p:
|
||||
"1":
|
||||
default_profile: limited
|
||||
messages:
|
||||
- role: user
|
||||
content: hi
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("id: limited\nbackend: limited\nmodel: test\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{PromptDir: promptDir, ProfileDir: profileDir, TransientRetries: retries}, promptkit.WithBackend(promptkit.Backend{ID: "limited", Endpoint: endpoint, ConcurrencyLimit: concurrency, QueueCapacity: queue}))
|
||||
if err != nil {
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n limited:\n backend: limited\n model: test\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend := promptkit.BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrency, QueueCapacity: queue}
|
||||
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, retries, &backend)
|
||||
return NewHandler(engine)
|
||||
}
|
||||
|
||||
@@ -1770,34 +1771,29 @@ func newHandlerEngineWithOptions(t *testing.T, options ...promptkit.Option) *pro
|
||||
|
||||
promptDir := t.TempDir()
|
||||
profileDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`id: p
|
||||
version: "1"
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
||||
p:
|
||||
"1":
|
||||
default_profile: exec
|
||||
messages:
|
||||
- role: user
|
||||
content: "hi"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: exec
|
||||
endpoint: http://example.invalid/v1
|
||||
model: model
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
||||
exec:
|
||||
endpoint: http://example.invalid/v1
|
||||
model: model
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: promptDir,
|
||||
ProfileDir: profileDir,
|
||||
}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct public engine: %v", err)
|
||||
}
|
||||
return engine
|
||||
return newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, nil, options...)
|
||||
}
|
||||
|
||||
func newDefinitionHandler(t *testing.T, definition string) *Handler {
|
||||
@@ -1808,21 +1804,46 @@ func newDefinitionHandler(t *testing.T, definition string) *Handler {
|
||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(definition), 0o644); err != nil {
|
||||
t.Fatalf("write prompt fixture: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: exec
|
||||
endpoint: http://example.invalid/v1
|
||||
model: model
|
||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`profiles:
|
||||
exec:
|
||||
endpoint: http://example.invalid/v1
|
||||
model: model
|
||||
`), 0o644); err != nil {
|
||||
t.Fatalf("write profile fixture: %v", err)
|
||||
}
|
||||
|
||||
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, nil, nil, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||
return NewHandler(engine)
|
||||
}
|
||||
|
||||
func newHandlerEngineFromDirectories(t *testing.T, promptDir, profileDir string, retries *int, backend *promptkit.BackendConfig, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
|
||||
promptInputs, err := promptkit.DirectoryInputs(promptDir)
|
||||
if err != nil {
|
||||
t.Fatalf("load prompt inputs: %v", err)
|
||||
}
|
||||
profileInputs, err := promptkit.DirectoryInputs(profileDir)
|
||||
if err != nil {
|
||||
t.Fatalf("load profile inputs: %v", err)
|
||||
}
|
||||
underlays := []configkit.Underlay{{Inputs: promptInputs}, {Inputs: profileInputs}}
|
||||
if backend != nil {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource("test backend", map[string]promptkit.BackendConfig{"limited": *backend})}}})
|
||||
}
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: promptDir,
|
||||
ProfileDir: profileDir,
|
||||
}, promptkit.WithLLMClient(handlerLLMClient{}))
|
||||
Sources: promptkit.ConfigurationSources{Underlays: underlays},
|
||||
TransientRetries: retries,
|
||||
}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("construct public engine: %v", err)
|
||||
}
|
||||
return NewHandler(engine)
|
||||
return engine
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string {
|
||||
return &value
|
||||
}
|
||||
|
||||
func assertHTTPErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) {
|
||||
|
||||
Reference in New Issue
Block a user