base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Use resolved targets for client routing

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-05 19:52:48 +00:00
parent de46935a0e
commit 448e64dc40
15 changed files with 157 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
internal/llm/prepare_generation_benchmark_test.go
View File

@@ -12,13 +12,7 @@ import (
var prepareGenerationBenchmarkSink *PreparedGeneration
func BenchmarkPrepareGeneration(b *testing.B) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://api.example.test/v1",
Model: "benchmark-model",
})
if err != nil {
b.Fatalf("construct benchmark client: %v", err)
}
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
prompt := domain.RenderedPrompt{
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
@@ -31,14 +25,14 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "no_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"},
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
},
},
{
name: "small_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
"priority": "low",
"attempt": 3,
"enabled": true,
@@ -49,7 +43,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "1MiB_string_extra",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
"payload": strings.Repeat("x", 1<<20),
}},
},
@@ -58,7 +52,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "nested_json_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
"options": map[string]any{
"enabled": true,
"limits": []any{2, 4, 8},
@@ -71,7 +65,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "structured_output",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"},
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
StructuredOutput: &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{