base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Update examples and benchmarks for identified resources

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-02 12:56:05 +00:00
parent 130c4cc447
commit 0eee3ec962
17 changed files with 93 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
internal/configuration/benchmark_test.go
View File

@@ -31,13 +31,26 @@ type configurationShape struct {
func benchmarkPrompt(profileID string) map[string]any {
return map[string]any{
"version": "1",
"default_profile": profileID,
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
func benchmarkRecords(values map[string]any) []any {
records := make([]any, 0, len(values))
for _, id := range sortedKeys(values) {
value := values[id].(map[string]any)
record := make(map[string]any, len(value)+1)
for key, field := range value {
record[key] = field
}
record["id"] = id
records = append(records, record)
}
return records
}
func independentDefinitions(count int) configurationShape {
profiles := make(map[string]any, count)
prompts := make(map[string]any, count)
@@ -48,7 +61,7 @@ func independentDefinitions(count int) configurationShape {
prompts[promptID] = benchmarkPrompt(profileID)
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)})},
inputEntries: count * 2,
effectiveEntries: count * 2,
tiers: 1,
@@ -71,7 +84,7 @@ func fileBackedDefinitions(count int) configurationShape {
})
return configurationShape{
main: configkit.Input{
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)}),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
},
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
@@ -85,12 +98,12 @@ func sharedBaseDefinitions(count int) configurationShape {
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
}
for index := range count {
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": map[string]any{"id": "base"}, "temperature": float64(index%10) / 10}
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
"profiles": profiles,
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))},
"profiles": benchmarkRecords(profiles),
"prompts": benchmarkRecords(map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}),
})},
inputEntries: count + 2,
effectiveEntries: count + 2,
@@ -106,19 +119,19 @@ func layeredDefinitions(count int) configurationShape {
for index := range count {
id := fmt.Sprintf("profile-%03d", index)
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
operator[id] = map[string]any{"model": "operator-model"}
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
final[id] = map[string]any{"temperature": float64(index%10) / 10}
operator[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model"}
programmatic[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
final[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}, "temperature": float64(index%10) / 10}
}
return configurationShape{
underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": benchmarkRecords(base)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": benchmarkRecords(operator)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": benchmarkRecords(programmatic)})}}},
},
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
"profiles": final,
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))},
"profiles": benchmarkRecords(final),
"prompts": benchmarkRecords(map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}),
})},
inputEntries: count*4 + 1,
effectiveEntries: count + 1,
@@ -132,8 +145,8 @@ func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
collections := make(map[string]any, count)
for index := range count {
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
"prompts": map[string]any{
"example": map[string]any{
"prompts": []any{
map[string]any{"id": "example",
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
},
@@ -240,8 +253,8 @@ func inspectionDefinitions(extraParams map[string]any) configurationShape {
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
"profiles": profiles,
"prompts": prompts,
"profiles": benchmarkRecords(profiles),
"prompts": benchmarkRecords(prompts),
})},
inputEntries: 2, effectiveEntries: 2, tiers: 1,
}