Update examples and benchmarks for identified resources

This commit is contained in:
2026-11-02 12:56:05 +00:00
parent 130c4cc447
commit 0eee3ec962
17 changed files with 93 additions and 76 deletions

View File

@@ -71,7 +71,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
message := "Hello {{.name}}" message := "Hello {{.name}}"
response := "ok" response := "ok"
output := map[string]any{"format": "text", "validation_mode": "none"} output := map[string]any{"format": "text", "validation_mode": "none"}
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"} profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})} options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
var expectedMessage string var expectedMessage string
@@ -99,9 +99,8 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
} }
config := map[string]any{ config := map[string]any{
"profiles": map[string]any{"profile": profile}, "profiles": []any{profile},
"prompts": map[string]any{"workflow": map[string]any{ "prompts": []any{map[string]any{"id": "workflow",
"version": "v1",
"default_profile": "profile", "default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": message}}, "messages": []any{map[string]any{"role": "user", "content": message}},
"output": output, "output": output,
@@ -128,7 +127,7 @@ func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixtu
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
if prepared.PromptID != "workflow" || prepared.PromptVersion == nil || *prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 || if prepared.PromptID != "workflow" || len(prepared.Messages) != 1 ||
prepared.Messages[0].Content != fixture.expectedMessage { prepared.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared fixture output is invalid: %+v", prepared) b.Fatalf("prepared fixture output is invalid: %+v", prepared)
} }

View File

@@ -38,10 +38,11 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
} }
cfg := simpleConfig("hello") cfg := simpleConfig("hello")
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}} cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
endpoint, model := "http://example.test/v1", "model"
lowerName := name lowerName := name
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{ cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{ Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
{ID: "profile", APIKeyEnv: &lowerName}, {ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
}), }),
}}}) }}})
env := " " + name + " " env := " " + name + " "
@@ -49,7 +50,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
env = "" env = ""
} }
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{ cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
{ID: "profile", APIKeyEnv: &env}, {ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
})} })}
var calls int var calls int
var auth string var auth string

View File

@@ -60,11 +60,10 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
func simpleConfig(content string) promptkit.Config { func simpleConfig(content string) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{ return promptkit.Config{Sources: promptkit.ConfigurationSources{
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{ Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
"profiles": map[string]any{"profile": map[string]any{ "profiles": []any{map[string]any{"id": "profile",
"endpoint": "http://example.test/v1", "model": "model", "endpoint": "http://example.test/v1", "model": "model",
}}, }},
"prompts": map[string]any{"prompt": map[string]any{ "prompts": []any{map[string]any{"id": "prompt",
"version": "1",
"default_profile": "profile", "default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}}, "messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"}, "output": map[string]any{"format": "text", "validation_mode": "none"},

View File

@@ -803,6 +803,13 @@ Validation: run all three examples through the semantic checker once runnable.
Use a short focused construction benchmark if needed to investigate a suspected Use a short focused construction benchmark if needed to investigate a suspected
repeated-scan regression, not to claim unmeasured performance improvements. repeated-scan regression, not to claim unmeasured performance improvements.
Completion: the three maintained examples and benchmark/shared fixtures now use
identified resources and ordered builders. The preparation example includes a
headerless item import and an inherited replacement profile. The semantic example
checker, full public contract suite, and short configuration benchmark fixture
runs passed on Go 1.27.0. The public run also verified the previously deferred
preparation and observation tests after updating their shared fixture.
## Stage 20: Document Formats And Internal Handoff ## Stage 20: Document Formats And Internal Handoff
Scope: `docs/formats.md`, `docs/internal/sources.md`, and other current internal Scope: `docs/formats.md`, `docs/internal/sources.md`, and other current internal

View File

@@ -84,13 +84,15 @@ func run() (summary, error) {
return summary{}, err return summary{}, err
} }
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model" backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
backend.ID = backendID
inputs = append(inputs, inputs = append(inputs,
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{ configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1), backend,
})}, })},
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{ configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
"proposal-profile": {Backend: &backendID, Model: &proposalModel}, {ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
"assessment-profile": {Backend: &backendID, Model: &assessmentModel}, {ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
})}, })},
) )
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{

View File

@@ -1,6 +1,5 @@
prompts: prompts:
example.assessment: - id: example.assessment
version: "1.0.0"
default_profile: assessment-profile default_profile: assessment-profile
description: Assess a structured proposal independently of schema validity. description: Assess a structured proposal independently of schema validity.
inputs: inputs:

View File

@@ -1,6 +1,5 @@
prompts: prompts:
example.proposal: - id: example.proposal
version: "1.0.0"
default_profile: proposal-profile default_profile: proposal-profile
description: Propose a bounded improvement from a synthetic idea. description: Propose a bounded improvement from a synthetic idea.
inputs: inputs:

View File

@@ -1,12 +1,12 @@
backends: backends:
offline: - id: offline
endpoint: https://review.example.invalid/v1 endpoint: https://review.example.invalid/v1
profiles: profiles:
audit: - id: audit
backend: review/offline backend: review/offline
model: review-model model: review-model
prompts: prompts:
check: - id: check
default_profile: review/audit default_profile: review/audit
description: Inspect a review item without contacting a model provider. description: Inspect a review item without contacting a model provider.
messages: messages:

View File

@@ -1,6 +1,10 @@
version: "1" version: "2"
config: config:
imports: imports:
- path: fragments/resources.yaml - path: fragments/resources.yaml
required: true required: true
- path: fragments/prepare.yaml
destination: [prompts]
placement: sequence-item
required: true
release_version: "1.4.0" release_version: "1.4.0"

View File

@@ -0,0 +1,15 @@
id: prepare
default_profile: support/fast
description: Prepare a prompt without contacting a model provider.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content_file: ../messages/note.md
output:
format: text
validation_mode: basic

View File

@@ -1,28 +1,11 @@
backends: backends:
offline: - id: offline
endpoint: https://example.invalid/v1 endpoint: https://example.invalid/v1
profiles: profiles:
base: - id: base
backend: support/offline backend: support/offline
model: offline-model model: offline-model
timeout_seconds: 90 timeout_seconds: 90
fast: - id: fast
extends: base extends: {id: base}
timeout_seconds: 60 timeout_seconds: 60
prompts:
prepare:
version: "1.0.0"
default_profile: support/fast
description: Prepare a prompt without contacting a model provider.
inputs:
- name: note
required: true
content_type: text/plain
messages:
- role: system
content: Summarize the note in one sentence.
- role: user
content_file: ../messages/note.md
output:
format: text
validation_mode: basic

View File

@@ -1,4 +1,5 @@
# A later input intentionally replaces only the inherited profile timeout. # A later input replaces the profile with an inherited variant.
profiles: profiles:
fast: - id: fast
extends: {id: base}
timeout_seconds: 30 timeout_seconds: 30

View File

@@ -12,7 +12,6 @@ import (
type summary struct { type summary struct {
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
PromptVersion *string `json:"prompt_version,omitempty"`
SelectedProfile string `json:"selected_profile"` SelectedProfile string `json:"selected_profile"`
Model string `json:"model"` Model string `json:"model"`
MessageCount int `json:"message_count"` MessageCount int `json:"message_count"`
@@ -70,9 +69,7 @@ func main() {
if err != nil { if err != nil {
exit(err) exit(err)
} }
if _, found := view.Explain(configkit.Keys( if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
"collections", "support", "profiles", "fast", "timeout_seconds",
)); !found {
exit(fmt.Errorf("collection profile override was not found")) exit(fmt.Errorf("collection profile override was not found"))
} }
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{ requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
@@ -148,7 +145,6 @@ func main() {
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
if err := encoder.Encode(summary{ if err := encoder.Encode(summary{
PromptID: prepared.PromptID, PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
SelectedProfile: prepared.SelectedProfileID, SelectedProfile: prepared.SelectedProfileID,
Model: prepared.EffectiveModelParams.Model, Model: prepared.EffectiveModelParams.Model,
MessageCount: len(prepared.Messages), MessageCount: len(prepared.Messages),

View File

@@ -41,8 +41,8 @@ type summary struct {
func main() { func main() {
endpoint, model := "https://example.invalid/v1", "offline-model" endpoint, model := "https://example.invalid/v1", "offline-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
"offline-example": {Endpoint: &endpoint, Model: &model}, {ID: "offline-example", Endpoint: &endpoint, Model: &model},
})}}}}, })}}}},
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")}, Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
}}, promptkit.WithLLMClient(deterministicClient{})) }}, promptkit.WithLLMClient(deterministicClient{}))

View File

@@ -1,6 +1,5 @@
prompts: prompts:
example.run: - id: example.run
version: "1.0.0"
default_profile: offline-example default_profile: offline-example
description: Run a prompt with a deterministic injected model client. description: Run a prompt with a deterministic injected model client.
inputs: inputs:

View File

@@ -31,13 +31,26 @@ type configurationShape struct {
func benchmarkPrompt(profileID string) map[string]any { func benchmarkPrompt(profileID string) map[string]any {
return map[string]any{ return map[string]any{
"version": "1",
"default_profile": profileID, "default_profile": profileID,
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}}, "messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
"output": map[string]any{"format": "text", "validation_mode": "none"}, "output": map[string]any{"format": "text", "validation_mode": "none"},
} }
} }
func benchmarkRecords(values map[string]any) []any {
records := make([]any, 0, len(values))
for _, id := range sortedKeys(values) {
value := values[id].(map[string]any)
record := make(map[string]any, len(value)+1)
for key, field := range value {
record[key] = field
}
record["id"] = id
records = append(records, record)
}
return records
}
func independentDefinitions(count int) configurationShape { func independentDefinitions(count int) configurationShape {
profiles := make(map[string]any, count) profiles := make(map[string]any, count)
prompts := make(map[string]any, count) prompts := make(map[string]any, count)
@@ -48,7 +61,7 @@ func independentDefinitions(count int) configurationShape {
prompts[promptID] = benchmarkPrompt(profileID) prompts[promptID] = benchmarkPrompt(profileID)
} }
return configurationShape{ return configurationShape{
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})}, main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)})},
inputEntries: count * 2, inputEntries: count * 2,
effectiveEntries: count * 2, effectiveEntries: count * 2,
tiers: 1, tiers: 1,
@@ -71,7 +84,7 @@ func fileBackedDefinitions(count int) configurationShape {
}) })
return configurationShape{ return configurationShape{
main: configkit.Input{ main: configkit.Input{
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}), Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)}),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"}, Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
}, },
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1, inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
@@ -85,12 +98,12 @@ func sharedBaseDefinitions(count int) configurationShape {
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}}, "extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
} }
for index := range count { for index := range count {
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10} profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": map[string]any{"id": "base"}, "temperature": float64(index%10) / 10}
} }
return configurationShape{ return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{ main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
"profiles": profiles, "profiles": benchmarkRecords(profiles),
"prompts": map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}, "prompts": benchmarkRecords(map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}),
})}, })},
inputEntries: count + 2, inputEntries: count + 2,
effectiveEntries: count + 2, effectiveEntries: count + 2,
@@ -106,19 +119,19 @@ func layeredDefinitions(count int) configurationShape {
for index := range count { for index := range count {
id := fmt.Sprintf("profile-%03d", index) id := fmt.Sprintf("profile-%03d", index)
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"} base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
operator[id] = map[string]any{"model": "operator-model"} operator[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model"}
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}} programmatic[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
final[id] = map[string]any{"temperature": float64(index%10) / 10} final[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}, "temperature": float64(index%10) / 10}
} }
return configurationShape{ return configurationShape{
underlays: []configkit.Underlay{ underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": benchmarkRecords(base)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": benchmarkRecords(operator)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": benchmarkRecords(programmatic)})}}},
}, },
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{ main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
"profiles": final, "profiles": benchmarkRecords(final),
"prompts": map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}, "prompts": benchmarkRecords(map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}),
})}, })},
inputEntries: count*4 + 1, inputEntries: count*4 + 1,
effectiveEntries: count + 1, effectiveEntries: count + 1,
@@ -132,8 +145,8 @@ func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
collections := make(map[string]any, count) collections := make(map[string]any, count)
for index := range count { for index := range count {
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{ collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
"prompts": map[string]any{ "prompts": []any{
"example": map[string]any{ map[string]any{"id": "example",
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}}, "messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
"output": map[string]any{"format": "text", "validation_mode": "none"}, "output": map[string]any{"format": "text", "validation_mode": "none"},
}, },
@@ -240,8 +253,8 @@ func inspectionDefinitions(extraParams map[string]any) configurationShape {
} }
return configurationShape{ return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{ main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
"profiles": profiles, "profiles": benchmarkRecords(profiles),
"prompts": prompts, "prompts": benchmarkRecords(prompts),
})}, })},
inputEntries: 2, effectiveEntries: 2, tiers: 1, inputEntries: 2, effectiveEntries: 2, tiers: 1,
} }

View File

@@ -38,8 +38,8 @@ func BenchmarkRequirementInspection(b *testing.B) {
}, "", noValidationOutput) }, "", noValidationOutput)
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{ Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
"prompts": oneVersionPrompt("inspection", "1", definition), "prompts": identifiedRequirementRecords(onePrompt("inspection", definition)),
"profiles": profiles, "profiles": identifiedRequirementRecords(profiles),
}), }),
}}}) }}})
if err != nil { if err != nil {
@@ -51,7 +51,7 @@ func BenchmarkRequirementInspection(b *testing.B) {
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"}, PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
} }
inspection, err := engine.InspectRequirements(ctx, request) inspection, err := engine.InspectRequirements(ctx, request)
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion == nil || *inspection.PromptVersion != "1" || if err != nil || inspection.PromptID != "inspection" ||
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 || inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired || inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied { len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {