base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Update public preparation and execution fixtures

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-02 12:53:00 +00:00
parent 3e67743fbf
commit f661176b06
6 changed files with 71 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
capacity_contract_test.go
View File

@@ -247,11 +247,12 @@ func newBackendCapacityEngine(
promptID, content = "input-prompt", `{{input "input"}}`
inputs = []any{map[string]any{"name": "input", "required": true}}
}
backend := map[string]any{"endpoint": "http://backend.example/v1", "concurrency_limit": limit}
backend := map[string]any{"id": "limited", "endpoint": "http://backend.example/v1", "concurrency_limit": limit}
if queueCapacity != nil {
backend["queue_capacity"] = *queueCapacity
}
definition := map[string]any{
"id": promptID,
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
@@ -265,9 +266,9 @@ func newBackendCapacityEngine(
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("capacity fixture", map[string]any{
"backends": map[string]any{"limited": backend},
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
"prompts": map[string]any{promptID: promptDefinitionWithVersion("1", definition)},
"backends": []any{backend},
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
"prompts": []any{definition},
}),
}}}, options...)
if err != nil {

7
docs/roadmap/implementation.md
View File

@@ -750,6 +750,13 @@ requirements, and capacity contract fixtures.
Validation: run the focused root preparation/execution/requirements tests using
existing injected clients or controlled HTTP transports, with no live provider.
Completion: public prepared execution, frozen rendering, requirement inspection,
capacity, and reliable-execution fixtures now use identified resources and no
prompt labels. Focused offline preparation, requirement, snapshot, and capacity
tests passed on Go 1.27.0. The remaining shared legacy helper and observation
test dependencies are assigned to later work, so the full root suite remains
deferred.
## Stage 18: Migrate Public Diagnostics And Observation Contracts
Scope: root smoke diagnostic, observation, prepared-observation, and adapter

14
frozen_rendering_contract_test.go
View File

@@ -18,8 +18,7 @@ import (
const capturedPromptDocument = `
prompts:
captured:
version: "1"
- id: captured
default_profile: offline
session_id: '{{.session}}'
messages:
@@ -40,8 +39,8 @@ func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{
{Inputs: inputs},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &model},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
{ID: "offline", Endpoint: &endpoint, Model: &model},
})}}},
},
}})
@@ -145,12 +144,11 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
endpoint, model := "https://example.invalid/v1", "offline-model"
schemas := fstest.MapFS{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &model},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
{ID: "offline", Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
"prompts": map[string]any{"structured": map[string]any{
"version": "1",
"prompts": []any{map[string]any{"id": "structured",
"default_profile": "offline",
"inputs": []any{map[string]any{"name": "note", "required": true}},
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},

26
prepared_execution_contract_test.go
View File

@@ -87,18 +87,13 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
reader.set("changed artifact", "changed-input-hash")
first := prepared.Details()
if first.PromptVersion == nil || *first.PromptVersion != "1" {
t.Fatalf("prepared label = %v, want 1", *first.PromptVersion)
}
*first.PromptVersion = "mutated details"
first.Messages[0].Content = "changed details"
first.InputHashes["input"] = "changed-details-hash"
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
second := prepared.Details()
if second.PromptVersion == nil || *second.PromptVersion != "1" ||
second.Messages[0].Content != "Input=original artifact Label=original variable" {
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
t.Fatalf("details message changed: %q", second.Messages[0].Content)
}
if second.InputHashes["input"] != "original-input-hash" {
@@ -137,7 +132,6 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
}
if result.PromptID != second.PromptID ||
result.PromptVersion == nil || second.PromptVersion == nil || *result.PromptVersion != *second.PromptVersion ||
result.PromptHash != second.PromptHash ||
result.SessionID != second.SessionID ||
result.RenderedPromptHash != second.RenderedPromptHash ||
@@ -147,13 +141,6 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
}
if result.PromptVersion == second.PromptVersion {
t.Fatal("run result and prepared details share a mutable version label")
}
*result.PromptVersion = "mutated result"
if *second.PromptVersion != "1" {
t.Fatal("run result label mutation reached prepared details")
}
}
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
@@ -776,11 +763,11 @@ func newPreparedContractEngine(
func preparedContractConfig(message string, keyRequired bool) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("prepared execution", map[string]any{
"profiles": map[string]any{"profile": map[string]any{
"profiles": []any{map[string]any{"id": "profile",
"endpoint": "http://example.test/v1", "model": "model", "api_key_required": keyRequired,
}},
"prompts": map[string]any{"prepared": map[string]any{
"version": "1",
"prompts": []any{map[string]any{
"id": "prepared",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": message}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
@@ -792,8 +779,7 @@ func preparedContractConfig(message string, keyRequired bool) promptkit.Config {
func preparedPromptSource(label string) fstest.MapFS {
return fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`prompts:
prepared:
version: "1"
- id: prepared
default_profile: profile
inputs:
- name: input
@@ -813,7 +799,7 @@ func preparedPromptSource(label string) fstest.MapFS {
func preparedProfileSource(model string) fstest.MapFS {
return fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`profiles:
profile:
- id: profile
endpoint: http://example.test/v1
model: ` + model + `
`)},

28
reliable_execution_contract_test.go
View File

@@ -83,11 +83,21 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) {
}
func reliabilityConfig(backends, profiles map[string]any) promptkit.Config {
records := func(values map[string]any) []any {
result := make([]any, 0, len(values))
for id, value := range values {
record := map[string]any{"id": id}
for key, field := range value.(map[string]any) {
record[key] = field
}
result = append(result, record)
}
return result
}
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("reliable execution", map[string]any{
"backends": backends,
"profiles": profiles,
"prompts": map[string]any{"prompt": map[string]any{
"version": "1",
"backends": records(backends),
"profiles": records(profiles),
"prompts": []any{map[string]any{"id": "prompt",
"default_profile": "profile-a",
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
@@ -252,8 +262,7 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
const envName = "PROMPTKIT_PREPARED_RETRY_KEY"
t.Setenv(envName, "first-key")
source := fstest.MapFS{"prompt.yaml": &fstest.MapFile{Data: []byte(`prompts:
prompt:
version: "1"
- id: prompt
default_profile: profile
messages:
- role: user
@@ -291,8 +300,8 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
{Inputs: inputs},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
"profile": {Endpoint: &endpoint, Model: &model},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
{ID: "profile", Endpoint: &endpoint, Model: &model},
})}}},
}},
HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one,
@@ -306,8 +315,7 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
t.Fatal(err)
}
source["prompt.yaml"] = &fstest.MapFile{Data: []byte(`prompts:
prompt:
version: "1"
- id: prompt
default_profile: profile
messages:
- role: user

59
requirements_inspection_contract_test.go
View File

@@ -18,7 +18,7 @@ func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("requirements contract", map[string]any{
"prompts": prompts, "profiles": profiles,
"prompts": identifiedRequirementRecords(prompts), "profiles": identifiedRequirementRecords(profiles),
}),
}}}, options...)
if err != nil {
@@ -51,15 +51,19 @@ func contractPrompt(defaultProfile string, content string, inputs []any, session
return definition
}
func oneVersionPrompt(id, version string, definition map[string]any) map[string]any {
return map[string]any{id: promptDefinitionWithVersion(version, definition)}
func onePrompt(id string, definition map[string]any) map[string]any {
return map[string]any{id: definition}
}
func promptDefinitionWithVersion(version string, definition map[string]any) map[string]any {
result := make(map[string]any, len(definition)+1)
result["version"] = version
for key, value := range definition {
result[key] = value
func identifiedRequirementRecords(definitions map[string]any) []any {
result := make([]any, 0, len(definitions))
for id, raw := range definitions {
record := make(map[string]any)
for key, value := range raw.(map[string]any) {
record[key] = value
}
record["id"] = id
result = append(result, record)
}
return result
}
@@ -75,7 +79,7 @@ func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *te
}, `{{input "session-only"}} {{.session_variable}}`, map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json",
})
engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{
engine := requirementsEngine(t, onePrompt("requirements", definition), map[string]any{
"partial": map[string]any{"model": "partial-model", "api_key_required": true},
}, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
@@ -85,7 +89,7 @@ func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *te
if err != nil {
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
}
if inspection.PromptID != "requirements" || inspection.PromptVersion == nil || *inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
if inspection.PromptID != "requirements" || inspection.ProfileID != "partial" || !inspection.Complete {
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
}
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
@@ -134,10 +138,10 @@ func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *te
func TestInspectRequirementsUsesPromptIDsAndProfileIdentity(t *testing.T) {
prompts := map[string]any{
"first-prompt": promptDefinitionWithVersion("v1", contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput)),
"second-prompt": promptDefinitionWithVersion("v2", contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput)),
"first-prompt": contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput),
"second-prompt": contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput),
"without-default": contractPrompt("", "plain", nil, "", noValidationOutput),
"direct-required": promptDefinitionWithVersion("v1", contractPrompt("direct", "plain", nil, "", noValidationOutput)),
"direct-required": contractPrompt("direct", "plain", nil, "", noValidationOutput),
}
profiles := map[string]any{
"first": map[string]any{"model": "first-model"},
@@ -146,13 +150,13 @@ func TestInspectRequirementsUsesPromptIDsAndProfileIdentity(t *testing.T) {
}
engine := requirementsEngine(t, prompts, profiles)
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "first-prompt"})
if err != nil || first.PromptVersion == nil || *first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
if err != nil || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
t.Fatalf("first prompt ID/default profile=(%+v, %v)", first, err)
}
second, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
PromptID: "second-prompt", ProfileID: "first",
})
if err != nil || second.PromptVersion == nil || *second.PromptVersion != "v2" || second.ProfileID != "first" || second.Inputs[0].Name != "two" {
if err != nil || second.ProfileID != "first" || second.Inputs[0].Name != "two" {
t.Fatalf("second prompt ID/request profile=(%+v, %v)", second, err)
}
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "First-Prompt"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
@@ -178,7 +182,7 @@ func TestInspectRequirementsUsesPromptIDsAndProfileIdentity(t *testing.T) {
}
func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) {
prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt(
prompts := onePrompt("hash-equality", contractPrompt(
"profile", "Hello {{.name}}", nil, "", noValidationOutput,
))
engine := requirementsEngine(t, prompts, map[string]any{
@@ -201,11 +205,11 @@ func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.
func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) {
prompts := map[string]any{
"conditional": promptDefinitionWithVersion("v1", contractPrompt("profile",
`{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)),
"optional": promptDefinitionWithVersion("v1", contractPrompt("profile", `{{index . "lookup"}}`,
[]any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)),
"empty": promptDefinitionWithVersion("v1", contractPrompt("profile", "plain", nil, "", noValidationOutput)),
"conditional": contractPrompt("profile",
`{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput),
"optional": contractPrompt("profile", `{{index . "lookup"}}`,
[]any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput),
"empty": contractPrompt("profile", "plain", nil, "", noValidationOutput),
}
engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}})
conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"})
@@ -230,7 +234,7 @@ func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testi
definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{
map[string]any{"name": "transcript", "required": true},
}, `{{.session_var}} {{input "session-only"}}`, noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{
engine := requirementsEngine(t, onePrompt("ready", definition), map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
})
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
@@ -280,7 +284,7 @@ func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *
definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{
map[string]any{"name": "declared", "required": true, "description": "captured description"},
}, `{{.session}}`, noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{
engine := requirementsEngine(t, onePrompt("owned", definition), map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
})
request := promptkit.RequirementInspectionRequest{PromptID: "owned"}
@@ -288,10 +292,6 @@ func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *
if err != nil {
t.Fatal(err)
}
if first.PromptVersion == nil || *first.PromptVersion != "v1" {
t.Fatalf("requirement inspection label = %v, want v1", first.PromptVersion)
}
*first.PromptVersion = "mutated label"
for index := range first.Inputs {
if first.Inputs[index].Name == "declared" {
first.Inputs[index].Name = "mutated"
@@ -312,8 +312,7 @@ func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *
description = input.Description
}
}
if second.PromptVersion == nil || *second.PromptVersion != "v1" ||
second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
description == nil || *description != "captured description" ||
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
@@ -352,7 +351,7 @@ func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *
func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) {
t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker")
definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{
engine := requirementsEngine(t, onePrompt("privacy", definition), map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"},
})
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{