base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Update diagnostic and observation contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-02 12:54:10 +00:00
parent f661176b06
commit 130c4cc447
6 changed files with 37 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/roadmap/implementation.md
View File

@@ -774,6 +774,13 @@ boundary fixtures not completed by Stage 10's serialization changes.
Validation: run these public contract tests offline; diagnostic operations in Validation: run these public contract tests offline; diagnostic operations in
tests use controlled transports, not actual backend endpoints or API keys. tests use controlled transports, not actual backend endpoints or API keys.
Completion: public diagnostic, scoped-capacity, and observation fixtures use
identified lists and retain selection, disclosure, and ownership checks without
prompt labels. Corrected remaining request-validation callers of the migrated
requirements helper. Focused offline diagnostic and prepared-observation tests
passed on Go 1.27.0; other observation tests still rely on the shared legacy
fixture assigned to the next assignment and will be rerun after its conversion.
## Stage 19: Update Maintained Examples And Benchmarks ## Stage 19: Update Maintained Examples And Benchmarks
Scope: the three `examples/go-library` workflows, their assets, and configuration- Scope: the three `examples/go-library` workflows, their assets, and configuration-

5
observation_boundary_contract_test.go
View File

@@ -26,9 +26,8 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) {
deadline, _ := ctx.Deadline() deadline, _ := ctx.Deadline()
child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("child execution", map[string]any{ Source: configkit.Mapping("child execution", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": "https://example.invalid/v1", "model": "child-model"}}, "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "child-model"}},
"prompts": map[string]any{"child": map[string]any{ "prompts": []any{map[string]any{"id": "child",
"version": "1",
"default_profile": "profile", "default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": "child message"}}, "messages": []any{map[string]any{"role": "user", "content": "child message"}},
"output": map[string]any{"format": "text", "validation_mode": "none"}, "output": map[string]any{"format": "text", "validation_mode": "none"},

10
observation_contract_test.go
View File

@@ -56,17 +56,9 @@ func TestRunObservationHasResolvedIdentitiesAndAccounting(t *testing.T) {
if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished { if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished {
t.Fatalf("events=%+v", events) t.Fatalf("events=%+v", events)
} }
if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].PromptVersion == nil || if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].ProfileID != "profile" || events[1].Model != "model" {
*events[1].PromptVersion != "1" || events[1].ProfileID != "profile" || events[1].Model != "model" {
t.Fatalf("selection=%+v", events) t.Fatalf("selection=%+v", events)
} }
if events[1].PromptVersion == events[2].PromptVersion {
t.Fatal("execution events share a mutable version label")
}
*events[1].PromptVersion = "mutated event"
if events[2].PromptVersion == nil || *events[2].PromptVersion != "1" {
t.Fatal("mutating one event changed another event's version label")
}
if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess { if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess {
t.Fatalf("result=%+v terminal=%+v", result, events[2]) t.Fatalf("result=%+v terminal=%+v", result, events[2])
} }

5
prepared_observation_contract_test.go
View File

@@ -134,11 +134,10 @@ func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testin
t.Setenv(envName, "available") t.Setenv(envName, "available")
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("credential preparation", map[string]any{ Source: configkit.Mapping("credential preparation", map[string]any{
"profiles": map[string]any{"profile": map[string]any{ "profiles": []any{map[string]any{"id": "profile",
"endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true, "endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true,
}}, }},
"prompts": map[string]any{"prepared": map[string]any{ "prompts": []any{map[string]any{"id": "prepared",
"version": "1",
"default_profile": "profile", "default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": "hello"}}, "messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"}, "output": map[string]any{"format": "text", "validation_mode": "none"},

38
smoke_integration_contract_test.go
View File

@@ -65,11 +65,9 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) {
var inputs []configkit.Input var inputs []configkit.Input
for _, namespace := range []string{"north", "south"} { for _, namespace := range []string{"north", "south"} {
input, err := promptkit.CollectionInput(namespace, configkit.Mapping(namespace+" resources", map[string]any{ input, err := promptkit.CollectionInput(namespace, configkit.Mapping(namespace+" resources", map[string]any{
"profiles": map[string]any{ "profiles": []any{map[string]any{"id": "shared", "backend": "capacity/backend", "model": namespace + "-model"}},
"shared": map[string]any{"backend": "capacity/backend", "model": namespace + "-model"}, "prompts": []any{
}, map[string]any{"id": "shared",
"prompts": map[string]any{
"shared": map[string]any{
"default_profile": namespace + "/shared", "default_profile": namespace + "/shared",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}}, "messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"}, "output": map[string]any{"format": "text", "validation_mode": "none"},
@@ -83,8 +81,8 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) {
inputs = append(inputs, input) inputs = append(inputs, input)
} }
capacityInput, err := promptkit.CollectionInput("capacity", configkit.Mapping("shared capacity", map[string]any{ capacityInput, err := promptkit.CollectionInput("capacity", configkit.Mapping("shared capacity", map[string]any{
"backends": map[string]any{ "backends": []any{
"backend": map[string]any{ map[string]any{"id": "backend",
"endpoint": "https://example.invalid/v1", "endpoint": "https://example.invalid/v1",
"concurrency_limit": 1, "concurrency_limit": 1,
"queue_capacity": 0, "queue_capacity": 0,
@@ -334,9 +332,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
schemaSource := &failingSchemaFS{} schemaSource := &failingSchemaFS{}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}}
input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{ input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": "http://example.test/v1", "model": "model"}}, "profiles": []any{map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "model"}},
"prompts": map[string]any{"application-prompt": map[string]any{ "prompts": []any{map[string]any{"id": "application-prompt",
"version": "1",
"default_profile": "profile", "default_profile": "profile",
"session_id": `application-session {{input "document"}}`, "session_id": `application-session {{input "document"}}`,
"inputs": []any{map[string]any{"name": "document", "required": true}}, "inputs": []any{map[string]any{"name": "document", "required": true}},
@@ -374,9 +371,6 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
if err != nil || report == nil || !report.Completed { if err != nil || report == nil || !report.Completed {
t.Fatalf("structured smoke test=(%+v, %v)", report, err) t.Fatalf("structured smoke test=(%+v, %v)", report, err)
} }
if report.Target.PromptVersion == nil || *report.Target.PromptVersion != "1" {
t.Fatalf("smoke target label = %v, want 1", report.Target.PromptVersion)
}
if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore { if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore {
t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount()) t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount())
} }
@@ -408,14 +402,12 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
t.Errorf("smoke report exposed %q", private) t.Errorf("smoke report exposed %q", private)
} }
} }
firstLabel := report.Target.PromptVersion report.Target.EffectiveSettings.Model = "mutated report"
*firstLabel = "mutated report"
secondReport, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{ secondReport, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker", Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker",
}) })
if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.PromptVersion == nil || if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.EffectiveSettings.Model != "model" {
*secondReport.Target.PromptVersion != "1" || secondReport.Target.PromptVersion == firstLabel { t.Fatalf("smoke target was not independently owned: (%+v, %v)", secondReport, err)
t.Fatalf("smoke target label was not independently owned: (%+v, %v)", secondReport, err)
} }
profilesAfter, err := engine.ListProfiles(context.Background()) profilesAfter, err := engine.ListProfiles(context.Background())
if err != nil { if err != nil {
@@ -489,14 +481,14 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin
}, nil }, nil
})} })}
input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{ input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{
"backends": map[string]any{ "backends": []any{
"backend-a": map[string]any{ map[string]any{"id": "backend-a",
"endpoint": "http://backend-a.example/v1", "endpoint": "http://backend-a.example/v1",
"smoke_tests": map[string]any{"default_profile": "profile-b"}, "smoke_tests": map[string]any{"default_profile": "profile-b"},
}, },
"backend-b": map[string]any{"endpoint": "http://backend-b.example/v1", "concurrency_limit": 1}, map[string]any{"id": "backend-b", "endpoint": "http://backend-b.example/v1", "concurrency_limit": 1},
}, },
"profiles": map[string]any{"profile-b": map[string]any{"backend": "backend-b", "model": "model"}}, "profiles": []any{map[string]any{"id": "profile-b", "backend": "backend-b", "model": "model"}},
})} })}
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient, Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient,
@@ -544,7 +536,7 @@ func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t
}, nil }, nil
})} })}
input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{ input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}}, "profiles": []any{map[string]any{"id": "profile", "endpoint": base, "model": "model"}},
})} })}
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client,

20
validate_request_contract_test.go
View File

@@ -18,7 +18,7 @@ func TestValidateRequestMatchesPrepareContract(t *testing.T) {
} }
newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine { newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine {
t.Helper() t.Helper()
return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles, return requirementsEngine(t, onePrompt("synthetic", definition), profiles,
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
} }
@@ -92,10 +92,10 @@ func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testi
map[string]any{"name": "details", "required": true}, map[string]any{"name": "details", "required": true},
}, "", noValidationOutput) }, "", noValidationOutput)
engine := requirementsEngine(t, map[string]any{ engine := requirementsEngine(t, map[string]any{
"optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"], "optional": onePrompt("optional", optionalDefinition)["optional"],
"required": oneVersionPrompt("required", "v1", requiredDefinition)["required"], "required": onePrompt("required", requiredDefinition)["required"],
"direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"], "direct": onePrompt("direct", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
"index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"], "index": onePrompt("index", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
}, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) }, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
cases := []struct { cases := []struct {
@@ -141,9 +141,9 @@ func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
"format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json", "format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json",
}) })
prompts := map[string]any{ prompts := map[string]any{
"branch": oneVersionPrompt("branch", "v1", definition)["branch"], "branch": onePrompt("branch", definition)["branch"],
"profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"], "profile": onePrompt("profile", profilePrompt)["profile"],
"schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"], "schema": onePrompt("schema", schemaPrompt)["schema"],
} }
engine := requirementsEngine(t, prompts, profiles, engine := requirementsEngine(t, prompts, profiles,
promptkit.WithSchemaFS(fstest.MapFS{}, "."), promptkit.WithSchemaFS(fstest.MapFS{}, "."),
@@ -177,7 +177,7 @@ func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{ definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{
map[string]any{"name": "details", "required": false}, map[string]any{"name": "details", "required": false},
}, "", noValidationOutput) }, "", noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ engine := requirementsEngine(t, onePrompt("synthetic", definition), map[string]any{
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"}, "offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
}, promptkit.WithArtifactReader(canceledValidationArtifactReader{}), }, promptkit.WithArtifactReader(canceledValidationArtifactReader{}),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
@@ -195,7 +195,7 @@ func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) { func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) {
var modelCalls atomic.Int64 var modelCalls atomic.Int64
definition := contractPrompt("partial", "base message", nil, "", noValidationOutput) definition := contractPrompt("partial", "base message", nil, "", noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{ engine := requirementsEngine(t, onePrompt("synthetic", definition), map[string]any{
"partial": map[string]any{"model": "partial-model"}, "partial": map[string]any{"model": "partial-model"},
}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls})) }, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
request := promptkit.RunRequest{ request := promptkit.RunRequest{