base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Update diagnostic and observation contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-02 12:54:10 +00:00
parent f661176b06
commit 130c4cc447
6 changed files with 37 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/roadmap/implementation.md
View File

@@ -774,6 +774,13 @@ boundary fixtures not completed by Stage 10's serialization changes.
Validation: run these public contract tests offline; diagnostic operations in
tests use controlled transports, not actual backend endpoints or API keys.
Completion: public diagnostic, scoped-capacity, and observation fixtures use
identified lists and retain selection, disclosure, and ownership checks without
prompt labels. Corrected remaining request-validation callers of the migrated
requirements helper. Focused offline diagnostic and prepared-observation tests
passed on Go 1.27.0; other observation tests still rely on the shared legacy
fixture assigned to the next assignment and will be rerun after its conversion.
## Stage 19: Update Maintained Examples And Benchmarks
Scope: the three `examples/go-library` workflows, their assets, and configuration-

5
observation_boundary_contract_test.go
View File

@@ -26,9 +26,8 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) {
deadline, _ := ctx.Deadline()
child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("child execution", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": "https://example.invalid/v1", "model": "child-model"}},
"prompts": map[string]any{"child": map[string]any{
"version": "1",
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "child-model"}},
"prompts": []any{map[string]any{"id": "child",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": "child message"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},

10
observation_contract_test.go
View File

@@ -56,17 +56,9 @@ func TestRunObservationHasResolvedIdentitiesAndAccounting(t *testing.T) {
if len(events) != 3 || events[0].Kind != promptkit.ExecutionStarted || events[1].Kind != promptkit.ExecutionAttempt || events[2].Kind != promptkit.ExecutionFinished {
t.Fatalf("events=%+v", events)
}
if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].PromptVersion == nil ||
*events[1].PromptVersion != "1" || events[1].ProfileID != "profile" || events[1].Model != "model" {
if events[0].PromptID != "" || events[1].PromptID != "prompt" || events[1].ProfileID != "profile" || events[1].Model != "model" {
t.Fatalf("selection=%+v", events)
}
if events[1].PromptVersion == events[2].PromptVersion {
t.Fatal("execution events share a mutable version label")
}
*events[1].PromptVersion = "mutated event"
if events[2].PromptVersion == nil || *events[2].PromptVersion != "1" {
t.Fatal("mutating one event changed another event's version label")
}
if result.RunID == "" || events[0].RunID != result.RunID || events[2].RunID != result.RunID || events[2].Attempts != 1 || events[2].TransientRetries != 0 || events[2].RepairCalls != 0 || events[2].Outcome != promptkit.ExecutionOutcomeSuccess {
t.Fatalf("result=%+v terminal=%+v", result, events[2])
}

5
prepared_observation_contract_test.go
View File

@@ -134,11 +134,10 @@ func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testin
t.Setenv(envName, "available")
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("credential preparation", map[string]any{
"profiles": map[string]any{"profile": map[string]any{
"profiles": []any{map[string]any{"id": "profile",
"endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true,
}},
"prompts": map[string]any{"prepared": map[string]any{
"version": "1",
"prompts": []any{map[string]any{"id": "prepared",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},

38
smoke_integration_contract_test.go
View File

@@ -65,11 +65,9 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) {
var inputs []configkit.Input
for _, namespace := range []string{"north", "south"} {
input, err := promptkit.CollectionInput(namespace, configkit.Mapping(namespace+" resources", map[string]any{
"profiles": map[string]any{
"shared": map[string]any{"backend": "capacity/backend", "model": namespace + "-model"},
},
"prompts": map[string]any{
"shared": map[string]any{
"profiles": []any{map[string]any{"id": "shared", "backend": "capacity/backend", "model": namespace + "-model"}},
"prompts": []any{
map[string]any{"id": "shared",
"default_profile": namespace + "/shared",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
@@ -83,8 +81,8 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) {
inputs = append(inputs, input)
}
capacityInput, err := promptkit.CollectionInput("capacity", configkit.Mapping("shared capacity", map[string]any{
"backends": map[string]any{
"backend": map[string]any{
"backends": []any{
map[string]any{"id": "backend",
"endpoint": "https://example.invalid/v1",
"concurrency_limit": 1,
"queue_capacity": 0,
@@ -334,9 +332,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
schemaSource := &failingSchemaFS{}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}}
input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": "http://example.test/v1", "model": "model"}},
"prompts": map[string]any{"application-prompt": map[string]any{
"version": "1",
"profiles": []any{map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "model"}},
"prompts": []any{map[string]any{"id": "application-prompt",
"default_profile": "profile",
"session_id": `application-session {{input "document"}}`,
"inputs": []any{map[string]any{"name": "document", "required": true}},
@@ -374,9 +371,6 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
if err != nil || report == nil || !report.Completed {
t.Fatalf("structured smoke test=(%+v, %v)", report, err)
}
if report.Target.PromptVersion == nil || *report.Target.PromptVersion != "1" {
t.Fatalf("smoke target label = %v, want 1", report.Target.PromptVersion)
}
if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore {
t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount())
}
@@ -408,14 +402,12 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
t.Errorf("smoke report exposed %q", private)
}
}
firstLabel := report.Target.PromptVersion
*firstLabel = "mutated report"
report.Target.EffectiveSettings.Model = "mutated report"
secondReport, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker",
})
if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.PromptVersion == nil ||
*secondReport.Target.PromptVersion != "1" || secondReport.Target.PromptVersion == firstLabel {
t.Fatalf("smoke target label was not independently owned: (%+v, %v)", secondReport, err)
if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.EffectiveSettings.Model != "model" {
t.Fatalf("smoke target was not independently owned: (%+v, %v)", secondReport, err)
}
profilesAfter, err := engine.ListProfiles(context.Background())
if err != nil {
@@ -489,14 +481,14 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin
}, nil
})}
input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{
"backends": map[string]any{
"backend-a": map[string]any{
"backends": []any{
map[string]any{"id": "backend-a",
"endpoint": "http://backend-a.example/v1",
"smoke_tests": map[string]any{"default_profile": "profile-b"},
},
"backend-b": map[string]any{"endpoint": "http://backend-b.example/v1", "concurrency_limit": 1},
map[string]any{"id": "backend-b", "endpoint": "http://backend-b.example/v1", "concurrency_limit": 1},
},
"profiles": map[string]any{"profile-b": map[string]any{"backend": "backend-b", "model": "model"}},
"profiles": []any{map[string]any{"id": "profile-b", "backend": "backend-b", "model": "model"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient,
@@ -544,7 +536,7 @@ func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t
}, nil
})}
input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}},
"profiles": []any{map[string]any{"id": "profile", "endpoint": base, "model": "model"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client,

20
validate_request_contract_test.go
View File

@@ -18,7 +18,7 @@ func TestValidateRequestMatchesPrepareContract(t *testing.T) {
}
newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine {
t.Helper()
return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles,
return requirementsEngine(t, onePrompt("synthetic", definition), profiles,
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
}
@@ -92,10 +92,10 @@ func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testi
map[string]any{"name": "details", "required": true},
}, "", noValidationOutput)
engine := requirementsEngine(t, map[string]any{
"optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"],
"required": oneVersionPrompt("required", "v1", requiredDefinition)["required"],
"direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
"index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
"optional": onePrompt("optional", optionalDefinition)["optional"],
"required": onePrompt("required", requiredDefinition)["required"],
"direct": onePrompt("direct", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
"index": onePrompt("index", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
}, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
cases := []struct {
@@ -141,9 +141,9 @@ func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
"format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json",
})
prompts := map[string]any{
"branch": oneVersionPrompt("branch", "v1", definition)["branch"],
"profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"],
"schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"],
"branch": onePrompt("branch", definition)["branch"],
"profile": onePrompt("profile", profilePrompt)["profile"],
"schema": onePrompt("schema", schemaPrompt)["schema"],
}
engine := requirementsEngine(t, prompts, profiles,
promptkit.WithSchemaFS(fstest.MapFS{}, "."),
@@ -177,7 +177,7 @@ func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{
map[string]any{"name": "details", "required": false},
}, "", noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
engine := requirementsEngine(t, onePrompt("synthetic", definition), map[string]any{
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
}, promptkit.WithArtifactReader(canceledValidationArtifactReader{}),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
@@ -195,7 +195,7 @@ func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) {
var modelCalls atomic.Int64
definition := contractPrompt("partial", "base message", nil, "", noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
engine := requirementsEngine(t, onePrompt("synthetic", definition), map[string]any{
"partial": map[string]any{"model": "partial-model"},
}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
request := promptkit.RunRequest{