base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Update diagnostic and observation contracts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-02 12:54:10 +00:00
parent f661176b06
commit 130c4cc447
6 changed files with 37 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
smoke_integration_contract_test.go
View File

@@ -65,11 +65,9 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) {
var inputs []configkit.Input
for _, namespace := range []string{"north", "south"} {
input, err := promptkit.CollectionInput(namespace, configkit.Mapping(namespace+" resources", map[string]any{
"profiles": map[string]any{
"shared": map[string]any{"backend": "capacity/backend", "model": namespace + "-model"},
},
"prompts": map[string]any{
"shared": map[string]any{
"profiles": []any{map[string]any{"id": "shared", "backend": "capacity/backend", "model": namespace + "-model"}},
"prompts": []any{
map[string]any{"id": "shared",
"default_profile": namespace + "/shared",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
@@ -83,8 +81,8 @@ func TestScopedCollectionRunsShareBackendCapacity(t *testing.T) {
inputs = append(inputs, input)
}
capacityInput, err := promptkit.CollectionInput("capacity", configkit.Mapping("shared capacity", map[string]any{
"backends": map[string]any{
"backend": map[string]any{
"backends": []any{
map[string]any{"id": "backend",
"endpoint": "https://example.invalid/v1",
"concurrency_limit": 1,
"queue_capacity": 0,
@@ -334,9 +332,8 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
schemaSource := &failingSchemaFS{}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: outputMarker}}
input := configkit.Input{Source: configkit.Mapping("private diagnostic test", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": "http://example.test/v1", "model": "model"}},
"prompts": map[string]any{"application-prompt": map[string]any{
"version": "1",
"profiles": []any{map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "model"}},
"prompts": []any{map[string]any{"id": "application-prompt",
"default_profile": "profile",
"session_id": `application-session {{input "document"}}`,
"inputs": []any{map[string]any{"name": "document", "required": true}},
@@ -374,9 +371,6 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
if err != nil || report == nil || !report.Completed {
t.Fatalf("structured smoke test=(%+v, %v)", report, err)
}
if report.Target.PromptVersion == nil || *report.Target.PromptVersion != "1" {
t.Fatalf("smoke target label = %v, want 1", report.Target.PromptVersion)
}
if reader.calls != 0 || schemaSource.openCount() != schemaOpensBefore {
t.Fatalf("application resources used: artifact reads=%d schema opens before=%d after=%d", reader.calls, schemaOpensBefore, schemaSource.openCount())
}
@@ -408,14 +402,12 @@ func TestSmokeTestUsesPrivateInferenceWithoutApplicationResourcesOrListingChange
t.Errorf("smoke report exposed %q", private)
}
}
firstLabel := report.Target.PromptVersion
*firstLabel = "mutated report"
report.Target.EffectiveSettings.Model = "mutated report"
secondReport, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{
Kind: promptkit.SmokeTestStructuredOutput, PromptID: "application-prompt", APIKey: "synthetic-smoke-key-marker",
})
if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.PromptVersion == nil ||
*secondReport.Target.PromptVersion != "1" || secondReport.Target.PromptVersion == firstLabel {
t.Fatalf("smoke target label was not independently owned: (%+v, %v)", secondReport, err)
if err != nil || secondReport == nil || !secondReport.Completed || secondReport.Target.EffectiveSettings.Model != "model" {
t.Fatalf("smoke target was not independently owned: (%+v, %v)", secondReport, err)
}
profilesAfter, err := engine.ListProfiles(context.Background())
if err != nil {
@@ -489,14 +481,14 @@ func TestSmokeTestCrossBackendDefaultUsesEndpointOverrideThroughFacade(t *testin
}, nil
})}
input := configkit.Input{Source: configkit.Mapping("cross-backend smoke test", map[string]any{
"backends": map[string]any{
"backend-a": map[string]any{
"backends": []any{
map[string]any{"id": "backend-a",
"endpoint": "http://backend-a.example/v1",
"smoke_tests": map[string]any{"default_profile": "profile-b"},
},
"backend-b": map[string]any{"endpoint": "http://backend-b.example/v1", "concurrency_limit": 1},
map[string]any{"id": "backend-b", "endpoint": "http://backend-b.example/v1", "concurrency_limit": 1},
},
"profiles": map[string]any{"profile-b": map[string]any{"backend": "backend-b", "model": "model"}},
"profiles": []any{map[string]any{"id": "profile-b", "backend": "backend-b", "model": "model"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: httpClient,
@@ -544,7 +536,7 @@ func TestSmokeTestInferenceEndpointsDistinguishRedirectFromRequestComposition(t
}, nil
})}
input := configkit.Input{Source: configkit.Mapping("redirect diagnostic", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": base, "model": "model"}},
"profiles": []any{map[string]any{"id": "profile", "endpoint": base, "model": "model"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client,