104 Commits

Author SHA1 Message Date
207677e994 Prepare the v0.25.0 release documentation 2026-11-22 19:08:29 +00:00
20c5ea47af Document invocation observation and cost reporting 2026-11-22 18:38:11 +00:00
d4509b163e Cover concurrent multi-provider invocation accounting 2026-11-22 18:32:29 +00:00
ba4de9d11e Isolate cost accounting for prepared executions 2026-11-22 18:30:02 +00:00
d77e3e2f83 Report costs for ordinary runs 2026-11-22 18:28:39 +00:00
81a2a7591d Collect costs for dispatched model attempts 2026-11-22 18:27:01 +00:00
cd0e1f5d39 Capture optional provider reported costs 2026-11-22 18:25:32 +00:00
6ed0e037c4 Define observed cost values and aggregation 2026-11-22 18:24:33 +00:00
3d82cad2c9 Aggregate reasoning usage and derive cache hit ratios 2026-11-22 18:22:07 +00:00
db1cad6d2d Decode provider reasoning and cache usage details 2026-11-22 18:20:11 +00:00
c3257b63f0 Add vLLM models authentication diagnostics 2026-11-22 18:17:30 +00:00
ca8cf1b827 Resolve authentication diagnostics by provider 2026-11-22 18:16:13 +00:00
adc700f96e Capture provider policy in backend execution targets 2026-11-22 18:13:17 +00:00
6056f93e55 Add backend provider and cost configuration 2026-11-22 18:11:10 +00:00
f2b0ed8d6c Select prepared execution observers at run time 2026-11-22 18:08:28 +00:00
ea7cc170ad Select execution observers from context 2026-11-22 18:07:17 +00:00
f762c0358e Plan invocation observation and usage implementation 2026-11-22 18:01:31 +00:00
a47df4ec20 Consolidate the observation roadmap around the agreed target state 2026-11-22 17:52:58 +00:00
fed9ad082d Clarify inspection and diagnostic accounting roadmap 2026-11-22 17:16:14 +00:00
4ee64b6eda Add explicit zero-cost declarations and cost-source reporting to the roadmap 2026-11-22 17:11:57 +00:00
440d9a2c88 Add vLLM provider conventions to the observation roadmap 2026-11-22 15:57:37 +00:00
23ad9e2a26 Add reasoning and cache usage reporting to the roadmap 2026-11-22 15:54:02 +00:00
80649c579d Define the default OpenAI-compatible provider in the roadmap 2026-11-22 15:49:17 +00:00
0254eb1b5b Clarify provider selection and diagnostic defaults in the roadmap 2026-11-22 15:41:20 +00:00
a10d1a2245 Select provider conventions through backend configuration 2026-11-22 15:27:12 +00:00
0564c8086c Clarify observation accounting and implementation planning 2026-11-22 10:02:22 +00:00
03209947f6 Add optional OpenRouter cost reporting to the observation roadmap 2026-11-22 09:48:38 +00:00
1070be8d89 Define invocation-scoped observation roadmap 2026-11-22 09:39:27 +00:00
5ea60e9b96 Prepare v0.24.0 release documentation 2026-11-21 18:09:47 +00:00
ba5ac651e2 Share immutable backend defaults during profile capture 2026-11-21 17:55:06 +00:00
6f45fd9fd0 Avoid inheritance expansion during input presence capture 2026-11-21 17:53:40 +00:00
79e23ecdba Preserve defaulted prompt input inspection metadata 2026-11-21 17:52:56 +00:00
63e2968038 Keep application extension sensitivity policy with consumers 2026-11-21 17:51:52 +00:00
1408001fce Clarify configuration handoff consistency guidance 2026-11-21 17:51:36 +00:00
df8384426d Record configuration capture scalability opportunities 2026-11-21 17:49:05 +00:00
8758c350e9 Record defaulted input metadata integration gap 2026-11-21 17:27:06 +00:00
4a9def2138 Record configuration migration review findings 2026-11-21 16:48:34 +00:00
40f8a68253 Record final configuration migration verification 2026-11-21 14:50:06 +00:00
08d3d0ea3b Document shared configuration and repair defaults 2026-11-21 14:48:19 +00:00
56795bb49a Update offline examples for shared configuration 2026-11-21 14:43:14 +00:00
891feb8c27 Cover focused configuration inspection behavior 2026-11-21 14:42:02 +00:00
424d35c798 Cover application-owned configuration handoff 2026-11-21 14:39:48 +00:00
bab3422e62 Migrate facade tests to the new configuration contracts 2026-11-21 14:36:07 +00:00
cf03bca753 Migrate engine tests to captured configuration 2026-11-21 14:30:15 +00:00
92e1e225b2 Migrate configuration tests to captured resources 2026-11-21 14:29:06 +00:00
74a77b3c32 Construct engines from resolved application configuration 2026-11-21 14:24:30 +00:00
3a948a1e02 Mount configuration fragments under PromptKit 2026-11-21 14:22:41 +00:00
eb66f82b4f Use focused native configuration inspection 2026-11-21 14:21:39 +00:00
4d5841e087 Capture owned resources from resolved settings 2026-11-21 14:20:46 +00:00
34c0cf3f14 Compose reusable configuration rules at the PromptKit path 2026-11-21 14:18:15 +00:00
76c8789e65 Share configuration settings across the public boundary 2026-11-21 14:15:14 +00:00
38a2dcca52 Separate output contract inputs from resolved values 2026-11-21 14:12:34 +00:00
dad302f1ff Resolve repair budgets from validation modes 2026-11-21 14:10:22 +00:00
645bff20d7 Upgrade ConfigKit to version 1.4.0 2026-11-21 14:07:51 +00:00
248d0dbcf5 Adopt shared configuration model in architecture and implementation plan 2026-11-21 13:51:26 +00:00
515649fee0 Plan configuration handoff and repair defaults 2026-11-21 13:42:10 +00:00
07aaee8ef6 Add shared configuration model to future roadmap 2026-11-21 13:29:06 +00:00
49ea85680e Record configuration handoff integration results 2026-11-21 13:13:49 +00:00
16f8197a2d Update configuration roadmap for ConfigKit v1.4.0 2026-11-21 12:45:17 +00:00
2a726d60ff Set inspection boundary for application-specific fields 2026-11-21 04:18:22 +00:00
d3523000bf Allow application-specific configuration fields in roadmap 2026-11-21 04:13:27 +00:00
bb8ec2c792 Clarify resource version policy and roadmap planning scope 2026-11-21 04:09:32 +00:00
f80ec1c42c Adopt focused ConfigKit inspection in configuration roadmap 2026-11-21 03:39:11 +00:00
649bb7dcf2 Narrow resolved configuration handoff to typed value lookup 2026-11-21 01:50:53 +00:00
55276c9429 Plan application-owned ConfigKit resolution and whole-result handoff 2026-11-21 00:12:38 +00:00
aae523c916 Clarify repair defaults and configuration inspection boundaries 2026-11-20 19:20:32 +00:00
dfe9806228 Record fixed PromptKit configuration root decision 2026-11-20 19:13:46 +00:00
63c96b7d61 Record default repair budget decisions 2026-11-20 19:10:24 +00:00
e1af2d6d9c Retire completed schema planning documents 2026-11-20 16:24:16 +00:00
bd6b2848bb Avoid copying unused slice capacity 2026-11-20 16:22:09 +00:00
eae4d18a9c Clarify request value limits for inline schemas 2026-11-20 16:20:27 +00:00
2c436d14d9 Keep prompt selection free of schema payloads 2026-11-20 16:20:27 +00:00
1bfb6eb301 Omit inactive runtime schemas from prepared snapshots 2026-11-20 16:19:29 +00:00
37582e9c30 Record schema copying efficiency finding 2026-11-20 16:13:53 +00:00
4120d784a2 Record schema integration review findings 2026-11-20 16:09:20 +00:00
3bc16bffe6 Record schema snapshot ownership finding 2026-11-20 15:56:42 +00:00
e9a12d43c2 Document prompt-owned schemas and finalize migration 2026-11-20 15:46:39 +00:00
9bed36282a Cover mixed schema sources and captured execution 2026-11-20 15:41:13 +00:00
950d3ccfb2 Verify schema projection and snapshot ownership 2026-11-20 15:38:40 +00:00
6fbbaa9cfa Bind schema selections throughout execution 2026-11-20 15:36:44 +00:00
2d96dfa696 Unify inline and bound schema preparation 2026-11-20 15:31:30 +00:00
be5a508554 List feature considerations for the next release 2026-11-20 15:28:08 +00:00
e8420d229e Capture prompt schema selections and source references 2026-11-20 15:27:46 +00:00
596e985cff Capture schema roots from configuration sources 2026-11-20 15:25:00 +00:00
fbe3d427a2 Add schema selection values and ownership 2026-11-20 15:20:29 +00:00
e775a3c116 Plan prompt-owned schema support 2026-11-20 14:41:43 +00:00
adfccb5370 Settle schema selection and runtime API decisions 2026-11-20 14:19:34 +00:00
0603a4f0dd Record collection schema roots and native composition 2026-11-20 13:10:42 +00:00
eaf9a76fd3 Plan prompt-owned inline and source-relative schemas 2026-11-20 12:52:26 +00:00
3d4c4c940e Update ConfigKit to version 1.3.0 2026-11-20 12:46:09 +00:00
6668673c8d Retire completed message hash roadmaps 2026-11-20 12:32:50 +00:00
019e077ba9 Document prepared message content hashes 2026-11-20 04:22:33 +00:00
718cd05043 Verify prepared message content hash semantics 2026-11-20 04:18:33 +00:00
dd40171c01 Expose content hashes on prepared runs 2026-11-20 04:16:50 +00:00
e415ad43ce Add versioned message content hashing 2026-11-20 04:15:25 +00:00
ae1efc8049 Plan prepared message-content hashing 2026-11-20 04:10:15 +00:00
0f8e0e49b1 Settle message hash compatibility across releases 2026-11-20 03:56:18 +00:00
ffce326dd7 Describe prepared message-content hashing 2026-11-20 03:41:51 +00:00
804a4d32b2 Prepare v0.23.3 release notes 2026-11-19 14:27:21 +00:00
67d446f611 Preserve context and original causes in HTTP failures 2026-11-19 14:07:43 +00:00
bcdd6fe06c Preserve operation timeout identity during HTTP generation 2026-11-19 13:44:42 +00:00
382ccfb14b Refresh dependencies and prepare v0.23.2 release notes 2026-11-17 16:49:53 +00:00
7406616794 Prepare v0.23.1 release notes 2026-11-16 19:55:18 +00:00
4939fdb283 Preserve construction error causes without extra branches 2026-11-16 19:44:09 +00:00
193 changed files with 7026 additions and 1753 deletions

View File

@@ -26,8 +26,8 @@ source-relative prompt content, supplies an inline input, and prepares through
a scoped view. The example also demonstrates collection inventories and a scoped view. The example also demonstrates collection inventories and
namespace-filtered discovery, without contacting a provider or requiring namespace-filtered discovery, without contacting a provider or requiring
credentials. credentials.
The example disables the bundled OpenRouter configuration to keep its The example loads application-owned settings once and selects no bundled
inventories limited to its own offline fixtures. underlay, keeping inventories limited to its own offline fixtures.
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend, The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
ordered file/embedded sources, programmatic mappings, and how to replace an ordered file/embedded sources, programmatic mappings, and how to replace an
engine when source files change. Applications adapting the prior keyed engine when source files change. Applications adapting the prior keyed
@@ -45,6 +45,29 @@ boundary and constraints that framework work must preserve.
## Release Guidance ## Release Guidance
The [v0.25.0 release notes](docs/releases/v0.25.0.md) cover invocation-scoped
observation, provider conventions, reasoning and cache usage, and optional USD
cost reporting. Upgrading consumers should review provider selection,
authentication diagnostic defaults, and the corrected OpenRouter cache-write
mapping.
The [v0.24.0 release notes](docs/releases/v0.24.0.md) cover application-owned
configuration loading, prompt-owned schemas, default output repair, and the
prepared message-content hash. Upgrading consumers should review the breaking
configuration and request-API changes in the
[migration guide](docs/consumers/configuration-migration.md#move-settings-under-promptkit).
The [v0.23.3 release notes](docs/releases/v0.23.3.md) cover HTTP operation-timeout
attribution, custom caller cancellation causes, and response-read error identity.
The [v0.23.2 release notes](docs/releases/v0.23.2.md) cover configuration,
validation, and release-tool dependency updates without an API migration or
Go minimum increase.
The [v0.23.1 release notes](docs/releases/v0.23.1.md) cover corrected constructor
error wrapping for downstream classification, with loading diagnostics and
underlying causes preserved.
The [v0.23.0 release notes](docs/releases/v0.23.0.md) cover per-operation The [v0.23.0 release notes](docs/releases/v0.23.0.md) cover per-operation
timeouts and opt-in prepared-request notifications for ordinary and coordinated timeouts and opt-in prepared-request notifications for ordinary and coordinated
execution. execution.

View File

@@ -0,0 +1,325 @@
package promptkit_test
import (
"bytes"
"context"
"errors"
"fmt"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type integrationComponent struct {
ID string `yaml:"id"`
}
type integrationApplication struct {
Version string `yaml:"version"`
Sibling string `yaml:"sibling"`
Components []integrationComponent `yaml:"components"`
Prompts []string `yaml:"prompts"`
PromptKit promptkit.Settings `yaml:"promptkit"`
}
func integrationDefinition(resolves, validates *atomic.Int64) configkit.Definition[integrationApplication] {
definition := configkit.Definition[integrationApplication]{
Defaults: func() integrationApplication { return integrationApplication{Sibling: "default"} },
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(version configkit.Version, validation *configkit.Validation) error {
if version.Present && version.Text != "app-v1" {
validation.Error(configkit.Keys("version"), "invalid application version")
}
return nil
}},
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("components"), "id")},
Sensitive: func(path configkit.Path) bool { return path.String() == "/sibling" },
Resolve: func(_ *integrationApplication, _ *configkit.Resolution) error { resolves.Add(1); return nil },
Validate: func(_ *integrationApplication, validation *configkit.Validation) {
validates.Add(1)
validation.Warning(configkit.Keys("sibling"), "application warning")
},
}
configured := promptkit.Configure(definition, func(value *integrationApplication) *promptkit.Settings { return &value.PromptKit })
previous := configured.Validate
configured.Validate = func(value *integrationApplication, validation *configkit.Validation) {
previous(value, validation)
if len(value.PromptKit.Backends) != 1 || value.PromptKit.Backends[0].Endpoint == nil || strings.Contains(*value.PromptKit.Backends[0].Endpoint, " ") {
validation.Error(configkit.Keys("promptkit"), "backend normalization missing")
}
if value.PromptKit.Application["section_extra"] != "section" || value.PromptKit.Backends[0].Application["backend_extra"] != "backend" || value.PromptKit.Profiles[0].Application["profile_extra"] != "profile" || value.PromptKit.Prompts[0].Application["prompt_extra"] != "prompt" || value.PromptKit.Collections["group"].Application["collection_extra"] != "collection" {
validation.Error(configkit.Keys("promptkit"), "application fields missing")
}
}
return configured
}
const integrationDocument = `version: app-v1
prompts: [application-only]
components: [{id: second-component}]
promptkit:
section_extra: section
backends:
- id: local
endpoint: ' https://example.test/v1 '
extra_params: {provider: {mode: original}}
backend_extra: backend
profiles:
- id: profile
backend: local
model: original-model
profile_extra: profile
prompts:
- id: example
default_profile: profile
inputs: [{name: document, required: false}]
messages: [{role: user, content: hello}]
output:
format: json
validation_mode: json_schema
schema: {type: object, required: [ok], properties: {ok: {type: boolean}}}
prompt_extra: prompt
collections:
group: {collection_extra: collection}
`
func TestApplicationOwnedLoadAndCapture(t *testing.T) {
var resolves, validates atomic.Int64
definition := integrationDefinition(&resolves, &validates)
result, err := configkit.Load(configkit.Bytes("application", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
if err != nil {
t.Fatal(err)
}
if result.Config.Sibling != "default" || result.Config.Prompts[0] != "application-only" || len(result.Diagnostics()) != 1 || result.Config.PromptKit.Prompts[0].Output.RepairAttempts == nil || *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
t.Fatalf("loaded application = %+v, diagnostics = %+v", result.Config, result.Diagnostics())
}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"ok":true}`}}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
if resolves.Load() != 1 || validates.Load() != 1 {
t.Fatalf("callbacks rerun during handoff: %d %d", resolves.Load(), validates.Load())
}
before, err := engine.InspectPrompt(context.Background(), "example")
if err != nil {
t.Fatal(err)
}
app := &result.Config.PromptKit
*app.Backends[0].Endpoint = "https://changed.test/v1"
app.Backends[0].ExtraParams["provider"].(map[string]any)["mode"] = "changed"
*app.Profiles[0].Model = "changed-model"
*app.Prompts[0].Messages[0].Content = "changed"
app.Prompts[0].Inputs[0].Name = "changed"
app.Prompts[0].Output.Schema.(map[string]any)["required"].([]any)[0] = "changed"
app.Prompts = nil
app.Collections = nil
after, err := engine.InspectPrompt(context.Background(), "example")
if err != nil || !reflect.DeepEqual(before, after) {
t.Fatalf("captured prompt changed: %+v, %+v, %v", before, after, err)
}
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "example"})
if err != nil || run.Validation.Status != promptkit.ValidationPassed || len(client.requests) != 1 || client.requests[0].Prompt.Messages[0].Content != "hello" || client.requests[0].Target.Model != "original-model" {
t.Fatalf("run = %+v, requests = %+v, error = %v", run, client.requests, err)
}
if strings.Contains(fmt.Sprintf("%+v", client.requests[0].Target.ExtraParams), "backend_extra") || strings.Contains(fmt.Sprintf("%+v", after), "prompt_extra") {
t.Fatal("application fields leaked into resources or provider parameters")
}
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
var report bytes.Buffer
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "section_extra: section") || !strings.Contains(report.String(), "prompt_extra: prompt") {
t.Fatalf("inspection = %q, %v", report.String(), err)
}
if len(view.Diagnostics()) != 1 {
t.Fatalf("diagnostics = %+v", view.Diagnostics())
}
var wg sync.WaitGroup
for range 4 {
wg.Go(func() {
_, err := configkit.Load(configkit.Bytes("repeat", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
if err != nil {
t.Error(err)
}
})
}
wg.Wait()
if resolves.Load() != 5 || validates.Load() != 5 {
t.Fatalf("reused callbacks = %d %d", resolves.Load(), validates.Load())
}
}
func TestHandoffMountedSourcesAndExecution(t *testing.T) {
baseFS := &countingMountFS{FS: fstest.MapFS{
"base/collection.yaml": {Data: []byte("prompts:\n - id: parent\n default_profile: profile\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")},
"base/original.txt": {Data: []byte("original")}, "base/schema.json": {Data: []byte(`{"type":"object"}`)},
}}
operatorFS := &countingMountFS{FS: fstest.MapFS{
"operator/item.yaml": {Data: []byte("id: child\nextends: {id: parent}\nmessages: [{role: user, content_file: replacement.txt}]\n")},
"operator/replacement.txt": {Data: []byte("replacement")},
}}
base := configkit.NewFilesystem("embedded", baseFS)
operator := configkit.NewFilesystem("operator", operatorFS)
collection, err := promptkit.CollectionInput("team", base.File("base/collection.yaml"))
if err != nil {
t.Fatal(err)
}
collection.Filesystem = &configkit.FilesystemContext{FS: &base, Root: "."}
item := configkit.Input{Source: operator.File("operator/item.yaml"), Destination: configkit.Keys("promptkit", "collections", "team", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem, Filesystem: &configkit.FilesystemContext{FS: &operator, Root: "."}}
profile := promptkit.SettingsInput(configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.test/v1", "model": "model"}}}))
type root struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
layers := []configkit.Underlay{{Inputs: []configkit.Input{profile, collection}}, {Inputs: []configkit.Input{item}}}
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: layers})
if err != nil {
t.Fatal(err)
}
beforeBase, beforeOperator := baseFS.opens, operatorFS.opens
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{}`}}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
if baseFS.opens-beforeBase != 1 || operatorFS.opens-beforeOperator != 1 {
t.Fatalf("unexpected construction reads: base=%d operator=%d", baseFS.opens-beforeBase, operatorFS.opens-beforeOperator)
}
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: layers}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
wantPrompt, err := standalone.InspectPrompt(context.Background(), "team/child")
if err != nil {
t.Fatal(err)
}
gotPrompt, err := engine.InspectPrompt(context.Background(), "team/child")
if err != nil || !reflect.DeepEqual(gotPrompt, wantPrompt) {
t.Fatalf("resource reports = %+v, %+v, %v", gotPrompt, wantPrompt, err)
}
wantInventory, err := standalone.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
gotInventory, err := engine.ListPrompts(context.Background())
if err != nil || !reflect.DeepEqual(gotInventory, wantInventory) {
t.Fatalf("inventories = %+v, %+v, %v", gotInventory, wantInventory, err)
}
collectionSettings := result.Config.PromptKit.Collections["team"]
*collectionSettings.Prompts[1].Output.SchemaFile = "changed.json"
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
if err != nil || prepared.Messages[0].Content != "replacement" {
t.Fatalf("prepared = %+v, %v", prepared, err)
}
standalonePrepared, err := standalone.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
if err != nil || !reflect.DeepEqual(prepared.StructuredOutput, standalonePrepared.StructuredOutput) || !reflect.DeepEqual(prepared.OutputContract, standalonePrepared.OutputContract) || !reflect.DeepEqual(prepared.Messages, standalonePrepared.Messages) {
t.Fatalf("prepared details differ: %+v, %+v, %v", prepared, standalonePrepared, err)
}
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
if err != nil || run.Validation.Status != promptkit.ValidationPassed {
t.Fatalf("run = %+v, %v", run, err)
}
}
func TestHandoffUsesOnlyBundlesSelectedBeforeLoad(t *testing.T) {
type root struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{promptkit.BundledOpenRouterUnderlay()}})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(false))
if err != nil {
t.Fatal(err)
}
backends, err := engine.ListBackends(context.Background())
if err != nil || len(backends) != 1 || backends[0] != "openrouter" {
t.Fatalf("bundles = %+v, %v", backends, err)
}
}
func TestApplicationFieldsComposeAndInherit(t *testing.T) {
type root struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
underlay := promptkit.SettingsInput(configkit.Mapping("defaults", map[string]any{
"section_data": map[string]any{"first": "section"},
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.test/v1", "app_data": map[string]any{"first": "backend"}}},
"profiles": []any{map[string]any{"id": "base", "model": "model", "app_data": map[string]any{"first": "profile"}}},
"prompts": []any{map[string]any{"id": "base", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "app_data": map[string]any{"first": "prompt"}}},
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"first": "collection"}}},
}))
main := promptkit.SettingsInput(configkit.Mapping("operator", map[string]any{
"section_data": map[string]any{"second": "section"},
"backends": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "backend"}}},
"profiles": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "profile"}}},
"prompts": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "prompt"}}},
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"second": "collection"}}},
}))
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}})
if err != nil {
t.Fatal(err)
}
settings := result.Config.PromptKit
for name, fields := range map[string]map[string]any{"section": settings.Application, "backend": settings.Backends[1].Application, "profile": settings.Profiles[1].Application, "prompt": settings.Prompts[1].Application, "collection": settings.Collections["team"].Application} {
key := "app_data"
if name == "section" {
key = "section_data"
}
data, ok := fields[key].(map[string]any)
if !ok || data["first"] != name || data["second"] != name {
t.Errorf("%s extension = %#v", name, fields[key])
}
}
}
func TestHandoffMissingAndNullSections(t *testing.T) {
type noSection struct {
Prompts []string `yaml:"prompts"`
}
missing, err := configkit.Load[noSection](configkit.Bytes("application-only", []byte("prompts: [application]\n")))
if err != nil {
t.Fatal(err)
}
empty, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
if prompts, err := empty.ListPrompts(context.Background()); err != nil || len(prompts) != 0 {
t.Fatalf("application root resources were consumed: %+v, %v", prompts, err)
}
for _, source := range []string{"{}", "promptkit: null", "promptkit: {}"} {
t.Run(source, func(t *testing.T) {
type root struct {
PromptKit *promptkit.Settings `yaml:"promptkit"`
Prompts []string `yaml:"prompts"`
}
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return value.PromptKit })
result, err := configkit.Load(configkit.Bytes("missing", []byte(source)), configkit.Options[root]{Definition: definition})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil || len(prompts) != 0 {
t.Fatalf("prompts = %+v, %v", prompts, err)
}
})
}
if _, err := promptkit.NewFromConfig(nil, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil result = %v", err)
}
}

View File

@@ -0,0 +1,90 @@
package promptkit_test
import (
"context"
"errors"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestApplicationDefaultInputInspection(t *testing.T) {
for _, derived := range []bool{false, true} {
name := "Go defaults"
if derived {
name = "derived collection defaults"
}
t.Run(name, func(t *testing.T) {
settings := promptkit.Settings{
Profiles: []promptkit.ProfileSettings{{ID: "profile", Endpoint: new("https://example.test/v1"), Model: new("model")}},
Prompts: []promptkit.PromptSettings{{ID: "prompt", DefaultProfile: new("profile"),
Inputs: []promptkit.PromptInputSettings{{Name: "doc", Required: true, ContentType: new("text/plain"), Description: new("")}},
Messages: []promptkit.PromptMessageSettings{{Role: new("user"), Content: new("hello")}},
Output: &promptkit.PromptOutputSettings{Format: new("text"), ValidationMode: new("none")},
}},
}
definition := configkit.Definition[handoffApplication]{Defaults: func() handoffApplication { return handoffApplication{PromptKit: settings} }}
source := configkit.NoSource()
id := "prompt"
if derived {
definition.Defaults = nil
source = configkit.Bytes("derived inputs", []byte(`promptkit:
profiles: [{id: profile, endpoint: https://example.test/v1, model: model}]
collections:
team:
prompts:
- id: prompt
default_profile: profile
inputs: [{name: doc}]
messages: [{role: user, content: hello}]
output: {format: text, validation_mode: none}
`))
id = "team/prompt"
definition.Resolve = func(value *handoffApplication, resolution *configkit.Resolution) error {
input := &value.PromptKit.Collections["team"].Prompts[0].Inputs[0]
input.Required, input.ContentType, input.Description = true, new("text/plain"), new("")
base := configkit.Keys("promptkit", "collections", "team", "prompts").Index(0).Key("inputs").Index(0)
for _, field := range []string{"required", "content_type", "description"} {
resolution.RecordDerivedDefault(base.Key(field), "Application input default", base.Key("name"))
}
return nil
}
}
definition = promptkit.Configure(definition, func(value *handoffApplication) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.Load(source, configkit.Options[handoffApplication]{Definition: definition})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
resource, err := engine.InspectPrompt(ctx, id)
if err != nil {
t.Fatal(err)
}
input := (*resource.Inputs)[0]
if input.Required == nil || !*input.Required || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil || *input.Description != "" {
t.Fatalf("resource input = %+v", input)
}
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: id})
if err != nil {
t.Fatal(err)
}
required := requirements.Inputs[0]
if required.Kind != promptkit.RequirementRequired || required.ContentType == nil || *required.ContentType != "text/plain" || required.Description == nil || *required.Description != "" {
t.Fatalf("requirement = %+v", required)
}
request := promptkit.RunRequest{PromptID: id}
if err := engine.ValidateRequest(ctx, request); !errors.Is(err, promptkit.ErrPromptRender) {
t.Fatalf("missing input: %v", err)
}
request.Inputs = map[string]promptkit.ArtifactRef{"doc": promptkit.Inline("document")}
if err := engine.ValidateRequest(ctx, request); err != nil {
t.Fatal(err)
}
})
}
}

View File

@@ -17,7 +17,7 @@ func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactR
} }
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) { func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
input := configkit.Input{Source: configkit.Mapping("ownership", map[string]any{ input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("ownership", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}}, "profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})} })}

View File

@@ -73,6 +73,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
output := map[string]any{"format": "text", "validation_mode": "none"} output := map[string]any{"format": "text", "validation_mode": "none"}
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"} profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})} options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
var underlays []configkit.Underlay
var expectedMessage string var expectedMessage string
switch name { switch name {
@@ -91,9 +92,9 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json", "format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
} }
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response}) options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{ underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)}, "workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
}, ".")) }, ".")}})
default: default:
b.Fatalf("unknown assembled workflow fixture %q", name) b.Fatalf("unknown assembled workflow fixture %q", name)
} }
@@ -106,8 +107,8 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
"output": output, "output": output,
}}, }},
} }
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
Source: configkit.Mapping("assembled workflow benchmark", config), Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("assembled workflow benchmark", config),
}}}, options...) }}}, options...)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)

View File

@@ -26,6 +26,10 @@ type BackendConfig struct {
ID string ID string
// Extends names a sibling backend to inherit through ConfigKit. // Extends names a sibling backend to inherit through ConfigKit.
Extends *string Extends *string
// Provider selects the backend's supported API conventions.
Provider *string
// ZeroCost enables a zero-cost fallback when no usable charge is reported.
ZeroCost *bool
// Endpoint is the required absolute HTTP(S) base URL after all layers merge. // Endpoint is the required absolute HTTP(S) base URL after all layers merge.
Endpoint *string Endpoint *string
// APIKeyEnv optionally names an environment variable, not a credential. // APIKeyEnv optionally names an environment variable, not a credential.
@@ -41,41 +45,6 @@ type BackendConfig struct {
SmokeTests *BackendSmokeTestConfig SmokeTests *BackendSmokeTestConfig
} }
// BackendSmokeTestConfig contains optional diagnostic settings associated with
// a backend. Nil nested fields are omitted by BackendSource.
type BackendSmokeTestConfig struct {
// DefaultProfile is the exact profile ID selected as the backend's
// diagnostic inference default. A supplied value must not be blank; the ID
// is resolved only when that default is selected.
DefaultProfile *string `json:"default_profile,omitempty" yaml:"default_profile,omitempty"`
// Reachability optionally selects a full HTTP destination for reachability
// diagnostics. A nil endpoint uses the effective backend endpoint.
Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"`
// Authentication optionally declares a named authentication diagnostic.
// A supplied object requires Kind.
Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"`
}
// ReachabilityProbeConfig configures an optional reachability destination.
type ReachabilityProbeConfig struct {
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
// Nil selects the effective backend endpoint; an explicit blank value is
// invalid.
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
}
// AuthenticationProbeConfig configures an optional named authentication
// diagnostic.
type AuthenticationProbeConfig struct {
// Kind selects the authentication adapter. Supported values are "none" and
// "openrouter_key"; surrounding whitespace and case are normalized.
Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"`
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
// Nil selects the adapter's backend-relative destination; an explicit blank
// value is invalid.
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
}
// BackendSource creates a labeled ConfigKit mapping containing backend records // BackendSource creates a labeled ConfigKit mapping containing backend records
// in caller order. IDs are explicit and duplicates are left for loading to reject. // in caller order. IDs are explicit and duplicates are left for loading to reject.
// It encodes fields without validating them; NewEngine validates the final // It encodes fields without validating them; NewEngine validates the final
@@ -90,6 +59,12 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
if config.Extends != nil { if config.Extends != nil {
fields["extends"] = map[string]any{"id": *config.Extends} fields["extends"] = map[string]any{"id": *config.Extends}
} }
if config.Provider != nil {
fields["provider"] = *config.Provider
}
if config.ZeroCost != nil {
fields["zero_cost"] = *config.ZeroCost
}
if config.Endpoint != nil { if config.Endpoint != nil {
fields["endpoint"] = *config.Endpoint fields["endpoint"] = *config.Endpoint
} }

View File

@@ -11,7 +11,7 @@ import (
) )
func TestBundledCompositionReplacementAndInheritance(t *testing.T) { func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
underlay := configkit.Input{Source: configkit.Mapping("caller underlay", map[string]any{ underlay := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller underlay", map[string]any{
"profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}}, "profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}},
})} })}
underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}}) underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}})
@@ -22,7 +22,7 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil { if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err) t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
} }
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{ main := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller main", map[string]any{
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}}, "backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
"profiles": []any{ "profiles": []any{
map[string]any{"id": "gpt-sol-latest", "model": "main-model"}, map[string]any{"id": "gpt-sol-latest", "model": "main-model"},
@@ -76,8 +76,8 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
{"replace", "replace", "custom", false, false}, {"replace", "replace", "custom", false, false},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{ input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("sequence", map[string]any{
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"profiles"}, "mode": test.mode}}}, "config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "profiles"}, "mode": test.mode}}},
"profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}}, "profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}},
})} })}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
@@ -99,13 +99,13 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
} }
func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) { func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) {
profile := configkit.Input{Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})} profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})}
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}}) _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}})
if err == nil { if err == nil {
t.Fatal("strict group accepted duplicate records") t.Fatal("strict group accepted duplicate records")
} }
input := configkit.Input{Source: configkit.Mapping("replace backends", map[string]any{ input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replace backends", map[string]any{
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"backends"}, "mode": "replace"}}}, "config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "backends"}, "mode": "replace"}}},
"backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}}, "backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
})} })}
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil { if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil {

View File

@@ -15,7 +15,7 @@ import (
) )
func bundledPromptConfig(client *http.Client) promptkit.Config { func bundledPromptConfig(client *http.Client) promptkit.Config {
input := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{ input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"},
}}})} }}})}
return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}} return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}

View File

@@ -69,7 +69,7 @@ func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") { if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") {
t.Fatal("bundle source missing") t.Fatal("bundle source missing")
} }
if explanation, found := view.Explain(configkit.Keys("backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 { if explanation, found := view.Explain(configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 {
t.Fatalf("bundle explanation = %+v", explanation) t.Fatalf("bundle explanation = %+v", explanation)
} }
} }
@@ -94,7 +94,7 @@ func fmtSources(sources []configkit.SourceInfo) string {
func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) { func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}}) source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}})
groups := make([]configkit.Underlay, 1, 2) groups := make([]configkit.Underlay, 1, 2)
groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Source: source}}} groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}
before := append([]configkit.Underlay(nil), groups...) before := append([]configkit.Underlay(nil), groups...)
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil { if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -105,7 +105,7 @@ func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
} }
func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) { func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) {
input := configkit.Input{Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})} input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false)) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -122,7 +122,7 @@ func TestBundledSmokeHTTPDefaults(t *testing.T) {
} }
func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) { func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
replacement := configkit.Input{Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})} replacement := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}} replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient)) replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
if err != nil { if err != nil {
@@ -137,7 +137,7 @@ func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
if err != nil || mismatch == nil || !mismatch.Completed { if err != nil || mismatch == nil || !mismatch.Completed {
t.Fatalf("concluded mismatch = %+v, %v", mismatch, err) t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
} }
input := configkit.Input{Source: configkit.Mapping("application", map[string]any{ input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("application", map[string]any{
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}}, "backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}}, "profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},

View File

@@ -0,0 +1,70 @@
package promptkit_test
import (
"context"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestDuplicateContextAndRequestGroupEnrollment(t *testing.T) {
charge := .5
engine := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe}); err != nil {
t.Fatal(err)
}
got := group.Snapshot()
if len(got.Members) != 1 || got.Attempts != 1 || got.CostUSD.Value != .5 {
t.Fatalf("summary=%+v", got)
}
}
func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) {
known := func(v int) promptkit.ObservedTokenCount {
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
}
for _, tc := range []struct {
name string
input, reads promptkit.ObservedTokenCount
ratio float64
ok bool
}{
{name: "fraction", input: known(8), reads: known(3), ratio: .375, ok: true},
{name: "zero reads", input: known(8), reads: known(0), ok: true},
{name: "zero input", input: known(0), reads: known(0)},
{name: "negative reads", input: known(8), reads: known(-1)},
{name: "reads exceed input", input: known(8), reads: known(9)},
{name: "unknown", input: known(8)},
{name: "incomplete", input: known(8), reads: promptkit.ObservedTokenCount{Value: 3, Known: true}},
} {
t.Run(tc.name, func(t *testing.T) {
usage := promptkit.ObservedTokenUsage{PromptTokens: tc.input, CachedTokens: tc.reads}
got, ok := usage.CacheHitRatio()
if ok != tc.ok || math.Abs(got-tc.ratio) > 1e-12 {
t.Fatalf("ratio=%v available=%t", got, ok)
}
})
}
}
func TestGroupCacheRatioUsesSummedTokensAndDeduplicatesTerminals(t *testing.T) {
known := func(v int) promptkit.ObservedTokenCount {
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
}
var group promptkit.ExecutionGroup
first := promptkit.ExecutionEvent{RunID: "a", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(10), CachedTokens: known(10), ReasoningTokens: known(1)}}
second := promptkit.ExecutionEvent{RunID: "b", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(90), CachedTokens: known(0), ReasoningTokens: known(2)}}
group.Observe(first)
group.Observe(first)
group.Observe(second)
snapshot := group.Snapshot()
ratio, ok := snapshot.Usage.CacheHitRatio()
if !ok || math.Abs(ratio-.1) > 1e-12 || snapshot.Usage.ReasoningTokens != known(3) {
t.Fatalf("summary=%+v ratio=%v available=%t", snapshot, ratio, ok)
}
}

View File

@@ -265,7 +265,7 @@ func newBackendCapacityEngine(
options = append(options, promptkit.WithArtifactReader(reader)) options = append(options, promptkit.WithArtifactReader(reader))
} }
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("capacity fixture", map[string]any{ Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("capacity fixture", map[string]any{
"backends": []any{backend}, "backends": []any{backend},
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}}, "profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
"prompts": []any{definition}, "prompts": []any{definition},

38
configuration.go Normal file
View File

@@ -0,0 +1,38 @@
package promptkit
import (
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/configuration"
)
// Configure composes PromptKit's settings rules with an application definition.
// selectSettings identifies the application's PromptKit section and must be
// non-nil. A nil selected section is left untouched. Existing resolution and
// validation callbacks run first; callers may wrap the returned callbacks to
// run additional work afterward. Defaults and version policy are preserved.
func Configure[T any](definition configkit.Definition[T], selectSettings func(*T) *Settings) configkit.Definition[T] {
previousResolve := definition.Resolve
previousValidate := definition.Validate
previousSensitive := definition.Sensitive
identities := append([]configkit.SequenceIdentity(nil), definition.SequenceIdentities...)
definition.SequenceIdentities = append(identities, configuration.SequenceIdentities()...)
definition.Resolve = func(value *T, resolution *configkit.Resolution) error {
if previousResolve != nil {
if err := previousResolve(value, resolution); err != nil {
return err
}
}
configuration.Resolve(selectSettings(value), resolution)
return nil
}
definition.Validate = func(value *T, validation *configkit.Validation) {
if previousValidate != nil {
previousValidate(value, validation)
}
configuration.Validate(selectSettings(value), validation)
}
definition.Sensitive = func(path configkit.Path) bool {
return (previousSensitive != nil && previousSensitive(path)) || configuration.Sensitive(path)
}
return definition
}

View File

@@ -21,7 +21,7 @@ func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences) t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
} }
difference := differences[0] difference := differences[0]
if got, want := difference.Path.String(), "/profiles"; got != want { if got, want := difference.Path.String(), "/promptkit/profiles"; got != want {
t.Fatalf("difference path = %q, want %q", got, want) t.Fatalf("difference path = %q, want %q", got, want)
} }
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") { if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
@@ -84,7 +84,7 @@ func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) { func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
inspect := func(endpoint string) promptkit.ConfigurationInspection { inspect := func(endpoint string) promptkit.ConfigurationInspection {
t.Helper() t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("backend", map[string]any{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("backend", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": endpoint}}, "backends": []any{map[string]any{"id": "local", "endpoint": endpoint}},
})}}}}}}) })}}}}}})
if err != nil { if err != nil {
@@ -104,7 +104,7 @@ func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
} }
changed := inspect("https://other.invalid/v1") changed := inspect("https://other.invalid/v1")
differences, err = left.Diff(changed) differences, err = left.Diff(changed)
if err != nil || len(differences) != 1 || differences[0].Path.String() != "/backends" { if err != nil || len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
t.Fatalf("changed diff = %+v, %v", differences, err) t.Fatalf("changed diff = %+v, %v", differences, err)
} }
} }
@@ -122,7 +122,7 @@ func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("compare presence and sequence values: %v", err) t.Fatalf("compare presence and sequence values: %v", err)
} }
if len(differences) != 1 || differences[0].Path.String() != "/backends" { if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
t.Fatalf("atomic backend list difference = %+v", differences) t.Fatalf("atomic backend list difference = %+v", differences)
} }
difference := differences[0] difference := differences[0]
@@ -147,7 +147,7 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
if err != nil { if err != nil {
t.Fatalf("compare sensitive values: %v", err) t.Fatalf("compare sensitive values: %v", err)
} }
if len(differences) != 1 || differences[0].Path.String() != "/backends" { if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
t.Fatalf("sensitive backend list difference = %+v", differences) t.Fatalf("sensitive backend list difference = %+v", differences)
} }
secret := differences[0] secret := differences[0]
@@ -176,7 +176,7 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
if _, exists := againByPath["/caller/mutation"]; exists { if _, exists := againByPath["/caller/mutation"]; exists {
t.Fatal("mutating returned path reached retained inspection") t.Fatal("mutating returned path reached retained inspection")
} }
if _, exists := againByPath["/backends"]; !exists { if _, exists := againByPath["/promptkit/backends"]; !exists {
t.Fatalf("original backend path was lost after mutation: %+v", again) t.Fatalf("original backend path was lost after mutation: %+v", again)
} }
for _, difference := range again { for _, difference := range again {
@@ -201,7 +201,7 @@ func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T)
if err != nil { if err != nil {
t.Fatalf("compare sensitive shape replacement: %v", err) t.Fatalf("compare sensitive shape replacement: %v", err)
} }
if len(differences) != 1 || differences[0].Path.String() != "/backends" { if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
t.Fatalf("shape replacement differences = %+v", differences) t.Fatalf("shape replacement differences = %+v", differences)
} }
difference := differences[0] difference := differences[0]
@@ -225,7 +225,7 @@ func comparisonInspectionFrom(t *testing.T, source, model string, extraParams ma
backend["extra_params"] = extraParams backend["extra_params"] = extraParams
} }
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping(source, map[string]any{
"backends": []any{backend}, "backends": []any{backend},
"profiles": []any{map[string]any{ "profiles": []any{map[string]any{
"id": "offline", "backend": "local", "id": "offline", "backend": "local",

View File

@@ -0,0 +1,76 @@
package promptkit_test
import (
"context"
"errors"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type handoffApplication struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
type handoffWrapper struct {
result *configkit.Result[handoffApplication]
override any
useOverride bool
inspection bool
}
func (w *handoffWrapper) Value(path configkit.Path) (any, bool) {
if w.useOverride {
return w.override, true
}
return w.result.Value(path)
}
func (w *handoffWrapper) InspectSubtree(path configkit.Path) (configkit.Inspection, bool) {
if !w.inspection {
return configkit.Inspection{}, false
}
return w.result.InspectSubtree(path)
}
func (w *handoffWrapper) Diagnostics() []configkit.Diagnostic { return w.result.Diagnostics() }
func TestConstructorsShareCapturedResources(t *testing.T) {
source := configkit.Bytes("prompt", []byte("promptkit:\n prompts:\n - id: example\n messages: [{role: user, content: hello}]\n output: {format: text, validation_mode: basic}\n"))
definition := promptkit.Configure(configkit.Definition[handoffApplication]{}, func(value *handoffApplication) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.Load(source, configkit.Options[handoffApplication]{Definition: definition})
if err != nil {
t.Fatal(err)
}
wrapped := &handoffWrapper{result: result, inspection: true}
applicationEngine, err := promptkit.NewFromConfig(wrapped, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(true))
if err != nil {
t.Fatal(err)
}
input := configkit.Input{Source: source}
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
if err != nil {
t.Fatal(err)
}
for _, engine := range []*promptkit.Engine{applicationEngine, standalone} {
prompt, err := engine.InspectPrompt(context.Background(), "example")
if err != nil || prompt.Messages[0].Content != "hello" || prompt.OutputContract.RepairAttempts != 1 {
t.Fatalf("prompt = %+v, %v", prompt, err)
}
backends, err := engine.ListBackends(context.Background())
if err != nil || len(backends) != 0 {
t.Fatalf("backends = %+v, %v", backends, err)
}
}
}
func TestNewFromConfigRejectsInvalidHandoff(t *testing.T) {
var nilResult *handoffWrapper
if _, err := promptkit.NewFromConfig(nilResult, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil result: %v", err)
}
for _, wrapper := range []*handoffWrapper{{inspection: false}, {inspection: true, useOverride: true, override: "wrong"}} {
if _, err := promptkit.NewFromConfig(wrapper, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("invalid handoff: %v", err)
}
}
}

View File

@@ -9,8 +9,10 @@ import (
) )
// ConfigurationInspection is a read-only view of one Engine's captured // ConfigurationInspection is a read-only view of one Engine's captured
// effective configuration and source history. It does not expose typed // effective configuration and source history focused on /promptkit. Metadata,
// configuration values, filesystem contexts, or runtime credentials. // sources, and explicit path queries retain native whole-load evidence; the
// focus is not an isolation boundary. It does not expose typed configuration
// values, filesystem contexts, or runtime credentials.
// Report content may include application prompt and provider data, so callers // Report content may include application prompt and provider data, so callers
// choose where and to whom to disclose it. // choose where and to whom to disclose it.
type ConfigurationInspection struct { type ConfigurationInspection struct {
@@ -30,7 +32,9 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
// Diff compares this view's captured effective configuration with other. This // Diff compares this view's captured effective configuration with other. This
// view is the earlier (left) value and other is the later (right) value. The // view is the earlier (left) value and other is the later (right) value. The
// comparison delegates to ConfigKit's frozen effective-value semantics, so // comparison delegates to ConfigKit's frozen effective-value semantics, so
// source or resolution history alone does not create differences. It does not // source or resolution history alone does not create differences. Native
// ancestor differences are retained, while sibling-only records are excluded.
// It does not
// reread external sources or compare referenced file contents, resolved // reread external sources or compare referenced file contents, resolved
// credential contents, later execution preparation, or request-specific // credential contents, later execution preparation, or request-specific
// overrides. Either zero view returns ErrInvalidConfig and no records; other // overrides. Either zero view returns ErrInvalidConfig and no records; other
@@ -52,6 +56,14 @@ func (v ConfigurationInspection) Metadata() configkit.Metadata {
return v.source.Metadata() return v.source.Metadata()
} }
// HistoricalPaths lists native historical paths within the PromptKit focus.
func (v ConfigurationInspection) HistoricalPaths() []configkit.Path {
if v.source == nil {
return nil
}
return v.source.HistoricalPaths()
}
// Sources returns a detached list of loaded source descriptions. // Sources returns a detached list of loaded source descriptions.
func (v ConfigurationInspection) Sources() []configkit.SourceInfo { func (v ConfigurationInspection) Sources() []configkit.SourceInfo {
if v.source == nil { if v.source == nil {
@@ -78,7 +90,8 @@ func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explana
return v.source.Explain(path) return v.source.Explain(path)
} }
// WriteEffective writes ConfigKit's redacted effective YAML. Writer errors // WriteEffective writes the focused ConfigKit-redacted effective YAML. An absent
// effective focus fails without output; explicit null renders normally. Writer errors
// remain available through errors.Is and errors.As. The zero view returns // remain available through errors.Is and errors.As. The zero view returns
// ErrInvalidConfig. // ErrInvalidConfig.
func (v ConfigurationInspection) WriteEffective(w io.Writer) error { func (v ConfigurationInspection) WriteEffective(w io.Writer) error {

View File

@@ -19,7 +19,7 @@ import (
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) { func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{ Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{ {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("defaults", map[string]any{
"backends": []any{map[string]any{ "backends": []any{map[string]any{
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY", "id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY",
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"}, "extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
@@ -35,7 +35,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
}, },
}}, }},
})}}}, })}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{ {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("operator", map[string]any{
"collections": map[string]any{"literal": map[string]any{ "collections": map[string]any{"literal": map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}}, "backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}}, "profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
@@ -50,21 +50,21 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
if err != nil { if err != nil {
t.Fatalf("inspect configuration: %v", err) t.Fatalf("inspect configuration: %v", err)
} }
modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model") modelPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(0).Key("model")
model, found := view.Explain(modelPath) model, found := view.Explain(modelPath)
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 { if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
t.Fatalf("layered model explanation = (%+v, %t)", model, found) t.Fatalf("layered model explanation = (%+v, %t)", model, found)
} }
inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model") inheritedPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(1).Key("model")
inherited, found := view.Explain(inheritedPath) inherited, found := view.Explain(inheritedPath)
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) { if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found) t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
} }
if inheritedPath.String() != "/collections/literal/profiles/1/model" { if inheritedPath.String() != "/promptkit/collections/literal/profiles/1/model" {
t.Fatalf("literal-key path = %q", inheritedPath) t.Fatalf("literal-key path = %q", inheritedPath)
} }
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token") secretPath := configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("token")
endpointPath := configkit.Keys("backends").Index(0).Key("endpoint") endpointPath := configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")
endpoint, found := view.Explain(endpointPath) endpoint, found := view.Explain(endpointPath)
if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") { if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") {
t.Fatalf("normalized endpoint explanation = %+v", endpoint) t.Fatalf("normalized endpoint explanation = %+v", endpoint)
@@ -73,12 +73,12 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") { if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found) t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
} }
namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token") namespacedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
namespacedSecret, found := view.Explain(namespacedSecretPath) namespacedSecret, found := view.Explain(namespacedSecretPath)
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") { if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found) t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
} }
inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token") inheritedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
inheritedSecret, found := view.Explain(inheritedSecretPath) inheritedSecret, found := view.Explain(inheritedSecretPath)
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") { if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found) t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
@@ -124,7 +124,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) { func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("invalid backend", map[string]any{ Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid backend", map[string]any{
"collections": map[string]any{"private": map[string]any{"backends": []any{ "collections": map[string]any{"private": map[string]any{"backends": []any{
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}}, map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
}}}, }}},
@@ -139,9 +139,9 @@ func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing
} }
} }
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) { func TestConfigurationInspectionRetainsApplicationVersionField(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("main", map[string]any{"version": "2"}), Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("main", map[string]any{"version": "2"}),
}}}) }}})
if err != nil { if err != nil {
t.Fatalf("construct engine with accepted declaration: %v", err) t.Fatalf("construct engine with accepted declaration: %v", err)
@@ -150,7 +150,7 @@ func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T
if err != nil { if err != nil {
t.Fatalf("inspect configuration: %v", err) t.Fatalf("inspect configuration: %v", err)
} }
path := configkit.Keys("version") path := configkit.Keys("promptkit", "version")
field, found := inspection.Explain(path) field, found := inspection.Explain(path)
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") { if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found) t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
@@ -171,7 +171,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
t.Fatalf("nil engine inspection = %v", err) t.Fatalf("nil engine inspection = %v", err)
} }
var empty promptkit.ConfigurationInspection var empty promptkit.ConfigurationInspection
path := configkit.Keys("profiles", "missing") path := configkit.Keys("promptkit", "profiles", "missing")
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 { if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
t.Fatal("zero inspection reported configuration") t.Fatal("zero inspection reported configuration")
} }
@@ -185,7 +185,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
t.Fatalf("zero explanation writer = %v", err) t.Fatalf("zero explanation writer = %v", err)
} }
profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{ profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("inspection test profile", map[string]any{
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}}, "profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
})} })}
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
@@ -203,7 +203,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) { if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
t.Fatalf("effective writer error = %v", err) t.Fatalf("effective writer error = %v", err)
} }
if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) { if err := view.WriteExplanation(writer, configkit.Keys("promptkit", "profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
t.Fatalf("explanation writer error = %v", err) t.Fatalf("explanation writer error = %v", err)
} }
} }
@@ -230,7 +230,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
client := countingLLMClient{calls: &modelCalls} client := countingLLMClient{calls: &modelCalls}
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model" endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{ Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(inputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("offline profile", map[string]any{
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}}, "profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
})}}}}, })}}}},
}}, promptkit.WithLLMClient(client)) }}, promptkit.WithLLMClient(client))
@@ -238,7 +238,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
t.Fatalf("construct engine: %v", err) t.Fatalf("construct engine: %v", err)
} }
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{ Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(otherInputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("comparison profile", map[string]any{
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}}, "profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
})}}}}, })}}}},
}}, promptkit.WithLLMClient(client)) }}, promptkit.WithLLMClient(client))
@@ -261,7 +261,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err != nil || len(differences) != 1 { if err != nil || len(differences) != 1 {
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err) t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
} }
if differences[0].Path.String() != "/profiles" || if differences[0].Path.String() != "/promptkit/profiles" ||
!strings.Contains(differences[0].Left.YAML, model) || !strings.Contains(differences[0].Left.YAML, model) ||
!strings.Contains(differences[0].Right.YAML, otherModel) { !strings.Contains(differences[0].Right.YAML, otherModel) {
t.Fatalf("captured difference = %+v", differences[0]) t.Fatalf("captured difference = %+v", differences[0])
@@ -273,7 +273,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0]) t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
} }
} }
if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found { if _, found := view.Explain(configkit.Keys("promptkit", "prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
t.Fatal("captured prompt content path was not inspectable") t.Fatal("captured prompt content path was not inspectable")
} }
var report bytes.Buffer var report bytes.Buffer

View File

@@ -0,0 +1,60 @@
package promptkit_test
import (
"io/fs"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type mountApplication struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
type countingMountFS struct {
fs.FS
opens int
}
func (c *countingMountFS) Open(name string) (fs.File, error) { c.opens++; return c.FS.Open(name) }
func TestSettingsAndCollectionMounts(t *testing.T) {
files := &countingMountFS{FS: fstest.MapFS{
"fragments/root.yaml": {Data: []byte("profiles:\n - id: main\n model: model\n")},
"fragments/collection.yaml": {Data: []byte("prompts:\n - id: child\n messages: [{role: user, content_file: body.txt}]\n output: {format: text, validation_mode: basic}\n")},
"fragments/item.yaml": {Data: []byte("id: item\nmessages: [{role: user, content: hello}]\noutput: {format: text, validation_mode: basic}\n")},
"fragments/body.txt": {Data: []byte("message")},
}}
namespace := configkit.NewFilesystem("mounts", files)
root := promptkit.SettingsInput(namespace.File("fragments/root.yaml"))
collection, err := promptkit.CollectionInput("docs", namespace.File("fragments/collection.yaml"))
if err != nil {
t.Fatal(err)
}
item := configkit.Input{Source: namespace.File("fragments/item.yaml"), Destination: configkit.Keys("promptkit", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem}
for _, input := range []*configkit.Input{&root, &collection, &item} {
input.Filesystem = &configkit.FilesystemContext{FS: &namespace, Root: "."}
}
bundled := promptkit.BundledOpenRouterUnderlay()
if files.opens != 0 || root.Imports || !collection.Imports || root.Destination.String() != "/promptkit" || collection.Destination.String() != "/promptkit/collections/docs" {
t.Fatalf("mount construction read files or chose wrong defaults: opens=%d", files.opens)
}
definition := promptkit.Configure(configkit.Definition[mountApplication]{}, func(value *mountApplication) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[mountApplication]{Definition: definition, Underlays: []configkit.Underlay{bundled, {Inputs: []configkit.Input{root, collection, item}}}})
if err != nil {
t.Fatal(err)
}
if len(result.Config.PromptKit.Profiles) < 1 || len(result.Config.PromptKit.Prompts) != 1 || len(result.Config.PromptKit.Collections["docs"].Prompts) != 1 {
t.Fatalf("mounted settings = %+v", result.Config.PromptKit)
}
info, ok := result.Metadata().Lookup(configkit.Keys("promptkit", "collections", "docs", "prompts").Index(0).Key("messages").Index(0).Key("content_file"))
if !ok || len(info.Origin.Locations) != 1 {
t.Fatalf("origin = %+v", info)
}
context, ok := result.SourceContext(info.Origin.Locations[0].SourceID)
if !ok || context.FS == nil || context.BaseDir != "fragments" {
t.Fatalf("context = %+v", context)
}
}

View File

@@ -0,0 +1,99 @@
package promptkit_test
import (
"errors"
"strings"
"sync"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type rulesApplication struct {
Version string `yaml:"version"`
Other []rulesOther `yaml:"other"`
PromptKit *promptkit.Settings `yaml:"promptkit"`
}
type rulesOther struct {
ID string `yaml:"id"`
}
func TestConfigureComposesApplicationRules(t *testing.T) {
var events []string
definition := configkit.Definition[rulesApplication]{
Defaults: func() rulesApplication { return rulesApplication{Version: "app"} },
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(v configkit.Version, validation *configkit.Validation) error {
events = append(events, "version")
return nil
}},
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("other"), "id")},
Resolve: func(_ *rulesApplication, _ *configkit.Resolution) error {
events = append(events, "application resolve")
return nil
},
Validate: func(_ *rulesApplication, _ *configkit.Validation) { events = append(events, "application validate") },
Sensitive: func(path configkit.Path) bool { return path.String() == "/application_secret" },
}
configured := promptkit.Configure(definition, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
if len(definition.SequenceIdentities) != 1 || len(configured.SequenceIdentities) != 7 {
t.Fatalf("registrations: %d, %d", len(definition.SequenceIdentities), len(configured.SequenceIdentities))
}
priorValidate := configured.Validate
configured.Validate = func(value *rulesApplication, validation *configkit.Validation) {
priorValidate(value, validation)
if value.PromptKit == nil || value.PromptKit.Backends[0].Endpoint == nil || strings.TrimSpace(*value.PromptKit.Backends[0].Endpoint) != *value.PromptKit.Backends[0].Endpoint {
validation.Error(configkit.Keys("promptkit"), "normalization did not run")
}
events = append(events, "after validate")
}
source := configkit.Bytes("rules", []byte("promptkit:\n backends:\n - id: local\n endpoint: ' https://example.test/v1/ '\n extra_params: {api_key: hidden}\nother: [{id: one}]\n"))
result, err := configkit.Load(source, configkit.Options[rulesApplication]{Definition: configured})
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
if result.Config.Version != "app" || strings.Join(events, ",") != "version,application resolve,application validate,after validate" {
t.Fatalf("callbacks: %+v, %+v", result.Config, events)
}
if !configured.Sensitive(configkit.Keys("application_secret")) || !configured.Sensitive(configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("api_key")) || configured.Sensitive(configkit.Keys("backends").Index(0).Key("extra_params").Key("api_key")) {
t.Fatal("sensitivity composition failed")
}
var rendered strings.Builder
if err := result.WriteEffective(&rendered); err != nil || strings.Contains(rendered.String(), "hidden") {
t.Fatalf("redaction: %q, %v", rendered.String(), err)
}
}
func TestConfigurePropagatesResolveError(t *testing.T) {
want := errors.New("resolve failed")
definition := promptkit.Configure(configkit.Definition[rulesApplication]{Resolve: func(*rulesApplication, *configkit.Resolution) error { return want }}, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
err := definition.Resolve(&rulesApplication{}, nil)
if !errors.Is(err, want) {
t.Fatalf("resolve error = %v", err)
}
}
func TestConfigureReusableAcrossConcurrentLoads(t *testing.T) {
definition := promptkit.Configure(configkit.Definition[rulesApplication]{}, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
var group sync.WaitGroup
for range 8 {
group.Go(func() {
for _, source := range []string{"{}", "promptkit: {prompts: [{id: test, output: {format: text, validation_mode: basic}}]}"} {
result, err := configkit.Load(configkit.Bytes("settings", []byte(source)), configkit.Options[rulesApplication]{Definition: definition})
if err != nil {
t.Error(err)
continue
}
if result.Config.PromptKit != nil && *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
t.Error("default was not resolved")
}
}
})
}
group.Wait()
}

View File

@@ -8,12 +8,13 @@ import (
"path/filepath" "path/filepath"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/configuration"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/filecatalog" "go.maximumdirect.net/promptkit/internal/filecatalog"
) )
// CollectionInput mounts source at the selected collection's configuration // CollectionInput mounts source at the selected collection's configuration
// destination, collections/<namespace>, and enables ConfigKit imports for that // destination, /promptkit/collections/<namespace>, and enables ConfigKit imports for that
// input. An invalid namespace returns an error matching ErrInvalidConfig. The // input. An invalid namespace returns an error matching ErrInvalidConfig. The
// source is not read or validated until ConfigKit loads it. Callers may adjust // source is not read or validated until ConfigKit loads it. Callers may adjust
// the returned input, including its filesystem context or imports setting. // the returned input, including its filesystem context or imports setting.
@@ -23,11 +24,23 @@ func CollectionInput(namespace string, source configkit.Source) (configkit.Input
} }
return configkit.Input{ return configkit.Input{
Source: source, Source: source,
Destination: configkit.Keys("collections", namespace), Destination: configkit.Keys("promptkit", "collections", namespace),
Imports: true, Imports: true,
}, nil }, nil
} }
// SettingsInput mounts a wrapperless PromptKit settings fragment at /promptkit.
// It does not acquire the source; callers may adjust the returned input.
func SettingsInput(source configkit.Source) configkit.Input {
return configkit.Input{Source: source, Destination: configkit.Keys("promptkit")}
}
// BundledOpenRouterUnderlay selects the bundled OpenRouter settings fragment.
// It does not acquire its source until ConfigKit loads it.
func BundledOpenRouterUnderlay() configkit.Underlay {
return configuration.BundledOpenRouterUnderlay()
}
// DirectoryInputs selects the YAML files beneath an explicitly chosen host // DirectoryInputs selects the YAML files beneath an explicitly chosen host
// directory as one strict-composition group. The returned order is deterministic; // directory as one strict-composition group. The returned order is deterministic;
// it does not assign precedence among those files. Discovery is recursive over // it does not assign precedence among those files. Discovery is recursive over

View File

@@ -23,8 +23,8 @@ func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if input.Destination.String() != "/collections/docs" || !input.Imports { if input.Destination.String() != "/promptkit/collections/docs" || !input.Imports {
t.Fatalf("collection input = %+v, want destination /collections/docs with imports enabled", input) t.Fatalf("collection input = %+v, want destination /promptkit/collections/docs with imports enabled", input)
} }
for _, namespace := range []string{"", " ", "docs/api"} { for _, namespace := range []string{"", " ", "docs/api"} {
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) { if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
@@ -51,7 +51,7 @@ func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}}, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
Main: &configkit.Input{ Main: &configkit.Input{
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}), Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
Destination: configkit.Keys("collections", "docs"), Destination: configkit.Keys("promptkit", "collections", "docs"),
}, },
}}) }})
if err != nil { if err != nil {
@@ -119,7 +119,7 @@ func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}}, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
Main: &configkit.Input{ Main: &configkit.Input{
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}), Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
Destination: configkit.Keys("collections", "docs"), Destination: configkit.Keys("promptkit", "collections", "docs"),
}, },
}}) }})
if err != nil { if err != nil {

View File

@@ -41,7 +41,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
endpoint, model := "http://example.test/v1", "model" endpoint, model := "http://example.test/v1", "model"
lowerName := name lowerName := name
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{ cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{ Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName}, {ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
}), }),
}}}) }}})
@@ -49,7 +49,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
if test.clear { if test.clear {
env = "" env = ""
} }
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{ cfg.Sources.Main = &configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env}, {ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
})} })}
var calls int var calls int
@@ -73,7 +73,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) { func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
backendID, model := "workstation", "offline-model" backendID, model := "workstation", "offline-model"
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{ prompt := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{
"prompts": []any{map[string]any{ "prompts": []any{map[string]any{
"id": "hello", "id": "hello",
"default_profile": "offline", "default_profile": "offline",
@@ -86,10 +86,10 @@ func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T)
localBackend.ID = backendID localBackend.ID = backendID
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{ {Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
localBackend, localBackend,
})}, })},
{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{ {Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
{ID: "offline", Backend: &backendID, Model: &model}, {ID: "offline", Backend: &backendID, Model: &model},
})}, })},
}}}, }}},
@@ -116,11 +116,11 @@ func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
model := "configured-model" model := "configured-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{ {Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
{ID: openRouterID, Endpoint: &openRouterEndpoint}, {ID: openRouterID, Endpoint: &openRouterEndpoint},
{ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint}, {ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint},
})}, })},
{Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{ {Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
{ID: "openrouter-profile", Backend: &openRouterID, Model: &model}, {ID: "openrouter-profile", Backend: &openRouterID, Model: &model},
{ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model}, {ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model},
})}, })},
@@ -170,7 +170,7 @@ prompts:
t.Fatalf("select operator inputs: %v", err) t.Fatalf("select operator inputs: %v", err)
} }
programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1" programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1"
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{ programmatic := configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
{ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel}, {ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel},
})} })}
for _, test := range []struct { for _, test := range []struct {
@@ -183,9 +183,9 @@ prompts:
{name: "programmatic", apply: true, want: "programmatic-model"}, {name: "programmatic", apply: true, want: "programmatic-model"},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
underlays := []configkit.Underlay{{Inputs: embedded}} underlays := []configkit.Underlay{{Inputs: mountSettingsInputs(embedded)}}
if test.apply { if test.apply {
underlays = append(underlays, configkit.Underlay{Inputs: operator}) underlays = append(underlays, configkit.Underlay{Inputs: mountSettingsInputs(operator)})
} }
if test.name == "programmatic" { if test.name == "programmatic" {
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}}) underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
@@ -207,7 +207,7 @@ prompts:
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) { func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("invalid", map[string]any{"profiles": []any{map[string]any{"id": "broken", "unknown": true}}}), Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid", map[string]any{"prompts": []any{map[string]any{"id": "broken", "output": map[string]any{"unknown": true}}}}),
}}}) }}})
var loadErr *configkit.LoadError var loadErr *configkit.LoadError
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) { if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
@@ -215,6 +215,36 @@ func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
} }
} }
func TestPublicConstructionPreservesMalformedYAMLCause(t *testing.T) {
const sourceLabel = "prompts.yml"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Bytes(sourceLabel, []byte("profiles: [\n")),
}}})
var loadErr *configkit.LoadError
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err)
}
cause := errors.Unwrap(err)
var causeLoadErr *configkit.LoadError
if cause == nil || errors.Is(cause, promptkit.ErrInvalidConfig) || !errors.As(cause, &causeLoadErr) || causeLoadErr != loadErr {
t.Fatalf("construction cause = %v, want the original loading failure without a category branch", cause)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Code != "yaml.invalid" {
continue
}
if diagnostic.Primary == nil {
t.Fatal("YAML diagnostic has no source location")
}
source, ok := loadErr.Source(diagnostic.Primary.SourceID)
if !ok || source.Label != sourceLabel {
t.Fatalf("YAML diagnostic source = %+v, found=%v, want %q", source, ok, sourceLabel)
}
return
}
t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics())
}
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) { func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
main := configkit.Input{} main := configkit.Input{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}}) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
@@ -227,10 +257,10 @@ func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model" endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{ {Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
{ID: backendID, Endpoint: &endpoint}, {ID: backendID, Endpoint: &endpoint},
})}, })},
{Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{ {Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
{ID: "application-profile", Backend: &backendID, Model: &model}, {ID: "application-profile", Backend: &backendID, Model: &model},
})}, })},
}}}, }}},
@@ -274,7 +304,7 @@ func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
authenticationEndpoint = "https://mutated.example/key" authenticationEndpoint = "https://mutated.example/key"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}}, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}},
}}) }})
if err != nil { if err != nil {
t.Fatalf("construct engine with dangling diagnostic default: %v", err) t.Fatalf("construct engine with dangling diagnostic default: %v", err)

View File

@@ -11,6 +11,14 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func mountSettingsInputs(inputs []configkit.Input) []configkit.Input {
result := append([]configkit.Input(nil), inputs...)
for i := range result {
result[i].Destination = configkit.Keys("promptkit")
}
return result
}
type fakeLLMClient struct { type fakeLLMClient struct {
response *promptkit.GenerateResponse response *promptkit.GenerateResponse
responses []*promptkit.GenerateResponse responses []*promptkit.GenerateResponse
@@ -59,7 +67,7 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
func simpleConfig(content string) promptkit.Config { func simpleConfig(content string) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{ return promptkit.Config{Sources: promptkit.ConfigurationSources{
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{ Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("consumer test", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "profiles": []any{map[string]any{"id": "profile",
"endpoint": "http://example.test/v1", "model": "model", "endpoint": "http://example.test/v1", "model": "model",
}}, }},

View File

@@ -0,0 +1,97 @@
package promptkit_test
import (
"context"
"errors"
"net/http"
"net/url"
"strings"
"testing"
"testing/synctest"
"time"
"go.maximumdirect.net/promptkit"
)
type contextIdentityResponseBody struct {
ctx context.Context
cause error
}
func (b contextIdentityResponseBody) Read([]byte) (int, error) { <-b.ctx.Done(); return 0, b.cause }
func (contextIdentityResponseBody) Close() error { return nil }
func TestExecutionContextAndOriginalCauseIdentities(t *testing.T) {
for _, split := range []bool{false, true} {
for _, kind := range []string{"caller cancel", "caller deadline", "body operation deadline"} {
t.Run(map[bool]string{false: "run/", true: "prepared/"}[split]+kind, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
cause := errors.New("private original cause")
ctx, cancel := context.WithCancelCause(context.Background())
defer cancel(nil)
want := error(context.Canceled)
failure := promptkit.ExecutionFailureCanceled
if kind == "caller deadline" {
var stop context.CancelFunc
ctx, stop = context.WithTimeoutCause(context.Background(), time.Second, cause)
defer stop()
}
if kind != "caller cancel" {
want = context.DeadlineExceeded
failure = promptkit.ExecutionFailureDeadline
}
cfg := simpleConfig("message")
cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
if kind == "body operation deadline" {
return &http.Response{StatusCode: http.StatusOK, Body: contextIdentityResponseBody{ctx: r.Context(), cause: cause}}, nil
}
if kind == "caller cancel" {
cancel(cause)
} else {
<-ctx.Done()
}
return nil, context.Cause(ctx)
})}
engine, err := promptkit.NewEngine(cfg)
if err != nil {
t.Fatal(err)
}
var events []promptkit.ExecutionEvent
req := simpleGenerationRequest()
req.OperationTimeout = time.Minute
if kind == "body operation deadline" {
req.OperationTimeout = time.Second
}
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
var result *promptkit.RunResult
if split {
handle, prepareErr := engine.PrepareExecution(ctx, req)
if prepareErr != nil {
t.Fatal(prepareErr)
}
result, err = engine.RunPrepared(ctx, handle)
} else {
result, err = engine.Run(ctx, req)
}
for _, identity := range []error{want, cause, promptkit.ErrLLMGenerate, promptkit.ErrTransport} {
if !errors.Is(err, identity) {
t.Errorf("error=%v, want identity %v", err, identity)
}
}
if kind != "body operation deadline" {
var transportErr *url.Error
if !errors.As(err, &transportErr) {
t.Errorf("lost HTTP transport error: %v", err)
}
}
if result != nil || errors.Is(err, promptkit.ErrOperationTimeout) != (kind == "body operation deadline") || strings.Contains(err.Error(), cause.Error()) {
t.Fatalf("incorrect outcome or disclosure: result=%v error=%v", result, err)
}
if len(events) != 3 || events[1].Failure != failure || events[2].Kind != promptkit.ExecutionFinished || events[2].Failure != failure {
t.Fatalf("attempt/terminal classification = %+v", events)
}
})
})
}
}
}

View File

@@ -27,7 +27,7 @@ func TestContinuationBranchesAcrossEngines(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: initial.Exchange, SessionID: "new-session", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON}, AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "branch one"}}} request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: initial.Exchange, SessionID: "new-session", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON}, AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "branch one"}}}
branch, err := scoped.Run(ctx, request) branch, err := scoped.Run(ctx, request)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -100,6 +100,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
InputHashes: copyStringMap(prepared.InputHashes), InputHashes: copyStringMap(prepared.InputHashes),
SessionID: prepared.SessionID, SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash, RenderedPromptHash: prepared.RenderedPromptHash,
MessageContentHash: prepared.MessageContentHash,
Messages: fromDomainRenderedMessages(prepared.Messages), Messages: fromDomainRenderedMessages(prepared.Messages),
StartTime: prepared.StartTime, StartTime: prepared.StartTime,
EndTime: prepared.EndTime, EndTime: prepared.EndTime,
@@ -133,6 +134,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
InputHashes: copyStringMap(result.InputHashes), InputHashes: copyStringMap(result.InputHashes),
Usage: fromDomainTokenUsage(result.Usage), Usage: fromDomainTokenUsage(result.Usage),
CostUSD: fromDomainCost(result.CostUSD),
StartTime: result.StartTime, StartTime: result.StartTime,
EndTime: result.EndTime, EndTime: result.EndTime,
Duration: result.Duration, Duration: result.Duration,
@@ -157,6 +159,7 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
Content: resp.Content, Content: resp.Content,
Usage: toDomainTokenUsage(resp.Usage), Usage: toDomainTokenUsage(resp.Usage),
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence), UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
CostUSD: copyFloat64Ptr(resp.CostUSD),
} }
} }
@@ -167,6 +170,7 @@ func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsage
return &domain.TokenUsagePresence{ return &domain.TokenUsagePresence{
PromptTokens: presence.PromptTokens, PromptTokens: presence.PromptTokens,
CompletionTokens: presence.CompletionTokens, CompletionTokens: presence.CompletionTokens,
ReasoningTokens: presence.ReasoningTokens,
TotalTokens: presence.TotalTokens, TotalTokens: presence.TotalTokens,
CachedTokens: presence.CachedTokens, CachedTokens: presence.CachedTokens,
CacheWriteTokens: presence.CacheWriteTokens, CacheWriteTokens: presence.CacheWriteTokens,
@@ -264,7 +268,7 @@ func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
} }
func fromBackendResource(v *configuration.BackendResource) *BackendInspection { func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)} out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, Provider: copyStringPtr(v.Provider), ZeroCost: copyBoolPtr(v.ZeroCost), APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
if v.ExtraParams != nil { if v.ExtraParams != nil {
data := copyAnyMap(v.ExtraParams) data := copyAnyMap(v.ExtraParams)
out.ExtraParams = &data out.ExtraParams = &data
@@ -282,7 +286,7 @@ func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
} }
func fromPromptResource(v *configuration.PromptResource) *PromptInspection { func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: copyIntPtr(v.Output.RepairAttempts)}} out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), Schema: jsonvalue.CloneTrusted(v.Output.Schema), SchemaFile: copyStringPtr(v.Output.SchemaFile), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: v.Output.RepairAttempts}}
if v.Inputs != nil { if v.Inputs != nil {
inputs := make([]PromptInputInspection, len(*v.Inputs)) inputs := make([]PromptInputInspection, len(*v.Inputs))
for i, input := range *v.Inputs { for i, input := range *v.Inputs {
@@ -317,20 +321,16 @@ func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence)
} }
} }
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract { func toDomainOutputContractPtr(contract *OutputContractInput) *domain.OutputContractInput {
if contract == nil { if contract == nil {
return nil return nil
} }
out := toDomainOutputContract(*contract) return &domain.OutputContractInput{
return &out
}
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
return domain.OutputContract{
Format: domain.OutputFormat(contract.Format), Format: domain.OutputFormat(contract.Format),
ValidationMode: domain.ValidationMode(contract.ValidationMode), ValidationMode: domain.ValidationMode(contract.ValidationMode),
Schema: contract.Schema,
SchemaPath: contract.SchemaPath, SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts, RepairAttempts: copyIntPtr(contract.RepairAttempts),
} }
} }
@@ -338,6 +338,7 @@ func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
return OutputContract{ return OutputContract{
Format: OutputFormat(contract.Format), Format: OutputFormat(contract.Format),
ValidationMode: ValidationMode(contract.ValidationMode), ValidationMode: ValidationMode(contract.ValidationMode),
Schema: jsonvalue.CloneTrusted(contract.Schema),
SchemaPath: contract.SchemaPath, SchemaPath: contract.SchemaPath,
RepairAttempts: contract.RepairAttempts, RepairAttempts: contract.RepairAttempts,
} }
@@ -358,6 +359,7 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
return TokenUsage{ return TokenUsage{
PromptTokens: usage.PromptTokens, PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens, CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens, TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens, CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens, CacheWriteTokens: usage.CacheWriteTokens,
@@ -368,6 +370,7 @@ func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
return domain.TokenUsage{ return domain.TokenUsage{
PromptTokens: usage.PromptTokens, PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens, CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens, TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens, CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens, CacheWriteTokens: usage.CacheWriteTokens,

View File

@@ -3,6 +3,7 @@ package promptkit_test
import ( import (
"context" "context"
"errors" "errors"
"reflect"
"testing" "testing"
"testing/synctest" "testing/synctest"
"time" "time"
@@ -10,6 +11,31 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestCoordinateObserverOrderWithContext(t *testing.T) {
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
if err != nil {
t.Fatal(err)
}
var order []string
observer := func(name string) promptkit.ExecutionObserver {
return func(promptkit.ExecutionEvent) { order = append(order, name) }
}
ctx := promptkit.WithExecutionObserver(context.Background(), observer("context"))
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: observer("coordinator"), Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
if turn.RemainingTurns == 0 {
return nil, nil
}
return &promptkit.RunRequest{PromptID: "prompt", Observer: observer("request")}, nil
}})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"request", "coordinator", "context", "request", "coordinator", "context", "request", "coordinator", "context"}) {
t.Fatalf("observer order = %v", order)
}
}
func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) { func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) {
synctest.Test(t, func(t *testing.T) { synctest.Test(t, func(t *testing.T) {
engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {

View File

@@ -23,7 +23,7 @@ func TestCoordinateReleasesCapacityBeforeCallback(t *testing.T) {
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
calls++ calls++
if turn.Result == nil { if turn.Result == nil {
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1}}, nil return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}}, nil
} }
if turn.Result.Validation.Status != promptkit.ValidationFailed || turn.Result.Validation.RepairAttempts != 1 || turn.RemainingTurns != 0 { if turn.Result.Validation.Status != promptkit.ValidationFailed || turn.Result.Validation.RepairAttempts != 1 || turn.RemainingTurns != 0 {
t.Fatalf("delivered turn = %#v", turn) t.Fatalf("delivered turn = %#v", turn)
@@ -98,7 +98,7 @@ func TestCoordinateRetriesAndRepairsDeliverOnce(t *testing.T) {
result, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { result, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
callbacks++ callbacks++
if turn.Result == nil { if turn.Result == nil {
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}}, nil return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}}, nil
} }
if !turn.Result.Validation.IsValid || turn.RemainingTurns != 0 { if !turn.Result.Validation.IsValid || turn.RemainingTurns != 0 {
t.Fatalf("turn = %#v", turn) t.Fatalf("turn = %#v", turn)

42
cost.go Normal file
View File

@@ -0,0 +1,42 @@
package promptkit
import "go.maximumdirect.net/promptkit/internal/domain"
// CostSource identifies how an observed USD cost was supplied.
type CostSource string
const (
// CostSourceNone denotes no known sourced contribution.
CostSourceNone CostSource = ""
// CostSourceProvider denotes a reported provider or injected-client amount.
CostSourceProvider CostSource = "provider"
// CostSourceConfiguredZero denotes an explicitly configured zero fallback.
CostSourceConfiguredZero CostSource = "configured_zero"
// CostSourceMixed denotes both provider and configured-zero contributions.
CostSourceMixed CostSource = "mixed"
)
// ObservedCost reports an available USD subtotal and its coverage and source.
// Known means at least one contribution is available. Complete means all
// included dispatched attempts have an amount under the selected policy; it
// does not verify billing. A completed run with no dispatched attempts, or an
// empty group, has known complete zero with no source. A group with members but
// no terminals has unknown cost. Amounts are unrounded binary floating point.
type ObservedCost struct {
// Value is the known USD subtotal.
Value float64 `json:"value"`
// Known reports whether a subtotal is available.
Known bool `json:"known"`
// Complete reports whether all included dispatched attempts are covered.
Complete bool `json:"complete"`
// Source summarizes the known contributions.
Source CostSource `json:"source"`
}
func fromDomainCost(value domain.ObservedCost) ObservedCost {
return ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: CostSource(value.Source)}
}
func toDomainCost(value ObservedCost) domain.ObservedCost {
return domain.ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: domain.CostSource(value.Source)}
}

View File

@@ -0,0 +1,131 @@
package promptkit_test
import (
"context"
"errors"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestPreparedCostAndRejectedClaims(t *testing.T) {
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "bad"}, nil
}
return nil, errors.New("repair failed")
})))
ctx := context.Background()
var events []promptkit.ExecutionEvent
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
if err != nil || len(events) != 0 {
t.Fatalf("prepare error=%v events=%+v", err, events)
}
result, err := engine.RunPrepared(ctx, handle)
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
}
events = nil
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
t.Fatalf("reused err=%v events=%+v", err, events)
}
}
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
childCalls := 0
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
childCalls++
charge := .2
if childCalls == 2 {
charge = .3
}
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
var childRun, childPrepared *promptkit.RunResult
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
var err error
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
return nil, err
}
childPrepared, err = child.RunPrepared(ctx, handle)
if err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
}
summary := group.Snapshot()
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 2.0
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
})))
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
summary := group.Snapshot()
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
}
}
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := .5
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var childResult *promptkit.RunResult
var childErr error
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
}})
if err != nil {
t.Fatal(err)
}
var group promptkit.ExecutionGroup
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
}
summary := group.Snapshot()
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}

127
cost_run_contract_test.go Normal file
View File

@@ -0,0 +1,127 @@
package promptkit_test
import (
"context"
"errors"
"io"
"math"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type costRoundTrip func(*http.Request) (*http.Response, error)
func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
options = append(options, promptkit.WithBundledOpenRouter(false))
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...)
if err != nil {
t.Fatal(err)
}
return engine
}
func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
for _, observed := range []bool{false, true} {
t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) {
calls := 0
transport := costRoundTrip(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil
})
engine := costEngine(t, false, &http.Client{Transport: transport})
var events []promptkit.ExecutionEvent
req := promptkit.RunRequest{PromptID: "prompt"}
if observed {
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
}
result, err := engine.Run(context.Background(), req)
want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider}
if err != nil || result == nil || result.CostUSD != want || calls != 2 {
t.Fatalf("result=%+v calls=%d err=%v", result, calls, err)
}
if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) {
t.Fatalf("events=%+v", events)
}
})
}
}
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
charge := .125
invalid := math.Inf(1)
for _, tc := range []struct {
name string
response *promptkit.GenerateResponse
err error
want promptkit.ObservedCost
}{
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
{"nonfinite report falls back", &promptkit.GenerateResponse{Content: "ok", CostUSD: &invalid}, nil, promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
} {
t.Run(tc.name, func(t *testing.T) {
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return tc.response, tc.err
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
})
}
}
func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) {
charge := .1
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "invalid"}, nil
}
return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed}
if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want {
t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err)
}
}
func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) {
engine := costEngine(t, true, nil)
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
}

8
doc.go
View File

@@ -1,8 +1,8 @@
// Package promptkit provides an embeddable engine for preparing and executing // Package promptkit provides an embeddable engine for preparing and executing
// prompt-defined LLM workflows. // prompt-defined LLM workflows.
// //
// Applications construct an [Engine] with [NewEngine], select ConfigKit // Applications construct an [Engine] with [NewEngine] or register [Configure]
// sources for prompts, profiles, and backends, and // during their own ConfigKit load and pass its result to [NewFromConfig], then
// call [Engine.InspectConfiguration], [Engine.InspectPrompt], // call [Engine.InspectConfiguration], [Engine.InspectPrompt],
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.InspectBackend], // [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.InspectBackend],
// [Engine.ListProfiles], [Engine.ListPrompts], [Engine.Prepare], // [Engine.ListProfiles], [Engine.ListPrompts], [Engine.Prepare],
@@ -23,8 +23,8 @@
// independently. An injected [LLMClient] or [ArtifactReader] can therefore // independently. An injected [LLMClient] or [ArtifactReader] can therefore
// still receive concurrent calls and must be safe for that use. // still receive concurrent calls and must be safe for that use.
// //
// ConfigKit captures the selected configuration when NewEngine constructs an // An Engine captures selected configuration resources during construction.
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer // Prepare, PrepareExecution, and Run copy request maps, slices, pointer
// values, and JSON-compatible extra parameters before using them. InspectPrompt, // values, and JSON-compatible extra parameters before using them. InspectPrompt,
// InspectRequirements, InspectProfile, and InspectBackend return copied // InspectRequirements, InspectProfile, and InspectBackend return copied
// inspection values. // inspection values.

View File

@@ -0,0 +1,63 @@
# 0002: Share Configuration Data Types Across The Public Boundary
## Status
Accepted.
## Date
2026-11-21
## Context
PromptKit is a Go library that applications use to configure and run prompts.
Applications use its public package; the code that loads settings and runs
prompts lives in internal packages. This separation normally lets the internal
code change without changing the application's interface.
The planned configuration API lets applications load settings once and pass them to
PromptKit. Keeping separate public and internal definitions of those same
settings would require duplicate declarations and code to copy each field back
and forth. That adds maintenance work and opportunities to miss a field without
changing what the settings mean.
## Decision
Define the public configuration settings and their nested data types once in a
small, data-only internal package. Expose them through type aliases in the public
`promptkit` package. An alias gives the same Go type a public name; consumers
continue to use names such as `promptkit.Settings` without importing an internal
package. The internal configuration code uses the same definitions directly.
This is a narrow exception to the rule against exposing internal types. These
settings are intentional public contracts, including their fields and
configuration tags. Their location does not make them private implementation
details that can be changed independently of the public API.
The rest of the public boundary stays in place. Loading, validation, and
execution logic remain internal. Source-building helpers, execution requests,
runtime objects, and inspection reports keep separate representations where
they serve different purposes. Internal packages still do not import the public
package. Sharing declarations does not remove the copies needed to keep a
running engine independent of later application changes.
## Alternatives Considered
- Keep separate public and internal settings. This allows independent changes,
but provides little benefit when both describe the same configuration and
must stay synchronized.
- Put the shared types in a new public settings package. This also removes
duplication, but adds a public package when the existing `promptkit` package
already provides a suitable consumer interface.
## Consequences
Configuration fields have one declaration, and registration and resource
capture no longer need conversions between equivalent settings types. The
primary benefit is simpler maintenance; performance improvements are secondary.
Conversion into execution-ready objects and necessary ownership copies remain.
This decision leaves the [ConfigKit ownership boundary](0001-configuration-boundary.md)
unchanged. The [shared-loading guide](../consumers/pkg-promptkit.md#reuse-an-application-owned-load)
describes the implemented consumer workflow; the
[internal overview](../internal/overview.md) describes its implementation.

View File

@@ -1,8 +1,7 @@
# Configuration And API Migration # Configuration And API Migration
This breaking pre-v1 minor migration adopts Promptkit's vanity module path, This guide covers the current configuration and repair-input migration, as well
changes the v0.16-era keyed resources to identified lists, adopts ConfigKit v1, as earlier module-path and keyed-resource migrations.
and removes prompt-version labels.
Promptkit installs a default-on bundled OpenRouter catalog; applications choose configuration Promptkit installs a default-on bundled OpenRouter catalog; applications choose configuration
inputs explicitly, and execution selects one prompt by its exact ID. This guide inputs explicitly, and execution selects one prompt by its exact ID. This guide
explains the shape changes and decisions an application must make; it is not an explains the shape changes and decisions an application must make; it is not an
@@ -10,6 +9,35 @@ automatic migration tool. The [framework format reference](../formats.md) owns
the current configuration contract, and the root package GoDoc owns exact API the current configuration contract, and the root package GoDoc owns exact API
signatures and errors. signatures and errors.
## Move Settings Under Promptkit
Place complete-document resources under `promptkit`, including `schema_root`,
`backends`, `profiles`, `prompts`, and `collections`. Old root-level resource
keys are application data and are not loaded as Promptkit resources; selecting
an unmigrated resource can therefore fail with a not-found error. IDs and
references do not acquire a `promptkit` prefix. `SettingsInput` mounts a
wrapperless fragment at `/promptkit`, and `CollectionInput` mounts one at
`/promptkit/collections/<namespace>`. Discovery helpers still return root-mounted
inputs; set destinations explicitly for discovered fragments. Standalone
id-first files use native sequence-item placement. The [format reference](../formats.md)
describes extensions and strict nested objects.
Applications can instead embed `promptkit.Settings`, call `Configure` on their
ConfigKit definition, load once, and pass the whole result to `NewFromConfig`.
The [preparation example](../../examples/go-library/prepare/main.go) shows the
workflow. Select `BundledOpenRouterUnderlay` before an application-owned load;
the default bundle option applies only to `NewEngine`. The application owns
version policy; Promptkit no longer validates resource-version fields.
Replace `RunRequest.Validation` values of type `*OutputContract` with
`*OutputContractInput` and use a pointer for `RepairAttempts`. Nil or null
selects zero for `none` and one for `basic`, `json`, and `json_schema`;
explicit zero disables correction. Runtime replacements resolve independently
of configured budgets. Inspection reports resolved integer budgets, including
zero. A failed validation can cause an additional generation and associated
latency and usage; a correction is not guaranteed. See the
[output contract](../formats.md#output-contract).
## Update Go Imports ## Update Go Imports
Promptkit's module and public package import path is now Promptkit's module and public package import path is now
@@ -37,8 +65,8 @@ The configuration and API adaptations below are also part of this migration.
## Select And Compose The Configuration You Need ## Select And Compose The Configuration You Need
Supply each backend, profile, prompt, schema, and collection source that the Supply each backend, profile, prompt, schema, and collection source that the
application intends to use. Promptkit does not discover directories or install application intends to use. `NewEngine` selects the bundled OpenRouter underlay
provider definitions implicitly. An application may select an explicit by default; callers explicitly select other sources. An application may select an explicit
collection input with `CollectionInput`: collection input with `CollectionInput`:
```go ```go
@@ -64,11 +92,10 @@ Promptkit does not rewrite references when a source is mounted. `extends`
selects a sibling ID in the same list, not a cross-collection reference. selects a sibling ID in the same list, not a cross-collection reference.
Review collisions and all references when moving resources into collections. Review collisions and all references when moving resources into collections.
The optional source-root `version` header now accepts `2` or `"2"`; headerless A collection's `release_version` remains an optional label. Prompt-version
sources remain valid. A collection's `release_version` remains an optional selection has been removed: choose one definition per ID. Application-specific
label. Prompt-version labels have been removed: choose one definition per ID version data may be retained at supported extension points, but is not used by
and store any application-specific release label outside Promptkit. In a Promptkit for resource selection.
standalone imported prompt item, root `version` is a source header, not a label.
For example, change a version-keyed definition: For example, change a version-keyed definition:
@@ -84,7 +111,7 @@ prompts:
validation_mode: none validation_mode: none
``` ```
to one identified definition: to one identified wrapperless fragment, mounted with `SettingsInput`:
```yaml ```yaml
prompts: prompts:
@@ -98,7 +125,8 @@ prompts:
``` ```
The same conversion applies to backends and profiles. Sibling inheritance uses The same conversion applies to backends and profiles. Sibling inheritance uses
identity references, and a later same-ID record replaces the whole lower record: identity references, and a later same-ID record replaces the whole lower record.
This is another wrapperless fragment to mount with `SettingsInput`:
```yaml ```yaml
profiles: profiles:
@@ -113,11 +141,11 @@ profiles:
To customize a lower resource's fields, add a distinct inherited variant as To customize a lower resource's fields, add a distinct inherited variant as
above; when replacing the same ID, supply the complete intended record. Generic above; when replacing the same ID, supply the complete intended record. Generic
append, replace, imports, and inheritance are described by the append, replace, imports, and inheritance are described by the
[ConfigKit v1 configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md). [ConfigKit v1 configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md).
Source-root controls are now `config:`, not `configkit:`. Imports are optional Source-root controls are now `config:`, not `configkit:`. Imports are optional
unless the selected import declares `required: true`. unless the selected import declares `required: true`.
Update ConfigKit imports to `go.maximumdirect.net/configkit` at v1.0.0: its Go Update ConfigKit imports to `go.maximumdirect.net/configkit` at v1.4.0: its Go
types are incompatible with the old module path. Promptkit now requires Go types are incompatible with the old module path. Promptkit now requires Go
1.27.0 or newer. Source builders take ordered slices with explicit IDs: 1.27.0 or newer. Source builders take ordered slices with explicit IDs:
@@ -133,7 +161,7 @@ source := promptkit.ProfileSource("application", []promptkit.ProfileConfig{
Replace former maps passed to `BackendSource` in the same way, setting each Replace former maps passed to `BackendSource` in the same way, setting each
`BackendConfig.ID`. Duplicates are retained for loading to reject. Resource `BackendConfig.ID`. Duplicates are retained for loading to reject. Resource
inspection paths now address list indexes, for example inspection paths now address list indexes, for example
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection `configkit.Keys("promptkit", "profiles").Index(0).Key("model")`. Configuration inspection
and diffs expose resolved values, recorded normalization and provenance. and diffs expose resolved values, recorded normalization and provenance.
`InspectProfile` reports the selected resolved profile without backend completion. `InspectProfile` reports the selected resolved profile without backend completion.
Use `Prepare` or `PreparedExecution.Details` for completed request settings, Use `Prepare` or `PreparedExecution.Details` for completed request settings,
@@ -165,7 +193,8 @@ These are conversion examples, not aliases or bundled resources. Mounting old
defaults, or diagnostic defaults inside them. Edit those values to the exact defaults, or diagnostic defaults inside them. Edit those values to the exact
qualified IDs you chose. qualified IDs you chose.
For example, an optional caller-owned standard profile may extend a selected For example, the following wrapperless fragments can be mounted with
`SettingsInput`. An optional caller-owned standard profile may extend a selected
Gemini profile in the same `openrouter` collection: Gemini profile in the same `openrouter` collection:
```yaml ```yaml

View File

@@ -18,7 +18,7 @@ import "go.maximumdirect.net/promptkit"
For migration from the former module path, see For migration from the former module path, see
[Update Go Imports](configuration-migration.md#update-go-imports). [Update Go Imports](configuration-migration.md#update-go-imports).
Configuration APIs use `go.maximumdirect.net/configkit` v1.2.0. Its types are Configuration APIs use `go.maximumdirect.net/configkit` v1.4.0. Its types are
not interchangeable with the old ConfigKit module path. not interchangeable with the old ConfigKit module path.
The maintained [offline preparation example](../../examples/go-library/prepare/main.go) The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
@@ -56,13 +56,13 @@ The bundled OpenRouter backend and profiles are installed in a lower underlay
by default. Select a profile explicitly, for example: by default. Select a profile explicitly, for example:
```go ```go
input := configkit.Input{Source: configkit.Mapping("application", map[string]any{ input := promptkit.SettingsInput(configkit.Mapping("application", map[string]any{
"prompts": []any{map[string]any{ "prompts": []any{map[string]any{
"id": "my-prompt", "id": "my-prompt",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}}, "messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"}, "output": map[string]any{"format": "text", "validation_mode": "none"},
}}, }},
})} }))
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
if err != nil { return err } if err != nil { return err }
result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "my-prompt", ProfileID: "gpt-luna-latest"}) result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "my-prompt", ProfileID: "gpt-luna-latest"})
@@ -83,25 +83,59 @@ promptInputs, err := promptkit.DirectoryInputs("prompts")
if err != nil { return err } if err != nil { return err }
profileInputs, err := promptkit.DirectoryInputs("profiles") profileInputs, err := promptkit.DirectoryInputs("profiles")
if err != nil { return err } if err != nil { return err }
// These selected files are wrapperless Promptkit fragments.
for i := range promptInputs { promptInputs[i].Destination = configkit.Keys("promptkit") }
for i := range profileInputs { profileInputs[i].Destination = configkit.Keys("promptkit") }
filesystem := configkit.NewFilesystem("application schemas", os.DirFS("."))
schemaInput := configkit.Input{
Source: promptkit.SchemaRootSource("application schemas", "schemas"),
Destination: configkit.Keys("promptkit"),
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."},
}
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
{Inputs: promptInputs}, {Inputs: promptInputs},
{Inputs: profileInputs}, {Inputs: profileInputs},
{Inputs: []configkit.Input{schemaInput}},
}}, }},
SchemaDir: "schemas",
}) })
``` ```
For a single highest-priority document, set `Sources.Main` to a For a single highest-priority document, set `Sources.Main` to a
`*configkit.Input`; omit it for ordinary layered construction. A single file `*configkit.Input`; omit it for ordinary layered construction. A single file
can use `configkit.File` directly. `PromptSource`, `ProfileSource`, `BackendSource`, and raw can use `configkit.File` directly. `PromptSource`, `ProfileSource`, `BackendSource`, and raw
`configkit.Mapping` supply programmatic definitions. Schema and runtime `configkit.Mapping` supply programmatic definitions. Runtime collaborator
collaborator options remain available. Construction validates the final options remain available. Construction validates the final
effective configuration and captures referenced prompt content; construct a effective configuration and captures referenced prompt content; construct a
new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go) new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go)
for the exact public contract and the [framework format reference](../formats.md) for the exact public contract and the [framework format reference](../formats.md)
for definition fields. for definition fields.
### Reuse An Application-Owned Load
An application that already loads ConfigKit configuration can embed
`promptkit.Settings` at `yaml:"promptkit"`, compose its definition with
`promptkit.Configure`, and pass the successful whole result to
`promptkit.NewFromConfig`. The [preparation example](../../examples/go-library/prepare/main.go)
shows a complete single-load workflow. Existing application resolve and
validate callbacks run before Promptkit's, and its defaults, version rule,
sequence registrations, and sensitivity predicate are preserved. Callers can
wrap the returned callbacks for work that must run afterward.
`NewFromConfig` accepts a completed result or a wrapper exposing the three
methods in [`Configuration`](../../engine.go). The selected `/promptkit`
value can be `Settings`, `*Settings`, absent, or nil; an incompatible value,
nil result, or unavailable inspection view returns `ErrInvalidConfig`. It
does not load sources, rerun callbacks, or add a bundle. Select
`BundledOpenRouterUnderlay()` before the application load if desired;
`WithBundledOpenRouter` affects only `NewEngine`. `RuntimeConfig` supplies only
client timeout, HTTP client, and transient-retry settings. Keep resolved settings
consistent with the load's frozen inspection evidence at handoff; subsequent mutation does not change captured
resources. Applications may register equivalent callbacks explicitly instead
of using `Configure`. Construction errors retain their underlying causes;
on the standalone path, a ConfigKit `LoadError` remains available through
`errors.As`.
### Load Companion Prompts Relative To The Main Configuration ### Load Companion Prompts Relative To The Main Configuration
A non-file underlay can use the selected main configuration's directory for A non-file underlay can use the selected main configuration's directory for
@@ -131,7 +165,7 @@ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.Configura
`Imports: true`. The binding itself does not enable imports, and an explicit `Imports: true`. The binding itself does not enable imports, and an explicit
`Input.Filesystem` takes precedence. Imported prompt `content_file` references `Input.Filesystem` takes precedence. Imported prompt `content_file` references
keep their source-local context. See ConfigKit's keep their source-local context. See ConfigKit's
[layered-loading contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#layered-loading) [layered-loading contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md#layered-loading)
for eligible inputs, fallback contexts, and behavior when the main has no usable for eligible inputs, fallback contexts, and behavior when the main has no usable
filesystem context. filesystem context.
@@ -174,6 +208,8 @@ defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS,
if err != nil { return err } if err != nil { return err }
operator, err := promptkit.DirectoryInputs(operatorDirectory) operator, err := promptkit.DirectoryInputs(operatorDirectory)
if err != nil { return err } if err != nil { return err }
for i := range defaults { defaults[i].Destination = configkit.Keys("promptkit") }
for i := range operator { operator[i].Destination = configkit.Keys("promptkit") }
profile := promptkit.ProfileSource("application choices", []promptkit.ProfileConfig{ profile := promptkit.ProfileSource("application choices", []promptkit.ProfileConfig{
{ID: "summary", Endpoint: &selectedEndpoint, Model: &selectedModel}, {ID: "summary", Endpoint: &selectedEndpoint, Model: &selectedModel},
}) })
@@ -181,7 +217,7 @@ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.Configura
Underlays: []configkit.Underlay{ Underlays: []configkit.Underlay{
{Inputs: defaults}, {Inputs: defaults},
{Inputs: operator}, {Inputs: operator},
{Inputs: []configkit.Input{{Source: profile}}}, {Inputs: []configkit.Input{promptkit.SettingsInput(profile)}},
}, },
}}) }})
``` ```
@@ -206,7 +242,7 @@ document := configkit.Mapping("application configuration", map[string]any{
}, },
}, },
}) })
input := configkit.Input{Source: document} input := promptkit.SettingsInput(document)
// Add input to a later underlay, or set Sources.Main = &input. // Add input to a later underlay, or set Sources.Main = &input.
``` ```
@@ -289,7 +325,7 @@ at index `i` in collection `team`, construct the path with `configkit.Keys`:
```go ```go
view, err := engine.InspectConfiguration() view, err := engine.InspectConfiguration()
if err != nil { return err } if err != nil { return err }
path := configkit.Keys("collections", "team", "profiles").Index(i).Key("model") path := configkit.Keys("promptkit", "collections", "team", "profiles").Index(i).Key("model")
explanation, found := view.Explain(path) explanation, found := view.Explain(path)
if found && explanation.Effective.Redacted { if found && explanation.Effective.Redacted {
// Apply the application's disclosure policy before sharing the report. // Apply the application's disclosure policy before sharing the report.
@@ -302,7 +338,11 @@ shaped provider keys are redacted, but arbitrary prompt and provider content
is not necessarily safe to share. Inspection uses the engine's captured result is not necessarily safe to share. Inspection uses the engine's captured result
and does not reopen sources or resolve credentials. It retains effective values, and does not reopen sources or resolve credentials. It retains effective values,
including recorded field normalization and queue defaulting, alongside source including recorded field normalization and queue defaulting, alongside source
provenance. provenance. The view focuses effective output and `HistoricalPaths` on
`/promptkit`; it is not an isolation boundary. Metadata and source catalogs
remain whole-load evidence, and explicit `Explain` queries may address paths
outside the focus. The empty path explains the whole application. Diagnostics
come from the original load and are returned as detached records.
`InspectPrompt`, `InspectProfile`, and `InspectBackend` instead show individual `InspectPrompt`, `InspectProfile`, and `InspectBackend` instead show individual
resolved resources without backend-to-profile execution completion or provenance. resolved resources without backend-to-profile execution completion or provenance.
`Prepare` or `PreparedExecution.Details` shows completed request-specific settings, `Prepare` or `PreparedExecution.Details` shows completed request-specific settings,
@@ -335,9 +375,11 @@ for _, difference := range differences {
``` ```
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go) See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
and ConfigKit's [effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#effective-configuration-diffs) and ConfigKit's [effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md#effective-configuration-diffs)
for the exact comparison contract. The operation compares only frozen effective for the exact comparison contract. The operation compares only frozen effective
configuration values: it does not reread external files or compare their configuration values, including native ancestor differences; sibling-only
records are excluded, but an ancestor record may contain sibling changes. It
does not reread external files or compare their
contents, resolved credential contents, later execution preparation, or contents, resolved credential contents, later execution preparation, or
request-specific overrides. Inputs normalizing to the same effective value do request-specific overrides. Inputs normalizing to the same effective value do
not create a value diff; lists are compared atomically. An empty diff is not not create a value diff; lists are compared atomically. An empty diff is not
@@ -559,12 +601,17 @@ if err != nil {
defer preparedExecution.Discard() defer preparedExecution.Discard()
details := preparedExecution.Details() details := preparedExecution.Details()
messageContentHash := details.MessageContentHash
_ = messageContentHash // An opaque value available before model execution.
// Inspect or persist an application-selected safe subset of details. // Inspect or persist an application-selected safe subset of details.
result, err := engine.RunPrepared(ctx, preparedExecution) result, err := engine.RunPrepared(ctx, preparedExecution)
``` ```
Preparation does not call the model or reserve backend capacity. Preparation does not call the model or reserve backend capacity.
`MessageContentHash` compares the ordered message roles and text, whereas
`RenderedPromptHash` also includes the session and cache-control hints. See
the [`PreparedRun` GoDoc](../../types.go) for their contracts and compatibility.
`RunPrepared` executes from the retained snapshot rather than reloading `RunPrepared` executes from the retained snapshot rather than reloading
consumer sources. The handle is opaque in-process state, while `Details` consumer sources. The handle is opaque in-process state, while `Details`
contains rendered content and remains subject to the application's data contains rendered content and remains subject to the application's data
@@ -681,6 +728,40 @@ counts, but its zero counts remain unknown to observations; supply explicit
presence when measured zeros matter. Existing `RunResult.Usage` sums returned presence when measured zeros matter. Existing `RunResult.Usage` sums returned
values independently of observation presence. values independently of observation presence.
For an application-owned invocation spanning several calls or engines, attach
an observer once to a context and propagate it to those calls:
```go
var group promptkit.ExecutionGroup
ctx = promptkit.WithExecutionObserver(ctx, group.Observe)
result, err := engine.Run(ctx, request)
// After joining all application-owned work, inspect group.Snapshot(), even on error.
```
Explicit request observers run first, followed by contextual observers in
registration order. Coordinated continuations pick up the context through
`Run`; prepared handles retain the request observer but use the execution
context's observers, not the preparation context's. Each nested call has its
own run ID and accounting. Duplicate registration can deliver duplicate events;
groups count each run's first terminal once.
Reasoning tokens are a subset of completion tokens; cache reads and writes are
input details, not extra tokens. `ObservedTokenUsage.CacheHitRatio()` derives
the cache-read fraction only when input and read totals are known, complete,
positive-denominator, and consistent. It divides aggregate counts rather than
averaging per-run ratios and is independent of cache writes.
`RunResult.CostUSD` is available even without an observer; attempt and terminal
events and group snapshots also report cost. `Known` denotes an available
subtotal, `Complete` covers dispatched attempts, and `Source` distinguishes
provider reports, an explicitly configured zero fallback, and mixed sources.
Reported cost, including zero, takes precedence over fallback. Unknown failed
attempts make a subtotal incomplete; configured zero may cover them. Terminal
events can report cost when a failed operation returns no result. Neither
complete reporting nor provider attribution verifies an invoice; consumers
choose presentation precision. See [backend definitions](../formats.md#backend-definitions)
and the [integration contract](../integrations/openai-compatible-chat.md#response-handling).
Callbacks run synchronously, in order for one operation, and can overlap for Callbacks run synchronously, in order for one operation, and can overlap for
different operations. Synchronize shared callback state and avoid blocking on different operations. Synchronize shared callback state and avoid blocking on
work that needs the same run's admission slot. Callback panics propagate; work that needs the same run's admission slot. Callback panics propagate;
@@ -697,22 +778,26 @@ Set a small additional-call budget when a structurally invalid result can be
corrected automatically: corrected automatically:
```go ```go
request.Validation = &promptkit.OutputContract{ request.Validation = &promptkit.OutputContractInput{
Format: promptkit.FormatJSON, Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSONSchema, ValidationMode: promptkit.ValidationJSONSchema,
SchemaPath: "events.schema.json", Schema: map[string]any{"type": "object"},
RepairAttempts: 1, RepairAttempts: new(1),
} }
``` ```
Each repair attempt is another model generation, so it can increase latency and Omission or null selects one repair for active validation and zero for `none`;
an explicit zero selects single-pass validation. A replacement resolves its
own budget and does not inherit the configured one. Each repair attempt is
another model generation, so it can increase latency and
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
repair generations rather than transient HTTP attempts. Exhaustion still repair generations rather than transient HTTP attempts. Exhaustion still
returns the final failed validation result. `basic` validation can also repair returns the final failed validation result. `basic` validation can also repair
an empty candidate, but structural an empty candidate, but structural
validity is not evidence of factual or domain correctness. See the validity is not evidence of factual or domain correctness and a correction is
not guaranteed. See the
[output-contract format reference](../formats.md#output-contract) and [output-contract format reference](../formats.md#output-contract) and
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility [`OutputContractInput` GoDoc](../../types.go) for the exact budget and eligibility
rules. rules.
The built-in client also retries eligible temporary HTTP statuses. By default, The built-in client also retries eligible temporary HTTP statuses. By default,
@@ -828,15 +913,15 @@ outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
}) })
``` ```
`assess` is application code and explicitly selects its observer on any nested `assess` is application code and can select its observer explicitly or inherit
run. A completed turn releases its admission lease before the callback, so that one from the context. A completed turn releases its admission lease before the callback, so that
assessment may use the same backend. `stopped` and `exhausted` describe control assessment may use the same backend. `stopped` and `exhausted` describe control
flow, not content acceptance. Cancellation before callback delivery can return one flow, not content acceptance. Cancellation before callback delivery can return one
completed response in `Undelivered`; errors retain their ordinary identities. completed response in `Undelivered`; errors retain their ordinary identities.
### Collect Selected Execution Facts ### Collect Selected Execution Facts
Groups are independent of continuation and enrollment is explicit: Groups are independent of continuation and can be enrolled explicitly:
```go ```go
var group, validatorGroup promptkit.ExecutionGroup var group, validatorGroup promptkit.ExecutionGroup
@@ -853,7 +938,7 @@ _, _ = summary, validatorSummary
``` ```
Only the first terminal event per run contributes counts and usage; coverage of Only the first terminal event per run contributes counts and usage; coverage of
currently observed members is separate from token-usage completeness. Empty currently observed members is separate from token and cost completeness. Empty
groups report complete, known zero totals, while members without terminals do groups report complete, known zero totals, while members without terminals do
not invent totals. Overlapping groups are not generally additive. See the not invent totals. Overlapping groups are not generally additive. See the
[offline example](../../examples/go-library/observe/main.go) for complete setup. [offline example](../../examples/go-library/observe/main.go) for complete setup.
@@ -1005,7 +1090,7 @@ endpoint, model := "http://localhost:8000/v1", "example-model"
profile := promptkit.ProfileSource("local profile", []promptkit.ProfileConfig{ profile := promptkit.ProfileSource("local profile", []promptkit.ProfileConfig{
{ID: "local-summary", Endpoint: &endpoint, Model: &model}, {ID: "local-summary", Endpoint: &endpoint, Model: &model},
}) })
// Add configkit.Input{Source: profile} to a caller underlay. // Add promptkit.SettingsInput(profile) to a caller underlay.
``` ```
Endpoint-only profiles have an empty backend ID and remain unrestricted by Endpoint-only profiles have an empty backend ID and remain unrestricted by
@@ -1114,10 +1199,20 @@ secret-handling policies.
## Extension Interfaces ## Extension Interfaces
JSON Schema validation and native structured output use a self-contained root JSON Schema validation and native structured output use a self-contained root
document. Internal references are supported; references to other files or document. See [schema selection and resource context](../formats.md#schemas).
network resources fail during preparation. `WithSchemaFile` selects one root Prepared executions retain their captured schema even if the source later changes.
file, while `WithSchemaFS` and `SchemaDir` select a root source. Prepared
executions retain their captured schema even if the source later changes. ### Migrating Schema Sources
`Config.SchemaDir`, `WithSchemaFS`, and `WithSchemaFile` have been removed.
Supply a context-backed `SchemaRootSource` at the desired ConfigKit layer, as
shown above, or declare `schema_root` in application or collection mappings.
For a former single-file source, configure `schema_file` relative to its
contributing source, or supply its directory as a root and use its basename in
`schema_path`. To substitute an unrelated file per request, read and decode it
in the caller and supply its value as `OutputContractInput.Schema`. There is no
implicit working-directory root. Public structs gained fields; change any
positional struct literals to keyed literals when migrating.
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
behavior does not fit the application. Their GoDoc defines concurrent use, behavior does not fit the application. Their GoDoc defines concurrent use,

View File

@@ -6,27 +6,23 @@ This is the canonical reference for Promptkit's backend, profile, prompt, and
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
how to select sources and use an engine; the how to select sources and use an engine; the
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md) [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md) owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md)
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics. owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
This reference describes what Promptkit does with the final effective values. This reference describes what Promptkit does with the final effective values.
A configuration document may contain top-level `backends`, `profiles`, and A complete configuration document places Promptkit settings beneath `promptkit`.
`prompts` lists, plus a `collections` mapping of namespaced resource groups. The `backends`, `profiles`, and `prompts` lists contain explicit literal
Each resource has an explicit literal `id`, not a filename-derived identity. `id` values; `collections` groups namespaced resources. IDs remain exact and
IDs and references must be nonblank and remain exact: Promptkit does not trim or do not include the `promptkit` path. Standalone resource fragments may omit
re-key them. Unknown fields are errors. Promptkit installs a default-on bundled the wrapper when explicitly mounted as described below.
OpenRouter underlay; callers still explicitly select a profile.
```yaml ```yaml
backends: promptkit:
- id: local profiles:
endpoint: http://localhost:8000/v1
concurrency_limit: 2
profiles:
- id: local-summary - id: local-summary
backend: local endpoint: http://localhost:8000/v1
model: example-model model: example-model
prompts: prompts:
- id: meeting.summary - id: meeting.summary
default_profile: local-summary default_profile: local-summary
messages: messages:
@@ -35,42 +31,33 @@ prompts:
output: output:
format: text format: text
validation_mode: none validation_mode: none
collections: collections:
support: support:
release_version: "1.4.0" release_version: "1.4.0"
backends: backends:
- id: offline - id: offline
endpoint: https://example.invalid/v1 endpoint: https://example.invalid/v1
profiles: profiles:
- id: base - id: fast
backend: support/offline backend: support/offline
model: offline-model model: offline-model
- id: fast
extends: {id: base}
timeout_seconds: 30
prompts:
- id: answer
default_profile: support/fast
messages:
- role: user
content: Answer the selected caller input.
output:
format: text
validation_mode: none
``` ```
## Source Headers, Collections, And Resource IDs ## Source Headers, Collections, And Resource IDs
An input may omit a source header. When present, the source-root `version` is Applications own ConfigKit's version policy and source controls. Promptkit adds
reserved for ConfigKit's specification header and accepts only the exact string no resource-version rule, including for collection, prompt, or profile records.
`"2"` or strict decimal integer `2`; null and other values are invalid. This An application's root `version` is distinct from a collection's descriptive
header is distinct from a collection's `release_version` and ConfigKit's own `release_version`. Old root-level `prompts` or `profiles` keys are application
software version. ConfigKit checks and consumes accepted headers on non-main data, not Promptkit resources, and are neither consumed nor specially rejected.
inputs; a declaration on the main input remains visible in explicit
configuration inspection. Headerless inputs still undergo ordinary final-value The `promptkit` section, each collection, backend, profile, and prompt retain
validation. ConfigKit owns generic import and source-history behavior. unrecognized fields as application data. Such fields compose normally and are
For a standalone prompt item, a root `version` is only a source header, not visible to application validation and configuration inspection, but do not
prompt metadata. become provider parameters or appear in resource reports. Grouping them under
an application-specific key reduces collisions; a misspelled optional Promptkit
field can also be accepted there. Nested Promptkit-owned objects, including
message, input, output, and smoke-test objects, remain strict.
The root resource lists contain unqualified IDs. Each `collections` entry The root resource lists contain unqualified IDs. Each `collections` entry
has a namespace key and may contain `release_version` plus the same three has a namespace key and may contain `release_version` plus the same three
@@ -98,8 +85,9 @@ inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
breaking identity and prompt-selection changes, see the breaking identity and prompt-selection changes, see the
[configuration migration guide](consumers/configuration-migration.md). [configuration migration guide](consumers/configuration-migration.md).
Every effective backend, profile, and prompt definition is validated when Every effective backend, profile, and prompt definition is validated during
`NewEngine` constructs the engine, even if it is never selected. A profile may the registered configuration load and engine capture, even if never selected.
A profile may
be partial: an endpoint and model need only be complete after request overrides be partial: an endpoint and model need only be complete after request overrides
are applied. A prompt's default profile similarly need not be runnable without are applied. A prompt's default profile similarly need not be runnable without
an overriding request. Missing request data, artifacts, schemas, credentials, an overriding request. Missing request data, artifacts, schemas, credentials,
@@ -116,7 +104,9 @@ see the consumer guide's [resource and collection listing](consumers/pkg-promptk
## Prompt Definitions ## Prompt Definitions
The `prompts` list contains definitions selected by their literal IDs: The `prompts` list contains definitions selected by their literal IDs. The
following examples in this reference are wrapperless `promptkit` fragments
unless explicitly shown as complete documents; mount them with `SettingsInput`.
```yaml ```yaml
prompts: prompts:
@@ -147,8 +137,9 @@ prompts:
`default_profile`, `description`, `session_id`, `inputs`, and cache-control `default_profile`, `description`, `session_id`, `inputs`, and cache-control
fields may be absent or cleared with null where appropriate. A null whole fields may be absent or cleared with null where appropriate. A null whole
definition, required role, output format, or validation mode is invalid. definition, required role, output format, or validation mode is invalid.
`required` and `repair_attempts` default to false and zero, respectively, and `required` defaults to false and is not nullable. An omitted or null
are not nullable. `InspectPrompt` selects the same captured definition without `repair_attempts` resolves according to `validation_mode`; see the output
contract below. `InspectPrompt` selects the same captured definition without
rendering or resolving a profile. rendering or resolving a profile.
### Inputs ### Inputs
@@ -201,18 +192,30 @@ messages; the integration contract defines its wire representation.
| --- | --- | --- | | --- | --- | --- |
| `format` | yes | `text`, `markdown`, or `json` | | `format` | yes | `text`, `markdown`, or `json` |
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` | | `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` |
| `schema_path` | for `json_schema` | Path in the selected schema source | | `schema` | one selection for `json_schema` | Inline JSON Schema value |
| `repair_attempts` | no | Integer 0–3; default 0. Positive values need `basic`, `json`, or `json_schema` | | `schema_file` | one selection for `json_schema` | File relative to the contributing field's source context |
| `schema_path` | one selection for `json_schema` | Path within the applicable explicit schema root |
| `repair_attempts` | no | Integer 0–3; omitted or null defaults to 0 for `none`, otherwise 1. Positive values need `basic`, `json`, or `json_schema` |
`none` skips content validation; `basic` requires nonempty content; `json` `none` skips content validation; `basic` requires nonempty content; `json`
accepts any valid JSON value; `json_schema` checks the selected schema. accepts any valid JSON value; `json_schema` checks the selected schema.
In `json_schema` mode exactly one nonnull `schema`, `schema_file`, or
`schema_path` must be selected. Null clears a selection; `false`, `true`, and
`{}` are values. Blank references are invalid. Other modes ignore these fields
for schema selection and loading. ConfigKit composes inline mappings normally;
switching forms can require explicitly clearing an inherited alternative.
Replacing an identified prompt record replaces its complete earlier record.
`format` determines output artifact metadata. JSON Schema mode also supplies `format` determines output artifact metadata. JSON Schema mode also supplies
structured-output metadata to compatible clients. Plain `json` does not structured-output metadata to compatible clients. Plain `json` does not
request a provider-native JSON-object constraint. A positive repair budget request a provider-native JSON-object constraint. A positive repair budget
counts additional generations after failed validation; exhaustion returns the counts additional generations after failed validation; exhaustion returns the
last candidate and its failed validation result. Operational errors remain last candidate and its failed validation result. Operational errors remain
errors. A request-level `OutputContract` replaces the whole configured one, errors. A request-level `OutputContractInput` replaces the whole configured
not individual fields; an empty request format defaults to `text`. contract, not individual fields; an empty request format defaults to `text`.
Its omitted budget resolves from its own validation mode, not the prompt's
budget. Explicit zero disables repair. A failed validation can incur one extra
generation by default, with corresponding latency and usage; correction is not
guaranteed.
## Backend Definitions ## Backend Definitions
@@ -220,7 +223,7 @@ Backend definitions are supplied through caller-selected configuration
sources. The conventional backend IDs are names only; PromptKit does not sources. The conventional backend IDs are names only; PromptKit does not
install definitions for them. install definitions for them.
A backend definition may have `endpoint`, `api_key_env`, A backend definition may have `provider`, `zero_cost`, `endpoint`, `api_key_env`,
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The `concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
final endpoint is required and must be an absolute HTTP(S) URL with a host, no final endpoint is required and must be an absolute HTTP(S) URL with a host, no
user information, query, or fragment. An omitted or null API-key environment user information, query, or fragment. An omitted or null API-key environment
@@ -229,7 +232,21 @@ negative value is invalid. For positive concurrency, omitted/null queue
capacity defaults to 1024. An explicit queue capacity, including zero, capacity defaults to 1024. An explicit queue capacity, including zero,
requires positive concurrency and must be nonnegative. Unlimited concurrency requires positive concurrency and must be nonnegative. Unlimited concurrency
has no queue. Provider maps contain JSON-compatible values; null entries are has no queue. Provider maps contain JSON-compatible values; null entries are
ordinary JSON null. Unknown fields, including a raw `api_key`, fail. ordinary JSON null. A raw `api_key` is not a Promptkit credential field: an
unrecognized backend field is application data and is not automatically redacted
by Promptkit. Applications own sensitivity declarations and disclosure policy
for their extension fields.
`provider` selects response and diagnostic conventions independently of the
backend ID, endpoint, or model. Its final value is trimmed and lowercased;
omitted, null, or blank selects `openai-compatible` at runtime. Supported
nonblank values are `openai-compatible`, `openrouter`, and `vllm`. `zero_cost`
is an optional Boolean: true supplies a zero-cost fallback when a dispatched
attempt has no usable reported cost; omitted, null, and false disable it.
Neither setting is sent to the provider. They follow normal layering and
inheritance, and endpoint overrides retain the selected backend's policy.
Resource inspection shows declared normalized values without inventing runtime
defaults; configuration inspection retains source history.
`smoke_tests` is optional backend metadata for explicit diagnostic operations. `smoke_tests` is optional backend metadata for explicit diagnostic operations.
It may contain `default_profile`, `reachability`, and `authentication`: It may contain `default_profile`, `reachability`, and `authentication`:
@@ -247,7 +264,7 @@ backends:
endpoint: https://openrouter.ai/api/v1/key endpoint: https://openrouter.ai/api/v1/key
``` ```
An omitted or null `smoke_tests` block disables these metadata defaults. An omitted or null `smoke_tests` block removes its declared metadata.
`default_profile` is preserved as an exact, nonblank profile ID; its existence `default_profile` is preserved as an exact, nonblank profile ID; its existence
is checked only if an inference diagnostic selects that default. Omitted/null is checked only if an inference diagnostic selects that default. Omitted/null
`default_profile` means there is no inference default. Omitted/null `default_profile` means there is no inference default. Omitted/null
@@ -256,12 +273,22 @@ reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
host; unlike the backend base endpoint, its path and query are allowed. Null host; unlike the backend base endpoint, its path and query are allowed. Null
means fallback, while an explicit blank value is invalid. means fallback, while an explicit blank value is invalid.
Omitted/null `authentication` means no authentication adapter is declared. A Omitted/null `authentication` leaves the provider's diagnostic default in
non-null object requires `kind`, which is trimmed and case-normalized; the effect; `openrouter` defaults to `openrouter_key`, `vllm` defaults to
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint `vllm_models`, and the generic provider has no default. A non-null object's
optional `kind` is trimmed and case-normalized; accepted
values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint
uses the adapter's backend-relative destination (`openrouter_key` appends uses the adapter's backend-relative destination (`openrouter_key` appends
`/key` to the effective backend base); an explicit endpoint follows the same `/key` to the effective backend base); an explicit endpoint follows the same
full-URL rules as reachability. `none` declares authentication inapplicable. full-URL rules as reachability. `none` declares authentication inapplicable.
Without a selected adapter, authentication diagnostics report an unsupported
selection. An endpoint-only authentication
block overrides the destination of a provider default; `kind: none` suppresses
that default.
Previously, a supplied authentication object required `kind`. Endpoint-only
objects now use a provider default where available; clearing the object
restores that default rather than suppressing it. Use `kind: none` to suppress
authentication testing.
These values are configuration only: engine construction does not contact a These values are configuration only: engine construction does not contact a
backend. backend.
@@ -302,7 +329,9 @@ Nonblank endpoints follow the backend URL rules; temperature is 0–2, top_p
is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept
JSON-compatible values with nonempty keys and cannot collide with the built-in JSON-compatible values with nonempty keys and cannot collide with the built-in
client's [standard request fields](integrations/openai-compatible-chat.md#request-body). client's [standard request fields](integrations/openai-compatible-chat.md#request-body).
Credential values do not belong in profiles: raw `api_key` is invalid. A raw `api_key` is not a Promptkit credential setting; an unrecognized profile
field is application data. Applications own its sensitivity declarations and
disclosure policy, just as for backend extension fields.
### Defaults And Overrides ### Defaults And Overrides
@@ -350,12 +379,13 @@ does not disable the caller context or HTTP transport timeout.
### Bundled OpenRouter Configuration ### Bundled OpenRouter Configuration
The default bundle installs root backend `openrouter`, endpoint The default bundle installs root backend `openrouter`, provider `openrouter`, endpoint
`https://openrouter.ai/api/v1`, optional credential environment name `https://openrouter.ai/api/v1`, optional credential environment name
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the `OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback `openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
applies. No prompt, implicit profile selection, or tuning defaults are installed. applies. No prompt, implicit profile selection, or tuning defaults are installed.
Disable the entire bundle with `WithBundledOpenRouter(false)`. Disable the bundle for `NewEngine` with `WithBundledOpenRouter(false)`; an
application-owned load selects `BundledOpenRouterUnderlay` explicitly.
The root profile IDs and their models are: The root profile IDs and their models are:
@@ -410,8 +440,12 @@ on an input. Single files can use ConfigKit inputs directly.
Root backends, profiles, and prompts, as well as collection declarations and Root backends, profiles, and prompts, as well as collection declarations and
their resource lists, follow these same source rules. `CollectionInput` their resource lists, follow these same source rules. `CollectionInput`
mounts one explicitly selected source at a collection destination and enables mounts one explicitly selected source at `/promptkit/collections/<namespace>` and enables
ConfigKit imports for that input; it does not discover or select sources. ConfigKit imports for that input; it does not discover or select sources.
`SettingsInput` mounts wrapperless settings fragments at `/promptkit`; an
id-first resource file uses ConfigKit sequence-item placement at the matching
resource list. Directory discovery retains root destinations, so callers mount
discovered wrapperless fragments explicitly. No document shape is inferred.
A later same-ID record replaces the lower definition in full. Use a distinct A later same-ID record replaces the lower definition in full. Use a distinct
inherited sibling when a variant should retain its parent's fields. inherited sibling when a variant should retain its parent's fields.
A request `ProfileID` takes precedence over a prompt `default_profile`; if A request `ProfileID` takes precedence over a prompt `default_profile`; if
@@ -429,18 +463,35 @@ references are exact IDs and use their qualified form when they select a
collection resource. There is no Promptkit-specific depth limit or second collection resource. There is no Promptkit-specific depth limit or second
base-chain resolver. Missing references and cycles fail at construction. For base-chain resolver. Missing references and cycles fail at construction. For
generic inheritance and layer mechanics, consult generic inheritance and layer mechanics, consult
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md). [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md).
## Schemas ## Schemas
Schemas are JSON documents selected by a prompt or request `schema_path`. Schemas can be inline, source-relative files, or files under an explicit root.
Directory and `fs.FS` schema paths resolve within the configured root. Each An optional `schema_root` may be set at the `promptkit` root and in each
schema must be self-contained: in-document references, IDs, and anchors are collection. A collection's nonnull root takes precedence over the application
root for its prompts; an absent or null collection root falls back to the
application root. A nonnull blank root is invalid. There is no implicit
working-directory root. Roots and `schema_file` references resolve using the
effective field's contributing ConfigKit filesystem and base directory;
missing usable context is a configuration error. An explicit `.` is valid
with usable context. A `schema_path` requires an applicable root. Root choice
is made before reading: a missing file never triggers another-root search.
Schema selection and context are captured during construction without reading
schema files. Files are read and compiled during preparation; a missing,
malformed, or unresolvable file is an operational validation error. Inline
configuration values follow native ConfigKit composition and null handling.
For request-level replacement, inline `Schema` needs no root and `SchemaPath`
uses the selected prompt's applicable root. A replacement does not inherit the
configured schema selection or `schema_file` context.
Runtime inline values are encoded as JSON before preparation; file-backed JSON
uses the existing number-preserving decoder. No additional numerical equivalence
between input representations is implied.
Each schema must be self-contained: in-document references, IDs, and anchors are
supported, but references requiring another file or network resource fail. supported, but references requiring another file or network resource fail.
References must also work without the source filename as a base; use fragment References must also work without the source filename as a base; use fragment
references or explicit schema IDs rather than relying on that filename. references or explicit schema IDs rather than relying on that filename.
`WithSchemaFile` exposes one schema under its base name. An unreadable,
invalid, or unresolvable schema causes an operational validation error.
Valid JSON that fails the schema instead yields a failed validation result. Valid JSON that fails the schema instead yields a failed validation result.
Schema documents are detached without applying the provider extra-parameter Schema documents are detached without applying the provider extra-parameter
copy limits; this does not change the schema decoder, compiler, or copy limits; this does not change the schema decoder, compiler, or

View File

@@ -58,7 +58,7 @@ report facts, and injected-client behavior.
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
one unauthenticated `GET` to the selected diagnostic URL; an observed response one unauthenticated `GET` to the selected diagnostic URL; an observed response
counts as reachable even when its status is non-2xx. Authentication behavior counts as reachable even when its status is non-2xx. Authentication behavior
is selected from backend metadata, not inferred from the host. The is selected from backend metadata and provider defaults, not inferred from the host. The
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective `openrouter_key` adapter makes one `GET` to `/key` relative to the effective
backend base, unless configuration supplies a full diagnostic endpoint (see backend base, unless configuration supplies a full diagnostic endpoint (see
the [backend format reference](../formats.md#backend-definitions)). It may send the [backend format reference](../formats.md#backend-definitions)). It may send
@@ -73,6 +73,15 @@ not validation of the full account document. Other statuses and unrecognized
recognition allowance reports `authentication_response_limit_exceeded`, leaving recognition allowance reports `authentication_response_limit_exceeded`, leaving
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go). credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
The `vllm_models` adapter issues one `GET` to `/models` relative to the
configured base unless an explicit diagnostic endpoint is supplied. It sends
an available credential but does not require one. A 200 response is recognized
when it is a JSON object with `object: "list"` and an array-valued `data`;
empty arrays are accepted. A 401 means rejection; other responses do not
establish acceptance. Recognition uses the same bounded response handling as
the key probe. Acceptance does not establish that authentication is enforced or
that inference is authorized.
Diagnostic HTTP evidence describes the last request observed at Promptkit's Diagnostic HTTP evidence describes the last request observed at Promptkit's
RoundTripper boundary and any response associated with it. It includes the RoundTripper boundary and any response associated with it. It includes the
selected destination without URL user information, response status, and selected destination without URL user information, response status, and
@@ -154,7 +163,22 @@ data, a second JSON value, absent choices, missing content, `null` content,
non-string content, and size overflow are malformed responses and return no non-string content, and size overflow are malformed responses and return no
partial result. partial result.
Each of the five token fields retains whether the provider reported a For OpenRouter and vLLM, the client also reads reasoning tokens from
`usage.completion_tokens_details.reasoning_tokens`. OpenRouter cache writes
come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
`usage.prompt_tokens_details.created_cache_tokens`. The generic provider keeps
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
fractional, or unrepresentable new details remain unknown without invalidating
otherwise usable content. Existing common count decoding remains strict.
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
amounts are ignored without invalidating content. The generic and vLLM
providers do not interpret that field.
vLLM prompt-token details depend on server configuration, including
`--enable-prompt-tokens-details`; reasoning details depend on a configured
reasoning parser. Servers without these details remain usable. vLLM cache
writes describe local prefix-cache creation, not a billable write charge.
Each common token field retains whether the provider reported a
non-null integer, including zero. Omitted and `null` fields are unknown for non-null integer, including zero. Omitted and `null` fields are unknown for
observations. A strictly decoded successful envelope can yield observed usage observations. A strictly decoded successful envelope can yield observed usage
even when a later missing choice or content makes the attempt malformed. even when a later missing choice or content makes the attempt malformed.

View File

@@ -79,7 +79,7 @@ other backend IDs.
The coordinator does not retry injected clients; it normalizes a nil successful The coordinator does not retry injected clients; it normalizes a nil successful
response as malformed generation. The built-in HTTP client remains responsible response as malformed generation. The built-in HTTP client remains responsible
for provider transport behavior. The runner, rather than the for provider transport behavior. The runner, rather than the
capacity layer, sums all five usage fields from the initial response and every capacity layer, sums usage fields from the initial response and every
completed repair response into the successful run result. completed repair response into the successful run result.
An invocation-local observer receives attempt events after active permit An invocation-local observer receives attempt events after active permit

View File

@@ -35,6 +35,12 @@ unknown usage. Transport, timeout, malformed, and noneligible status failures
stop the sequence without retry exhaustion. Context is checked before each stop the sequence without retry exhaustion. Context is checked before each
wait, acquisition, and dispatch. wait, acquisition, and dispatch.
At every dispatched completion the coordinator resolves any reported cost
against the frozen backend zero-cost policy and adds it once to the run's
[cost accumulator](observation.md), independently of observer selection.
Injected responses may retain reported cost alongside an error. Undispatched
acquisitions and retry waits contribute no cost.
The public client adapter translates explicitly supplied public transport and The public client adapter translates explicitly supplied public transport and
malformed-response identities into internal categories while retaining the malformed-response identities into internal categories while retaining the
original error chain. This keeps injected attempt classifications consistent original error chain. This keeps injected attempt classifications consistent

View File

@@ -142,6 +142,14 @@ known. Strictly decoded usage can still be observed if later choice/content
checks fail; invalid framing or a non-success error body contributes none. checks fail; invalid framing or a non-success error body contributes none.
Existing successful-result accounting remains value-based. Existing successful-result accounting remains value-based.
The captured provider selects additional response details and authentication
diagnostic defaults. Optional new reasoning, cache-write, and OpenRouter cost
fields are decoded independently, without relaxing common-field or framing
validation. Cost remains an optional raw amount at this boundary; shared
accounting applies configured-zero fallback after dispatch. The integration
contract specifies the [wire mappings](../integrations/openai-compatible-chat.md#response-handling)
and [probe recognition](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
For a non-success response, `ProviderHTTPError` retains the HTTP status and For a non-success response, `ProviderHTTPError` retains the HTTP status and
only normalized detail from the bounded recognized envelope. It retains only normalized detail from the bounded recognized envelope. It retains
`ErrUnexpectedStatus` through unwrapping. The client owns response closure; `ErrUnexpectedStatus` through unwrapping. The client owns response closure;
@@ -159,6 +167,13 @@ caller deadlines, generation deadlines, and whole-request client timeouts
retain `context.DeadlineExceeded`. The runner adds its generation category retain `context.DeadlineExceeded`. The runner adds its generation category
without discarding those identities or depending on HTTP status mapping. without discarding those identities or depending on HTTP status mapping.
When a transport failure carries a custom context cancellation cause, the
client also retains the standard context identity. It correlates that cause
with the returned failure rather than labeling unrelated transport errors from
context state alone. Cancellation while reading a successful response body
retains the original read failure alongside the context identity within the
redacting wrapper.
## Test Ownership ## Test Ownership
The The

View File

@@ -18,11 +18,14 @@ Timeout attribution is finalized and the child scope released before terminal
delivery, so observer latency cannot change the already selected outcome. delivery, so observer latency cannot change the already selected outcome.
Prepared-content notification is a separate opt-in callback, not an event. Prepared-content notification is a separate opt-in callback, not an event.
The facade masks inherited recorder and prepared-claim state when entering The facade masks inherited recorder, cost, and prepared-claim state when entering
preparation or execution. Delegated calls from an injected collaborator retain preparation or execution. Delegated calls from an injected collaborator retain
the caller's cancellation, deadline, and unrelated context values, but do not the caller's cancellation, deadline, and unrelated context values, but do not
reuse the enclosing invocation's accounting. Only the new request's observer reuse the enclosing invocation's accounting. Ordinary runs compose the request
(or the claimed handle's retained observer) enables its observation lifecycle. observer before any observers selected by the invocation context; contextual
selection survives this boundary. Prepared executions compose the retained
request observer with selection from the execution context after a successful
claim; the preparation context does not select execution observers.
A recorder keeps only counters, resolved identity fields, and the selected A recorder keeps only counters, resolved identity fields, and the selected
callback. It has no global history, queue, goroutine, or mutex held across a callback. It has no global history, queue, goroutine, or mutex held across a
@@ -53,12 +56,24 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its one dispatch. A terminal event is an aggregate and must not be added to its
attempt events. attempt events.
An execution group is a zero-ready synchronized collector of explicitly selected Ordinary runs and claimed prepared executions install invocation-local cost
accounting even without an observer.
The generation coordinator resolves cost once per dispatched attempt and adds
it to that state independently of event delivery. The result and recorder read
the same accumulator; groups merge first terminal costs independently of
terminal coverage.
Reasoning counts use the same per-field rules and are a subset of completion
tokens. The public cache-hit helper derives a ratio from complete, consistent
summed input and cache-read counts; it does not aggregate ratios or use writes.
An execution group is a zero-ready synchronized collector of selected
events. It enrolls by run ID and aggregates each run's first terminal counters and events. It enrolls by run ID and aggregates each run's first terminal counters and
presence-aware usage exactly once. Snapshots copy members and totals under one presence-aware usage exactly once. Snapshots copy members and totals under one
lock; terminal coverage and token completeness are distinct. Observer fan-out lock; terminal coverage and token completeness are distinct. Observer fan-out
copies the callback slice and invokes callbacks synchronously without holding a copies the callback slice and invokes callbacks synchronously without holding a
group lock. No group history, registry, or automatic nested membership exists. group lock. No group history or registry exists; contextual selection can enroll
nested ordinary runs that use the scoped context.
Smoke diagnostics mask inherited execution-recorder state while invoking Smoke diagnostics mask inherited execution-recorder state while invoking
generation or HTTP collaborators. Their private attempt evidence is assembled generation or HTTP collaborators. Their private attempt evidence is assembled

View File

@@ -16,11 +16,12 @@ contributor workflow and validation.
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) | | `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `examples/go-library/observe` | Demonstrates offline retry, observation, prepared notification, and separately budgeted coordinated proposal and assessment runs using two profiles on one finite backend. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) | | `examples/go-library/observe` | Demonstrates offline retry, observation, prepared notification, and separately budgeted coordinated proposal and assessment runs using two profiles on one finite backend. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) | | `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
| `internal/configuration` | Loads root and namespaced resources through ConfigKit; validates final values, captures indexed resource views and source-relative prompt content, compiled templates, requirement metadata, identity inventories, backend and presence-aware execution profiles, and the read-only provenance inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) | | `internal/configuration` | Supplies component rules and captures final shared settings and native inspection evidence into owned indexed resources, compiled templates, requirement metadata, identities, and execution profiles. | [Configuration capture](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/settings` | Declares the shared public configuration data model exposed through root aliases under ADR 0002; contains no loading or execution logic. | [Settings declarations](../../internal/settings/settings.go), [ADR 0002](../adr/0002-shared-configuration-data-model.md) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) | | `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) | | `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and identity listing directly to the captured configuration rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) | | `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) | | `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded retries, per-attempt cost, ordinary observations, and separate diagnostic evidence. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
@@ -28,9 +29,9 @@ contributor workflow and validation.
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) | | `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | | `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible requests, provider-specific response interpretation and authentication probes, bounded decoding, deadlines, and diagnostic evidence. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) | | `internal/observation` | Owns invocation-local run IDs, presence-aware token and cost accounting, contextual observer selection, synchronous event delivery, and terminal-based group aggregation; smoke diagnostics retain separate evidence. | [Internal execution observation](observation.md) |
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | | `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The internal engine assembles these internal components without exposing their The internal engine assembles these internal components without exposing their

View File

@@ -30,6 +30,11 @@ internal callers and tests.
Each invocation carries its state in request, prepared-run, and result values. Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store. The runner has no durable run or session store.
Request output-contract replacements carry an optional repair budget. The
runner resolves that budget from the replacement's validation mode before
building the concrete validation plan; prepared execution retains the resolved
integer. Configured budgets are already resolved by the configuration adapter.
Captured exchanges are immutable parent-linked domain values. Each holds only Captured exchanges are immutable parent-linked domain values. Each holds only
its new rendered suffix, final assistant message, and run identities; flattening its new rendered suffix, final assistant message, and run identities; flattening
iteratively fills a single detached message buffer for preparation or projects iteratively fills a single detached message buffer for preparation or projects
@@ -126,12 +131,12 @@ performs only the work needed to validate routing and admission:
1. validate the required prompt selection and normalize any direct session ID; 1. validate the required prompt selection and normalize any direct session ID;
2. select the captured prompt definition and hash; 2. select the captured prompt definition and hash;
3. select the request profile or the prompt's default profile; 3. resolve and validate the effective output contract and select its applicable
4. retain the profile's selected backend identity, when present; bound file without loading its schema;
5. apply explicit request overrides to its captured backend-completed target; 4. select the request profile or the prompt's default profile;
6. validate endpoint, model, numeric overrides, and credential requirements; 5. retain the profile's selected backend identity, when present;
7. resolve and validate the effective output contract without loading its 6. apply explicit request overrides to its captured backend-completed target;
schema; and 7. validate endpoint, model, numeric overrides, and credential requirements; and
8. retain the definition, source identities, effective settings, output 8. retain the definition, source identities, effective settings, output
contract, and preparation start time in invocation-local state. contract, and preparation start time in invocation-local state.
@@ -144,7 +149,7 @@ profile, or backend:
3. render messages and the prompt-defined session; 3. render messages and the prompt-defined session;
4. apply any direct session ID, then compose captured history, rendered 4. apply any direct session ID, then compose captured history, rendered
definition messages, and already-normalized request messages in that order; definition messages, and already-normalized request messages in that order;
5. hash the effective rendered prompt; and 5. hash the effective rendered prompt and its ordered message content; and
6. construct the prepared value and preparation timing. 6. construct the prepared value and preparation timing.
`Prepare` runs both phases consecutively and never performs capacity admission. `Prepare` runs both phases consecutively and never performs capacity admission.
@@ -185,6 +190,12 @@ sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
encoding of that session, every role and content value, and cache-control encoding of that session, every role and content value, and cache-control
presence and values; its hexadecimal value is opaque. presence and values; its hexadecimal value is opaque.
The separate message-content hash is computed once after that same composition.
It streams a versioned prefix, message count, and length-framed role and text
bytes into SHA-256, excluding session and cache metadata. Fixed reference
vectors protect the selected v1 scheme across refactors; the public
[`PreparedRun` GoDoc](../../types.go) owns its compatibility contract.
The configuration snapshot is read-only after engine construction. Concurrent The configuration snapshot is read-only after engine construction. Concurrent
`Prepare` and `Run` calls use detached captured targets and keep invocation `Prepare` and `Run` calls use detached captured targets and keep invocation
state local. state local.
@@ -205,10 +216,16 @@ ordinary `Run` reuses its run ID and carries it to the generation coordinator
through context. Prompt and profile identities are enriched only after their through context. Prompt and profile identities are enriched only after their
respective resolutions succeed, before admission. The facade starts observation respective resolutions succeed, before admission. The facade starts observation
before request conversion and emits its terminal event after public error before request conversion and emits its terminal event after public error
mapping and after the runner's deferred admission release. Preparation and mapping and after the runner's deferred admission release. Preparation emits no
prepared execution follows the same terminal mapping after a successful claim; events; prepared execution follows the same terminal mapping after a successful claim;
invalid or discarded handles do not acquire an observer. invalid or discarded handles do not acquire an observer.
Ordinary runs establish fresh private cost state before observation; a claimed
prepared execution establishes its own state after claim. Contextual observers
remain available at public boundaries, while enclosing recorder and cost state
are masked. Prepared handles retain the request observer and frozen provider
policy, not preparation-context selection or an accumulator.
The facade establishes an optional operation scope for ordinary runs before The facade establishes an optional operation scope for ordinary runs before
observation and preparation. A claimed prepared execution establishes its own observation and preparation. A claimed prepared execution establishes its own
scope from the retained duration; preparation and idle time do not consume it. scope from the retained duration; preparation and idle time do not consume it.
@@ -262,11 +279,13 @@ run identifier, and UTC timing. The same effective session reaches initial
generation and any repair attempt through the rendered prompt. The same generation and any repair attempt through the rendered prompt. The same
effective target and presence metadata, including backend identity and direct effective target and presence metadata, including backend identity and direct
credential during execution, reaches generation and every repair attempt. credential during execution, reaches generation and every repair attempt.
Result usage is the field-wise sum of all five usage values from the initial Result usage is the field-wise sum of usage values from the initial
response and every completed repair response. Final raw output, artifact, and response and every completed repair response. Final raw output, artifact, and
validation state still come from the last candidate. A repair error returns no validation state still come from the last candidate. A repair error returns no
partial run result or partial result usage, although the invocation-local partial run result or partial result usage, although the invocation-local
observer retains completed-attempt usage and reports it in the terminal event. observer retains completed-attempt usage and reports it in the terminal event.
Result cost instead snapshots the shared accumulator across all dispatched
attempts, including retries and repairs, whether or not observation is enabled.
## Failure Categories ## Failure Categories

View File

@@ -10,38 +10,24 @@ source semantics. Internal packages are not consumer APIs.
## Typed Configuration Loading ## Typed Configuration Loading
`internal/configuration` is the single adapter from ConfigKit's final The root facade composes Promptkit rules into a ConfigKit definition. An
effective configuration to execution-owned definitions and resource views. It performs one application-owned load or the standalone constructor produces final settings
layered load, captures the needed effective presence, records at `/promptkit`; `internal/configuration` captures execution-owned resources
field-level normalization and derived defaults during resolution, and validates all effective from those settings and a native inspection view without another load.
root and collection backend, profile, and prompt values. Replaced lower-layer `internal/settings` supplies the shared data declarations exposed through
domain values are not separately validated. It indexes root resources under root aliases. The adapter records normalization and validates effective values,
their unqualified IDs and collection resources under their full then uses one metadata snapshot for presence and source origins. Runtime/domain
`namespace/local-key` IDs while retaining ConfigKit's resolved tree. It builds an immutable backend index and presence-aware profile packages do not depend on ConfigKit.
targets, applying final backend defaults only after profile inheritance. Runtime
lookups use those final indexes and exact stored references; they do not rewrite
or resolve resources from ConfigKit source history. The runtime/domain packages
do not depend on ConfigKit.
ConfigKit owns source headers, imports, composition, and source evidence. Capture detaches known settings, builds indexed definitions and profile targets,
Promptkit does not preparse contributions: it validates the final effective and binds source-relative resources from the effective field's source context.
values once, captures the indexed identities and their original resource It retains a frozen native focused view and copied diagnostics, not the mutable
contexts, and retains ConfigKit's resolved result for explicit operator inspection. application result. Configuration inspection delegates reporting, redaction,
Accepted non-main source headers are consumed during composition; the main history and comparison to ConfigKit; explicit queries and metadata retain
source declaration remains available in that inspection view. Collection whole-application paths and evidence. Runtime never derives values from history.
release labels are descriptive metadata, not selection inputs. The The [format reference](../formats.md#source-headers-collections-and-resource-ids)
[format reference](../formats.md#source-headers-collections-and-resource-ids) owns the configuration contract and the [consumer guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
owns the exact source-header and collection contracts. describes reporting scope.
The `ConfigurationInspection.Diff` facade method compares two retained results
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
once to ConfigKit and returns its public difference records. Comparison
uses only frozen effective values and does not reopen sources, resolve runtime
credentials, or enter the execution path. ConfigKit applies sensitivity from
both results; reports may still contain ordinary application data. See the
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and ConfigKit's [comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#effective-configuration-diffs).
After resource materialization and effective-profile snapshot construction, the After resource materialization and effective-profile snapshot construction, the
adapter captures lexically ordered identity inventories for backends, profiles, adapter captures lexically ordered identity inventories for backends, profiles,
@@ -52,15 +38,15 @@ captured values without consulting ConfigKit or copying resource definitions.
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
files under an explicit root, provide resource context, and leave imports files under an explicit root, provide resource context, and leave imports
disabled. Each returned slice belongs to one strict-composition group; file disabled. Each returned slice belongs to one strict-composition group; file
order does not confer priority. Engine assembly prepends a dedicated embedded order does not confer priority. The standalone facade prepends a dedicated
OpenRouter underlay unless disabled, without modifying caller-owned slices. embedded OpenRouter underlay unless disabled, without modifying caller-owned slices.
Caller underlays are composed in their given Caller underlays are composed in their given
order, followed by the optional main input. The order, followed by the optional main input. The
[configuration tests](../../internal/configuration/load_test.go) exercise the [configuration tests](../../internal/configuration/load_test.go) exercise the
real loader and final-value handoff. real loader and final-value handoff.
The public `CollectionInput` helper mounts one caller-selected source at The public `CollectionInput` helper mounts one caller-selected source at
`collections/<namespace>` and enables ConfigKit imports for that input. The `/promptkit/collections/<namespace>` and enables ConfigKit imports for that input. The
helper does not read the source; ConfigKit retains responsibility for loading, helper does not read the source; ConfigKit retains responsibility for loading,
composition, imports, and source-local resource context. Imported or inherited composition, imports, and source-local resource context. Imported or inherited
`content_file` values continue resolving relative to the source that contributed `content_file` values continue resolving relative to the source that contributed
@@ -83,10 +69,13 @@ declaration; absent and null fields remain distinguishable from explicit
strings, including empty strings. It closes content-file handles before strings, including empty strings. It closes content-file handles before
returning and retains no source capability for public inspection. returning and retains no source capability for public inspection.
Prompt materialization shares a lazy metadata snapshot across root and collection Prompt materialization shares one metadata snapshot across root and collection
resources during construction, avoiding a full metadata copy per collection. resources during construction, avoiding a full metadata copy per collection.
Input metadata, message content and templates, and output contracts are captured Input metadata, message content and templates, and output contracts are captured
separately before the definition, inspection, and hash are assembled. separately before the definition, inspection, and hash are assembled. Schema
root and source-relative file bindings capture the effective field context
without opening schema files; capabilities are excluded from the hash. Inline
schema values and file-reference strings participate in the definition hash.
Normalization and defaults are recorded at individual resolved field paths, Normalization and defaults are recorded at individual resolved field paths,
including message roles and cache controls. List indexes identify fields for including message roles and cache controls. List indexes identify fields for
@@ -149,7 +138,9 @@ ConfigKit resolves profile inheritance before the handoff. Within a collection,
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference. `extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
The configuration adapter validates final effective resource references and The configuration adapter validates final effective resource references and
field values, then captures backend definitions and profile target snapshots field values, then captures backend definitions and profile target snapshots
with backend defaults and outbound-control presence. Null profile endpoint, with backend defaults and outbound-control presence. Profiles without provider
overrides share capture-owned immutable backend parameter maps; target lookups
still detach mutable values before runtime use. Null profile endpoint,
credential source, and whole provider map permit backend fallback after credential source, and whole provider map permit backend fallback after
composition, subject to the required-credential rule. A profile can remain composition, subject to the required-credential rule. A profile can remain
partial until runtime overrides complete it. Resource inspection selects indexed partial until runtime overrides complete it. Resource inspection selects indexed
@@ -212,18 +203,20 @@ buffer-to-string copy. The
## Schemas And Output Validation ## Schemas And Output Validation
`internal/validate` provides validators backed by an operating-system `internal/validate` prepares a source-neutral input containing an output contract
filesystem or an `fs.FS`. Invalid generated content is returned as a validation and an optional bound `fs.FS` file. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational result; inability to load, register, or compile a schema is an operational
error. Basic validation checks body emptiness on the byte slice directly error. Basic validation checks body emptiness on the byte slice directly
instead of first making a response-sized string copy. instead of first making a response-sized string copy.
Every preparation operation creates one operation-local validation plan. None, Every preparation operation creates one operation-local validation plan. None,
basic, and JSON modes retain the effective output contract without source basic, and JSON modes retain the effective output contract without source
access. JSON Schema mode loads the root document once, compiles its declared access. JSON Schema mode uses the captured inline document or loads the bound
root document once, compiles its declared
schema locations, and retains the compiled root validator. External resources schema locations, and retains the compiled root validator. External resources
are rejected without file or network loading; in-document references remain are rejected without file or network loading; in-document references remain
supported. Root resources use canonical escaped file or private-scheme URLs. The supported. Bound files use canonical escaped private-scheme URLs; inline values
use a stable private retrieval URL. The
provider-facing structured-output metadata uses the root document captured by provider-facing structured-output metadata uses the root document captured by
the same plan. Its detached schema document is not revalidated against the the same plan. Its detached schema document is not revalidated against the
runtime provider-value copier's traversal limits. Smoke inference uses a runtime provider-value copier's traversal limits. Smoke inference uses a

View File

@@ -26,81 +26,60 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
## ConfigKit ## ConfigKit
Selected module: `go.maximumdirect.net/configkit` at `v1.2.0`. Selected module: `go.maximumdirect.net/configkit` at `v1.4.0`.
Copyright (C) 2026 Eric Rakestraw Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL 3.0](../../LICENSE) | License and notices: [LGPL 3.0](../../LICENSE) |
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/COPYRIGHT) | [COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3/COPYRIGHT) |
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/LICENSE). [LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3/LICENSE).
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e). [Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3).
## ReleaseKit ## ReleaseKit
Selected module: `go.maximumdirect.net/releasekit` at `v0.9.0` (maintainer tool). Selected module: `go.maximumdirect.net/releasekit` at `v0.10.0` (maintainer tool).
Copyright (C) 2026 Eric Rakestraw Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL 3.0](../../LICENSE) | License and notices: [LGPL 3.0](../../LICENSE) |
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/COPYRIGHT) | [COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243/COPYRIGHT) |
[LICENSE](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/LICENSE). [LICENSE](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243/LICENSE).
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4). [Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243).
## jsonschema ## jsonschema
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.2`. Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.3`.
SPDX-License-Identifier: Apache-2.0 SPDX-License-Identifier: Apache-2.0
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) | License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
[LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/29cbed948d24a04700eb94436416b18a07953b71/LICENSE). [LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/b0fc661f4939578bc429f408c18202573dbafcc4/LICENSE).
[Source](https://github.com/santhosh-tekuri/jsonschema/tree/29cbed948d24a04700eb94436416b18a07953b71). [Source](https://github.com/santhosh-tekuri/jsonschema/tree/b0fc661f4939578bc429f408c18202573dbafcc4).
## Go Text ## Go Text
Selected module: `golang.org/x/text` at `v0.14.0` (transitive dependency). Selected module: `golang.org/x/text` at `v0.44.0` (transitive dependency).
Copyright (c) 2009 The Go Authors. All rights reserved. Copyright 2009 The Go Authors.
SPDX-License-Identifier: BSD-3-Clause SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) | License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
[LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) | [LICENSE](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/LICENSE) |
[PATENTS](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS). [PATENTS](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/PATENTS).
[Source](https://github.com/golang/text/tree/6c97a165dd661335ff7bce6104a008558123c353). [Source](https://github.com/golang/text/tree/6070ac32abcdd1670599eeb3ee4d513c8184a513).
## Go YAML v3
Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency).
Copyright (c) 2006-2010 Kirill Simonov
Copyright (c) 2006-2011 Kirill Simonov
Copyright (c) 2011-2019 Canonical Ltd
Copyright 2011-2016 Canonical Ltd.
SPDX-License-Identifier: MIT AND Apache-2.0
License and notices: [MIT](licenses/MIT.txt) |
[Apache 2.0](licenses/Apache-2.0.txt) |
[LICENSE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
[Source](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
## Go YAML v4 ## Go YAML v4
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0-rc.6` (transitive dependency). Selected module: `go.yaml.in/yaml/v4` at `v4.0.0`.
Copyright 2006-2010 Kirill Simonov Copyright 2006-2010 Kirill Simonov
@@ -112,7 +91,34 @@ SPDX-License-Identifier: MIT AND Apache-2.0
License and notices: [MIT](licenses/MIT.txt) | License and notices: [MIT](licenses/MIT.txt) |
[Apache 2.0](licenses/Apache-2.0.txt) | [Apache 2.0](licenses/Apache-2.0.txt) |
[LICENSE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/LICENSE) | [LICENSE](https://github.com/yaml/go-yaml/blob/f2f9563503f11b7a60da0c532175dcc693ce8d5a/LICENSE) |
[NOTICE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/NOTICE). [NOTICE](https://github.com/yaml/go-yaml/blob/f2f9563503f11b7a60da0c532175dcc693ce8d5a/NOTICE).
[Source](https://github.com/yaml/go-yaml/tree/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125). [Source](https://github.com/yaml/go-yaml/tree/f2f9563503f11b7a60da0c532175dcc693ce8d5a).
## Goldmark
Selected module: `github.com/yuin/goldmark/v2` at `v2.1.16` (maintainer-tool dependency).
Copyright (c) 2019 Yusuke Inuzuka
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT.txt) |
[LICENSE](https://github.com/yuin/goldmark/blob/c3f939122ba4dd6f80e05043ad195b82e77fcb41/LICENSE).
[Source](https://github.com/yuin/goldmark/tree/c3f939122ba4dd6f80e05043ad195b82e77fcb41).
## Go Modules
Selected module: `golang.org/x/mod` at `v0.43.0` (maintainer-tool dependency).
Copyright 2009 The Go Authors.
SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
[LICENSE](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde/LICENSE) |
[PATENTS](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde/PATENTS).
[Source](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde).

View File

@@ -2,7 +2,7 @@
**Base Policy Version:** 1.3 **Base Policy Version:** 1.3
**Last Updated**: 2026-09-27 **Last Updated**: 2026-11-21
**Upstream Source:** [architecture.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md) **Upstream Source:** [architecture.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
@@ -51,9 +51,10 @@ The [internal component overview](../internal/overview.md) owns the inventory
of implemented packages and their concrete responsibilities. This policy owns of implemented packages and their concrete responsibilities. This policy owns
the boundaries and dependency rules those components must preserve. the boundaries and dependency rules those components must preserve.
The internal engine composes caller-selected ConfigKit inputs once, then The facade either registers Promptkit rules in the application's ConfigKit load
assembles the captured definitions, capacity manager, renderer, validator, or performs one standalone load, then passes captured configuration to the
outbound client, and use-case runner. The configuration adapter interprets and internal engine. The internal engine assembles the captured definitions,
capacity manager, renderer, validator, outbound client, and use-case runner. The configuration adapter interprets and
validates only the final effective values, including presence, null, and validates only the final effective values, including presence, null, and
resource context. It retains source history for explicit operator inspection, resource context. It retains source history for explicit operator inspection,
not for validation or a second resolution pass. The root facade translates not for validation or a second resolution pass. The root facade translates
@@ -128,7 +129,8 @@ consumer contract requires a public package.
Each package must have one cohesive responsibility and a clear dependency Each package must have one cohesive responsibility and a clear dependency
direction. Internal packages must not expose their types merely to simplify direction. Internal packages must not expose their types merely to simplify
wiring, and the public facade must not leak internal representations through wiring, and the public facade must not leak internal representations through
exported signatures. Production packages under `internal/` must not import the exported signatures, except for the deliberately public configuration data model
specified below. Production packages under `internal/` must not import the
module root; the root facade may depend on internal implementation packages. module root; the root facade may depend on internal implementation packages.
New public packages require a durable consumer need that cannot be served New public packages require a durable consumer need that cannot be served
cleanly by the root facade. cleanly by the root facade.
@@ -136,12 +138,34 @@ cleanly by the root facade.
The [internal component overview](../internal/overview.md) must be updated as The [internal component overview](../internal/overview.md) must be updated as
packages are implemented or their responsibilities change. packages are implemented or their responsibilities change.
### Shared Configuration Data Exception
[ADR 0002](../adr/0002-shared-configuration-data-model.md) permits one canonical
set of public configuration settings and their nested data types in a small,
data-only internal package, exposed through aliases in the root `promptkit`
package. The facade and internal configuration adapter may use those same
declarations directly. Their fields, tags, and nested types are intentional
public contracts and receive the same API review as root-defined types.
The shared package owns data declarations, not loading, normalization,
validation, or execution behavior. This exception does not authorize aliases
for arbitrary internal implementation types or merging configuration settings
with source-construction helpers, execution requests, runtime objects, or
inspection reports that have different meanings. Internal packages still must
not import the root facade. Runtime capture still owns the values it retains;
shared declarations do not replace necessary ownership copies.
All other facade, dependency, and ownership rules remain in force. Concrete
package placement and implementation status belong in the active plan and,
once implemented, the internal component inventory.
## Exported API Discipline ## Exported API Discipline
Export the smallest contract required by real consumers. Exported declarations Export the smallest contract required by real consumers. Exported declarations
must have accurate GoDoc, stable semantics, and tests proportionate to their must have accurate GoDoc, stable semantics, and tests proportionate to their
compatibility risk. Avoid speculative extension points, aliases for internal compatibility risk. Avoid speculative extension points, aliases for internal
types, and public constructors that expose assembly details. types outside the shared configuration data exception, and public constructors
that expose assembly details.
Once an exported API exists, its Go declaration and GoDoc own its exact public Once an exported API exists, its Go declaration and GoDoc own its exact public
contract. Architecture documentation owns boundary rules, not a duplicate API contract. Architecture documentation owns boundary rules, not a duplicate API

View File

@@ -74,8 +74,8 @@ Successful publication needs no separate remote read-back or CI wait.
For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`. For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`.
It is not a required step before publication. The upstream It is not a required step before publication. The upstream
[candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/check.md) [candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/check.md)
and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/cli.md) and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/cli.md)
own shared checks and exit-status definitions. own shared checks and exit-status definitions.
## Deliver The Hosted Release ## Deliver The Hosted Release
@@ -116,7 +116,7 @@ module files and supply the token there. In a separate checkout, for example:
Resolve uncertain hosted writes before retrying the same immutable version. Resolve uncertain hosted writes before retrying the same immutable version.
Matching content is accepted; conflicts are not overwritten. Do not rerun Matching content is accepted; conflicts are not overwritten. Do not rerun
`publish` or move tags to repair delivery. Follow the upstream `publish` or move tags to repair delivery. Follow the upstream
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/delivery.md) [delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/delivery.md)
for result interpretation and recovery. These commands apply to tags containing for result interpretation and recovery. These commands apply to tags containing
this delivery configuration; older releases retain their original workflows. this delivery configuration; older releases retain their original workflows.
@@ -143,13 +143,13 @@ and explicitly select the matching attempt:
Discovery is local; reconciliation observes the original destination without Discovery is local; reconciliation observes the original destination without
rerunning validation, changing tags, or delivering hosted content. Neither is rerunning validation, changing tags, or delivering hosted content. Neither is
a routine gate after successful publication. Follow the upstream a routine gate after successful publication. Follow the upstream
[recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/release.md#resolve-publication-failures) [recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/release.md#resolve-publication-failures)
for outcomes, incomplete evidence, and older attempts without records. for outcomes, incomplete evidence, and older attempts without records.
Records are stored under Git's common directory by default and can contain Records are stored under Git's common directory by default and can contain
credential-bearing endpoints. Keep them private, out of source control and credential-bearing endpoints. Keep them private, out of source control and
shared logs; retain them while needed for recovery and remove them when that shared logs; retain them while needed for recovery and remove them when that
value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence) value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
for storage and retention details. for storage and retention details.
Current absence does not prove a version was never published. Remove a retained Current absence does not prove a version was never published. Remove a retained

33
docs/releases/v0.23.1.md Normal file
View File

@@ -0,0 +1,33 @@
# Promptkit v0.23.1
This patch release fixes construction-error wrapping so downstream consumers
can classify failures from their underlying causes without treating the
`ErrInvalidConfig` category as an independent failure.
## Fixed
`NewEngine` now uses single-cause wrapping for both configuration-loading and
option-application failures. `errors.Is` still recognizes `ErrInvalidConfig`,
and `errors.As` still exposes ConfigKit loading errors with their diagnostics
and source context. Genuine joined causes, including cancellation, remain
inspectable. Consumers retain responsibility for mapping those failures to
application outcomes or exit statuses.
See the [constructor GoDoc](../../engine.go) and
[error-handling guide](../consumers/pkg-promptkit.md#handle-errors) for the
public error contracts and usage.
## Compatibility And Upgrade
No configuration or request changes are required when upgrading from v0.23.0.
Consumers that inspect constructor error trees directly should account for
the category no longer appearing as a separate sibling branch; use
`errors.Is` and `errors.As` for category and typed-cause inspection.
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
```sh
go get go.maximumdirect.net/promptkit@v0.23.1
```
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/docs/oss/attribution.md)

35
docs/releases/v0.23.2.md Normal file
View File

@@ -0,0 +1,35 @@
# Promptkit v0.23.2
This patch release refreshes configuration, validation, and maintainer-tool
dependencies. No Promptkit configuration or request API migration is required.
## Dependency Updates
- ConfigKit v1.2.0 → v1.2.1 and go-yaml v4.0.0-rc.6 → v4.0.0 move
configuration loading to stable go-yaml, with ConfigKit's corresponding
loader-plugin adaptation.
- jsonschema v6.0.2 → v6.0.3 includes fixes to validation-location reporting,
numeric edge cases, and email validation.
- Go Text (`golang.org/x/text`) v0.14.0 → v0.44.0 refreshes the transitive
dependency used by jsonschema.
- ReleaseKit v0.9.0 → v0.10.0 updates maintainer tooling, selecting Goldmark
v2.1.16 and Go Modules (`golang.org/x/mod`) v0.43.0. ReleaseKit's Go 1.27.0
minimum is already satisfied by Promptkit's existing minimum.
Dependency notices and revision links are maintained in the
[attribution document](../oss/attribution.md). Maintainers should use the
updated [release runbook](../release.md); publication and hosted delivery
remain separate operations.
## Compatibility And Upgrade
The Go consumer minimum remains 1.27.0. Existing configuration, requests,
and release commands require no changes from v0.23.1. Upstream validator
fixes may affect validation results or diagnostic locations for edge cases.
See the [consumer guide](../consumers/pkg-promptkit.md) for current usage.
```sh
go get go.maximumdirect.net/promptkit@v0.23.2
```
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/oss/attribution.md)

36
docs/releases/v0.23.3.md Normal file
View File

@@ -0,0 +1,36 @@
# Promptkit v0.23.3
This patch release corrects timeout attribution and preserves error identities
in the built-in OpenAI-compatible HTTP client.
## Fixed
- Operation budgets expiring during an in-flight HTTP generation request now
match both `ErrOperationTimeout` and `context.DeadlineExceeded`. Terminal
observations report a deadline failure instead of a transport failure.
- HTTP failures carrying custom caller cancellation or deadline causes retain
both the custom cause and the standard context identity. Unrelated transport
failures are not relabeled solely because a context has been canceled.
- Cancellation while reading a successful response body preserves the original
reader error alongside the context identity, without exposing its diagnostic
text through the redacted error message.
Regression coverage exercises `Run` and `RunPrepared`, public error identities,
and attempt and terminal classifications. See the [request and timeout
contracts](../../types.go), [public error GoDoc](../../engine.go), and
[error-handling guide](../consumers/pkg-promptkit.md#handle-errors).
## Compatibility And Upgrade
No configuration or request API migration is required from v0.23.2. Consumers
using `errors.Is` for cancellation or deadline handling now receive the
correct identities on the affected paths; applications continue to own their
status mapping and diagnostic presentation.
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
```sh
go get go.maximumdirect.net/promptkit@v0.23.3
```
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/docs/oss/attribution.md)

83
docs/releases/v0.24.0.md Normal file
View File

@@ -0,0 +1,83 @@
# Promptkit v0.24.0
This minor release adds application-owned configuration loading, prompt-owned
schemas, and a prepared message-content hash. It also enables one repair attempt
by default for enabled output validation. This is a breaking pre-v1 release;
consumers upgrading from v0.23.x should review the migration steps below.
## Added And Changed
- Applications can embed `promptkit.Settings`, compose Promptkit's rules with
`Configure`, load configuration once through ConfigKit, and pass the entire
result to `NewFromConfig`. `NewEngine` remains the standalone convenience
constructor. See the [shared-loading guide](../consumers/pkg-promptkit.md#reuse-an-application-owned-load)
and [offline preparation example](../../examples/go-library/prepare/main.go).
- Prompts can select inline, source-relative, or explicit-root JSON Schemas.
Collections can supply their own schema root. Schema resources are loaded
during preparation and retained by prepared execution. See the
[schema reference](../formats.md#schemas).
- Prepared results include `MessageContentHash` (`message_content_hash` in
JSON), an opaque equality value for ordered message roles and exact text,
independent of session identity, cache hints, and other metadata. The existing
rendered-prompt hash algorithm is unchanged. See the
[prepared-result GoDoc](../../types.go) and
[preparation guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
- Configuration inspection uses a native ConfigKit view focused on `/promptkit`
and adds `HistoricalPaths`. It retains whole-load evidence and supports
explicit queries outside that focus; comparison can include ancestor changes.
See the [inspection guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration).
## Fixes And Efficiency
- Prompt-input inspection and requirements now retain effective metadata supplied
by application defaults, including inherited input requirements.
- Configuration capture avoids expanding inheritance provenance just to determine
input-field presence, and shares immutable backend defaults across profiles
without overrides while preserving defensive copies at public lookups.
- Schema handling avoids copying inactive runtime schemas and unused slice
capacity. Inline schemas are separate from provider extra-parameter copy limits.
See the [schema reference](../formats.md#schemas) and
[request GoDoc](../../types.go).
## Compatibility And Upgrade
The [configuration migration guide](../consumers/configuration-migration.md#move-settings-under-promptkit)
owns the full migration walkthrough. In particular:
- Move complete-document resources and `schema_root` under `promptkit`.
Previous root-level resources are now application data, not Promptkit
resources; IDs and references keep their existing names. `SettingsInput`
mounts wrapperless fragments at `/promptkit`; `CollectionInput` mounts them
at `/promptkit/collections/<namespace>`. Discovery helpers still produce
root-mounted inputs, so adjust fragment destinations explicitly.
- For an application-owned load, select `BundledOpenRouterUnderlay` before
loading if you want the bundled catalog. Bundling remains default-on for
`NewEngine`; it is not implicitly added by `NewFromConfig`.
- Replace `RunRequest.Validation` values of type `*OutputContract` with
`*OutputContractInput`; its `RepairAttempts` field is a pointer. Omitted or
null budgets now resolve to one for `basic`, `json`, and `json_schema`, or
zero for `none`. Set an explicit zero to retain single-pass behavior.
Runtime replacements still replace the whole contract and resolve their
defaults independently. A failed check may now incur another generation call,
latency, and usage, without guaranteeing a valid result. See the
[output contract](../formats.md#output-contract).
- `PromptOutputInspection.RepairAttempts` is now a resolved integer rather than
an optional pointer, and its JSON includes zero. `OutputContract` remains the
resolved contract type used by prepared details. See the
[inspection GoDoc](../../resource_inspection.go) and [contract GoDoc](../../types.go).
- Schema paths require an explicitly configured applicable root; there is no
implicit working-directory root. Review file references and make schemas
self-contained as described in the [schema reference](../formats.md#schemas).
- Promptkit no longer validates resource-version fields. Applications own version
policy and sensitivity rules for their extension fields; see the
[format reference](../formats.md).
Older prepared-result JSON without `message_content_hash` decodes with an empty
value. Callers using positional `PreparedRun` literals may need to adjust them.
ConfigKit is updated to v1.4.0; the Go consumer minimum remains 1.27.0.
```sh
go get go.maximumdirect.net/promptkit@v0.24.0
```
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/docs/oss/attribution.md)

88
docs/releases/v0.25.0.md Normal file
View File

@@ -0,0 +1,88 @@
# Promptkit v0.25.0
This minor release adds invocation-scoped execution observation, provider-specific
reasoning and cache usage, and optional USD cost reporting. Backend definitions
can now select API conventions independently of their names and endpoints.
Consumers upgrading from v0.24.x should review the configuration and accounting
changes below.
## Added And Changed
- `WithExecutionObserver` selects observation through a caller-owned context
across nested, concurrent, coordinated, and multi-engine calls. Request
observers run before contextual observers. Prepared handles retain their
explicit request observer but select contextual observers when executed,
not when prepared. Each execution retains independent accounting; groups
count each run's first terminal once. See the
[observation guide](../consumers/pkg-promptkit.md#observe-an-ordinary-run)
and [offline example](../../examples/go-library/observe/main.go).
- Backend `provider` selects `openai-compatible`, `openrouter`, or `vllm`
conventions. Omitted, null, or blank selects the generic OpenAI-compatible
behavior. The bundled OpenRouter backend explicitly selects `openrouter`.
`InspectBackend` exposes declared provider and `zero_cost` settings without
manufacturing runtime defaults. See the
[backend reference](../formats.md#backend-definitions).
- Usage values, presence indicators, observations, and group summaries include
`ReasoningTokens`. OpenRouter and vLLM decoding includes their respective
reasoning, cache-read, and cache-write fields. Reasoning is a subset of
completion tokens; cache activity does not add to reported token totals.
`ObservedTokenUsage.CacheHitRatio` derives a ratio from complete, consistent
input and cache-read totals. See the
[response contract](../integrations/openai-compatible-chat.md#response-handling).
- `RunResult.CostUSD`, execution events, and group summaries report a USD
subtotal with availability, completeness, and source metadata. OpenRouter
supplies optional `usage.cost`; injected clients can supply
`GenerateResponse.CostUSD`. Explicit backend `zero_cost: true` supplies a
zero fallback when no usable amount is reported. A reported amount, including
zero, takes precedence. See the [cost GoDoc](../../cost.go).
- Authentication diagnostics can use provider defaults: `openrouter_key` for
OpenRouter and the new `vllm_models` adapter for vLLM. The latter probes
base-relative `/models` and reports recognized response facts without
asserting authentication enforcement or inference authorization. See the
[diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
## Compatibility And Upgrade
- For application-defined OpenRouter backends, set `provider: openrouter` to
enable OpenRouter-specific usage and cost decoding and its authentication
default. Backend IDs and endpoint hostnames do not select conventions.
Generic backends retain their existing request and token conventions and
do not decode monetary cost. Unsupported nonblank provider names are
configuration errors; supplied names are trimmed and lowercased.
- OpenRouter cache-write reporting now reads
`usage.prompt_tokens_details.cache_write_tokens`, not the generic top-level
`usage.cache_write_tokens` field. vLLM uses
`usage.prompt_tokens_details.created_cache_tokens`. Missing or unusable new
details remain unknown rather than invalidating otherwise usable content.
Availability depends on the provider and server settings; consult the
[response contract](../integrations/openai-compatible-chat.md#response-handling).
- A supplied authentication object no longer requires `kind`: an omitted or
null kind uses the provider default. Clearing the authentication block also
restores that default. Set `kind: none` to declare authentication testing
inapplicable. The generic provider has no authentication default; explicit
adapters remain supported independently of provider selection. Probes still
run only when explicitly requested. See the
[backend reference](../formats.md#backend-definitions).
- Cost includes all dispatched attempts, including transient retries and
structural repairs, and is available on successful results without an
observer. On operational failure, observations can retain cost even though
no partial result is returned. Existing `RunResult.Usage` continues to sum
successful generation responses; presence-aware observations cover all
dispatched attempts. Do not add terminal totals to attempt totals.
- Unknown cost is distinct from reported or configured zero. No backend is
automatically declared zero-cost, and no token-rate estimation is performed.
Group cost completeness describes included terminals, independently of
terminal coverage. Consumers own presentation and interpretation; complete
reporting does not verify billing.
- Result JSON adds `cost_usd`, and token JSON adds `reasoning_tokens`.
Update strict decoders if necessary. Older result JSON without cost decodes
to unknown cost. Consumers using positional literals for expanded public
structs may need to update them. See the [public types](../../types.go).
The Go consumer minimum remains 1.27.0; dependencies are unchanged from v0.24.0.
```sh
go get go.maximumdirect.net/promptkit@v0.25.0
```
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/docs/oss/attribution.md)

194
engine.go
View File

@@ -4,12 +4,13 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io/fs"
"net/http" "net/http"
"reflect"
"time" "time"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/backend" "go.maximumdirect.net/promptkit/internal/backend"
"go.maximumdirect.net/promptkit/internal/configuration"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
internalengine "go.maximumdirect.net/promptkit/internal/engine" internalengine "go.maximumdirect.net/promptkit/internal/engine"
"go.maximumdirect.net/promptkit/internal/observation" "go.maximumdirect.net/promptkit/internal/observation"
@@ -126,9 +127,6 @@ type ConfigurationSources struct {
type Config struct { type Config struct {
// Sources combines maintained definitions with caller-selected inputs. // Sources combines maintained definitions with caller-selected inputs.
Sources ConfigurationSources Sources ConfigurationSources
// SchemaDir is the root for JSON Schema files. An empty value uses the
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
SchemaDir string
// Timeout is the transport-wide safety cap for the built-in LLM client // Timeout is the transport-wide safety cap for the built-in LLM client
// when HTTPClient is absent or has a non-positive timeout. A zero or negative // when HTTPClient is absent or has a non-positive timeout. A zero or negative
// value selects the 10-minute default. // value selects the 10-minute default.
@@ -148,7 +146,7 @@ type Config struct {
// Option customizes engine construction. // Option customizes engine construction.
// //
// NewEngine applies options in argument order and ignores nil options. Within // NewEngine applies options in argument order and ignores nil options. Within
// each schema-source, model-client, and artifact-reader category, the last // each model-client and artifact-reader category, the last
// non-nil valid option replaces earlier options. An invalid option fails // non-nil valid option replaces earlier options. An invalid option fails
// construction even if a later option would replace it. // construction even if a later option would replace it.
type Option interface { type Option interface {
@@ -163,13 +161,15 @@ func (f optionFunc) apply(options *engineOptions) error {
type engineOptions struct { type engineOptions struct {
internal internalengine.Options internal internalengine.Options
disableBundledOpenRouter bool
} }
// WithBundledOpenRouter enables or disables the built-in OpenRouter underlay. // WithBundledOpenRouter enables or disables the built-in OpenRouter underlay.
// It is enabled by default; when repeated, the last option wins. // It is enabled by default for NewEngine; when repeated, the last option wins.
// It has no effect on an already resolved NewFromConfig handoff.
func WithBundledOpenRouter(enabled bool) Option { func WithBundledOpenRouter(enabled bool) Option {
return optionFunc(func(options *engineOptions) error { return optionFunc(func(options *engineOptions) error {
options.internal.SetBundledOpenRouter(enabled) options.disableBundledOpenRouter = !enabled
return nil return nil
}) })
} }
@@ -207,30 +207,6 @@ func WithArtifactReader(reader ArtifactReader) Option {
}) })
} }
// WithSchemaFS loads JSON Schema documents from fsys under root.
//
// Prompt schema_path values resolve within this source when schema validation
// or structured output is requested. fsys must be non-nil and root must be
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
// replaces Config.SchemaDir and earlier schema-source options.
func WithSchemaFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetSchemaFS(fsys, root)
})
}
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
//
// Prompt schema_path values refer to the file's base name. path must name an
// existing non-directory file when NewEngine applies the option. This option
// replaces Config.SchemaDir and earlier schema-source options. The selected
// schema must be self-contained; external resources are not loaded.
func WithSchemaFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetSchemaFile(path)
})
}
// NewEngine constructs an Engine from configuration and options. // NewEngine constructs an Engine from configuration and options.
// //
// ConfigKit composes caller-selected underlays in their given order, followed // ConfigKit composes caller-selected underlays in their given order, followed
@@ -241,33 +217,135 @@ func WithSchemaFile(path string) Option {
// content. A missing prompt collection is allowed. Schema documents and // content. A missing prompt collection is allowed. Schema documents and
// request artifacts remain runtime resources. // request artifacts remain runtime resources.
// //
// NewEngine returns an error matching ErrInvalidConfig for invalid // NewEngine returns an error matching ErrInvalidConfig through errors.Is for
// configuration or options, preserving a ConfigKit LoadError when loading // invalid configuration or options. The error unwraps to the original
// fails. Each constructed // construction cause, preserving a ConfigKit LoadError when loading fails.
// Engine has independent backend-capacity pools. Construction does not perform // Each constructed Engine has independent backend-capacity pools. Construction
// model requests or require credentials. A negative Config.TransientRetries // does not perform model requests or require credentials. A negative
// is invalid. // Config.TransientRetries is invalid.
func NewEngine(cfg Config, opts ...Option) (*Engine, error) { func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
options, err := applyEngineOptions(opts)
if err != nil {
return nil, err
}
runtime := RuntimeConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: cfg.TransientRetries}
if _, err := domain.ResolveTransientRetries(2, runtime.TransientRetries); err != nil {
return nil, newPublicOperationError(ErrInvalidConfig, err)
}
underlays := append([]configkit.Underlay(nil), cfg.Sources.Underlays...)
if !options.disableBundledOpenRouter {
underlays = append([]configkit.Underlay{BundledOpenRouterUnderlay()}, underlays...)
}
main := configkit.Input{Source: configkit.NoSource()}
if cfg.Sources.Main != nil {
main = *cfg.Sources.Main
}
definition := Configure(configkit.Definition[standaloneConfiguration]{}, func(value *standaloneConfiguration) *Settings { return &value.PromptKit })
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[standaloneConfiguration]{Underlays: underlays, Definition: definition})
if err != nil {
return nil, newPublicOperationError(ErrInvalidConfig, err)
}
return newFromConfig(result, runtime, options)
}
type standaloneConfiguration struct {
PromptKit Settings `yaml:"promptkit"`
Application map[string]any `yaml:",inline"`
}
// Configuration is the completed application-owned configuration evidence
// needed to capture PromptKit resources without loading configuration again.
type Configuration interface {
Value(configkit.Path) (any, bool)
InspectSubtree(configkit.Path) (configkit.Inspection, bool)
Diagnostics() []configkit.Diagnostic
}
// RuntimeConfig selects runtime-only client and retry settings for NewFromConfig.
// Its fields have the same semantics as the corresponding Config fields.
type RuntimeConfig struct {
// Timeout is the built-in client's transport timeout fallback.
Timeout time.Duration
// HTTPClient is cloned for the built-in client and ignored by injected clients.
HTTPClient *http.Client
// TransientRetries selects additional built-in attempts; nil defaults to two.
TransientRetries *int
}
// NewFromConfig captures resources from an application-owned completed load.
// The application must register PromptKit rules during that load, using Configure
// or equivalent explicit composition. Missing or nil selected settings contribute
// no resources; Settings and *Settings are accepted, while other nonnil types,
// nil results, and unavailable inspection views fail with ErrInvalidConfig.
// Construction consumes resolved settings while inspection retains the load's
// frozen evidence. Callers should keep those consistent at handoff. After
// construction, later settings mutations do not alter the engine. This method
// does not load sources, rerun callbacks, or select bundled underlays.
func NewFromConfig(result Configuration, cfg RuntimeConfig, opts ...Option) (*Engine, error) {
options, err := applyEngineOptions(opts)
if err != nil {
return nil, err
}
return newFromConfig(result, cfg, options)
}
func applyEngineOptions(opts []Option) (engineOptions, error) {
var options engineOptions var options engineOptions
for _, opt := range opts { for _, opt := range opts {
if opt == nil { if opt == nil {
continue continue
} }
if err := opt.apply(&options); err != nil { if err := opt.apply(&options); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err) return engineOptions{}, newPublicOperationError(ErrInvalidConfig, err)
} }
} }
return options, nil
}
assembled, err := internalengine.New( func newFromConfig(result Configuration, cfg RuntimeConfig, options engineOptions) (*Engine, error) {
internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)}, if result == nil || isNilConfiguration(result) {
options.internal, return nil, newPublicOperationError(ErrInvalidConfig, errors.New("configuration result is nil"))
) }
if _, err := domain.ResolveTransientRetries(2, cfg.TransientRetries); err != nil {
return nil, newPublicOperationError(ErrInvalidConfig, err)
}
value, available := result.Value(configkit.Keys("promptkit"))
var selected *Settings
if available && value != nil {
switch section := value.(type) {
case Settings:
selected = &section
case *Settings:
selected = section
default:
return nil, newPublicOperationError(ErrInvalidConfig, fmt.Errorf("promptkit section has incompatible type %T", value))
}
}
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
if !ok {
return nil, newPublicOperationError(ErrInvalidConfig, errors.New("configuration inspection is unavailable"))
}
snapshot, err := configuration.Capture(selected, view, result.Diagnostics())
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err) return nil, newPublicOperationError(ErrInvalidConfig, err)
}
assembled, err := internalengine.New(snapshot, internalengine.Config{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)}, options.internal)
if err != nil {
return nil, newPublicOperationError(ErrInvalidConfig, err)
} }
return &Engine{internal: assembled}, nil return &Engine{internal: assembled}, nil
} }
func isNilConfiguration(result Configuration) bool {
value := reflect.ValueOf(result)
switch value.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
return value.IsNil()
default:
return false
}
}
// InspectPrompt resolves one prompt ID without selecting a profile or starting // InspectPrompt resolves one prompt ID without selecting a profile or starting
// execution work. // execution work.
// //
@@ -281,13 +359,13 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
// unrendered messages are returned in definition order. DefaultProfileID is // unrendered messages are returned in definition order. DefaultProfileID is
// declared metadata only and is not // declared metadata only and is not
// resolved. OutputContract is the normalized declared contract, with a JSON // resolved. OutputContract is the normalized declared contract, with a JSON
// Schema path when declared but without loading or compiling that schema. // Schema selection when declared but without loading or compiling schema files.
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for // PromptHash is the same opaque equality value as PreparedRun.PromptHash for
// the selected captured definition; its spelling, length, // the selected captured definition; its spelling, length,
// encoding, algorithm, and security properties are not contracts. // encoding, algorithm, and security properties are not contracts.
// //
// This method does not return parsed templates, source paths, schemas, // This method does not return parsed templates, filesystem capabilities,
// rendered messages, or execution settings. It does not resolve a profile or // schema file contents, rendered messages, or execution settings. It does not resolve a profile or
// credential, read artifacts or schemas, render, validate, admit capacity, // credential, read artifacts or schemas, render, validate, admit capacity,
// contact a provider, or generate model output. The returned PromptInspection // contact a provider, or generate model output. The returned PromptInspection
// and its input slice are caller-owned. Inspection does not reload configuration // and its input slice are caller-owned. Inspection does not reload configuration
@@ -457,7 +535,8 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr
// returns no handle on error. A nil Engine returns an error matching // returns no handle on error. A nil Engine returns an error matching
// ErrInvalidConfig. The effective transient retry budget is frozen privately // ErrInvalidConfig. The effective transient retry budget is frozen privately
// for RunPrepared and is absent from Details. A supplied Observer is retained // for RunPrepared and is absent from Details. A supplied Observer is retained
// privately for RunPrepared; preparation itself emits no events. // privately for RunPrepared; preparation itself emits no events and does not
// retain contextual observers.
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without // RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
// starting a timer or invoking the notification during preparation. // starting a timer or invoking the notification during preparation.
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) { func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
@@ -524,16 +603,24 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
// result on error. // result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
ctx = observation.WithoutExecutionState(ctx) ctx = observation.WithoutExecutionState(ctx)
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := usecase.NewOperationScope(ctx, req.OperationTimeout) scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
defer scope.Close() defer scope.Close()
ctx = scope.Context ctx = scope.Context
if req.Observer == nil { var explicit domain.ExecutionObserver
if req.Observer != nil {
explicit = func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) }
}
observer := observation.Compose(explicit, observation.ObserverFromContext(ctx))
if observer == nil {
result, err := e.run(ctx, req) result, err := e.run(ctx, req)
err = mapPublicError(scope.Finalize(err)) err = mapPublicError(scope.Finalize(err))
scope.Close() scope.Close()
return result, err return result, err
} }
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) }) recorder, idErr := observation.New(observer)
recorder.SetCostAccumulator(cost)
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil { if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
recorder.SetParentRunID(req.ContinueFrom.internal.RunID()) recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
} }
@@ -604,10 +691,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
// content-validation rejection, including repair exhaustion, is returned in // content-validation rejection, including repair exhaustion, is returned in
// RunResult, not as an operational // RunResult, not as an operational
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An // error. RunPrepared uses the retry choice frozen during PrepareExecution. An
// operational error returns no partial RunResult. A retained Observer begins // operational error returns no partial RunResult. The retained Observer and
// only after a successful atomic claim, with identities from the frozen // execution-context observers begin only after a successful atomic claim, with
// snapshot, and receives a terminal event on normal return after public error // identities from the frozen snapshot, and receive a terminal event on normal
// mapping. Invalid, foreign, discarded, or reused handles emit no events. // return after public error mapping. Invalid, foreign, discarded, or reused
// handles emit no events.
// Callbacks run outside the handle and capacity locks; panics propagate and // Callbacks run outside the handle and capacity locks; panics propagate and
// do not promise a terminal event. // do not promise a terminal event.
// The retained RunRequest.OperationTimeout starts after claim; the retained // The retained RunRequest.OperationTimeout starts after claim; the retained

View File

@@ -21,6 +21,10 @@ type publicOperationError struct {
message string message string
} }
func newPublicOperationError(owner, cause error) *publicOperationError {
return &publicOperationError{owner: owner, cause: cause, message: fmt.Sprintf("%v: %v", owner, cause)}
}
func (e *publicOperationError) Error() string { return e.message } func (e *publicOperationError) Error() string { return e.message }
func (e *publicOperationError) Unwrap() error { return e.cause } func (e *publicOperationError) Unwrap() error { return e.cause }
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) } func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
@@ -44,7 +48,7 @@ func mapPublicError(err error) error {
} }
// The projection already retains ordinary causes; adding the original // The projection already retains ordinary causes; adding the original
// again would duplicate its entire subtree at each enclosing operation. // again would duplicate its entire subtree at each enclosing operation.
return &publicOperationError{owner: owner, cause: mapped, message: fmt.Sprintf("%v: %v", owner, mapped)} return newPublicOperationError(owner, mapped)
} }
} }
return mapUnownedPublicError(err) return mapUnownedPublicError(err)

View File

@@ -10,6 +10,25 @@ import (
"go.maximumdirect.net/promptkit/internal/usecase" "go.maximumdirect.net/promptkit/internal/usecase"
) )
func TestConstructionErrorPreservesJoinedCause(t *testing.T) {
operational := errors.New("independent operational failure")
cause := errors.Join(operational, context.Canceled)
engine, err := NewEngine(Config{}, optionFunc(func(*engineOptions) error {
return cause
}))
if engine != nil || err == nil {
t.Fatalf("construction = (%#v, %v), want failure", engine, err)
}
if unwrapped := errors.Unwrap(err); unwrapped != cause {
t.Fatalf("construction cause = %v, want the unchanged joined cause", unwrapped)
}
for _, target := range []error{ErrInvalidConfig, operational, context.Canceled} {
if !errors.Is(err, target) {
t.Errorf("construction error = %v, want identity %v", err, target)
}
}
}
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) { func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled) internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)

View File

@@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error)
func modelResponse(content string, promptTokens, completionTokens int) *http.Response { func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
body, _ := json.Marshal(map[string]any{ body, _ := json.Marshal(map[string]any{
"choices": []any{map[string]any{"message": map[string]string{"content": content}}}, "choices": []any{map[string]any{"message": map[string]string{"content": content}}},
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens}, "usage": map[string]any{
"prompt_tokens": promptTokens, "completion_tokens": completionTokens,
"prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1},
"completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01,
},
}) })
return httpResponse(http.StatusOK, string(body)) return httpResponse(http.StatusOK, string(body))
} }
@@ -80,6 +84,10 @@ type summary struct {
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"` ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"` GroupMembers int `json:"group_members"`
GroupAttempts uint64 `json:"group_attempts"` GroupAttempts uint64 `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"` PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"` PreparedParentMatches bool `json:"prepared_parent_matches"`
} }
@@ -90,30 +98,38 @@ func run() (summary, error) {
if err != nil { if err != nil {
return summary{}, err return summary{}, err
} }
for i := range inputs {
inputs[i].Destination = configkit.Keys("promptkit")
}
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model" backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
filesystem := configkit.NewFilesystem("example schemas", assets)
backend := promptkit.LocalBackend("https://example.invalid/v1", 1) backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
backend.ID = backendID backend.ID = backendID
provider, zeroCost := "openrouter", true
backend.Provider, backend.ZeroCost = &provider, &zeroCost
inputs = append(inputs, inputs = append(inputs,
configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{ configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
backend, backend,
})}, })),
configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{ promptkit.SettingsInput(promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel}, {ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel}, {ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
})}, })),
) )
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}, Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
HTTPClient: &http.Client{Transport: transport}, HTTPClient: &http.Client{Transport: transport},
}, promptkit.WithBundledOpenRouter(false), promptkit.WithSchemaFS(assets, "schemas")) // Keep the example's configuration self-contained. }, promptkit.WithBundledOpenRouter(false)) // Keep the example's configuration self-contained.
if err != nil { if err != nil {
return summary{}, err return summary{}, err
} }
var proposalLog eventLog var proposalLog eventLog
var executions promptkit.ExecutionGroup var executions promptkit.ExecutionGroup
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{ ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe)
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe), proposal, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record,
}) })
if err != nil { if err != nil {
return summary{}, err return summary{}, err
@@ -128,8 +144,8 @@ func run() (summary, error) {
var assessmentLog eventLog var assessmentLog eventLog
var preparedDetails promptkit.PreparedRun var preparedDetails promptkit.PreparedRun
decision := "rejected" decision := "rejected"
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{ coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe, Start: proposal.Exchange, AdditionalTurns: 1,
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
if turn.Result == nil { if turn.Result == nil {
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
@@ -137,7 +153,7 @@ func run() (summary, error) {
if !turn.Result.Validation.IsValid { if !turn.Result.Validation.IsValid {
return nil, fmt.Errorf("follow-up failed structural validation") return nil, fmt.Errorf("follow-up failed structural validation")
} }
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)}) assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -176,6 +192,8 @@ func run() (summary, error) {
return summary{}, err return summary{}, err
} }
group := executions.Snapshot() group := executions.Snapshot()
_, ratioAvailable := group.Usage.CacheHitRatio()
assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio()
return summary{ return summary{
ProposalAttempts: proposalTerminal.Attempts, ProposalAttempts: proposalTerminal.Attempts,
ProposalTransientRetries: proposalTerminal.TransientRetries, ProposalTransientRetries: proposalTerminal.TransientRetries,
@@ -187,6 +205,10 @@ func run() (summary, error) {
ExhaustedOutcome: exhausted.Outcome, ExhaustedOutcome: exhausted.Outcome,
GroupMembers: len(group.Members), GroupMembers: len(group.Members),
GroupAttempts: group.Attempts, GroupAttempts: group.Attempts,
GroupCost: group.CostUSD,
GroupReasoning: group.Usage.ReasoningTokens,
GroupCacheRatioAvailable: ratioAvailable,
AssessmentCacheRatio: assessmentRatio,
PreparedMessages: len(preparedDetails.Messages), PreparedMessages: len(preparedDetails.Messages),
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID, PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
}, nil }, nil

View File

@@ -3,7 +3,10 @@ package main
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"math"
"testing" "testing"
"go.maximumdirect.net/promptkit"
) )
func TestObserveExample(t *testing.T) { func TestObserveExample(t *testing.T) {
@@ -22,6 +25,10 @@ func TestObserveExample(t *testing.T) {
ExhaustedOutcome string `json:"exhausted_outcome"` ExhaustedOutcome string `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"` GroupMembers int `json:"group_members"`
GroupAttempts int `json:"group_attempts"` GroupAttempts int `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"` PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"` PreparedParentMatches bool `json:"prepared_parent_matches"`
} }
@@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) {
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches { got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
t.Fatalf("unexpected observation summary: %+v", got) t.Fatalf("unexpected observation summary: %+v", got)
} }
if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 {
t.Fatalf("unexpected usage summary: %+v", got)
}
} }

View File

@@ -14,4 +14,12 @@ prompts:
output: output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: assessment.schema.json repair_attempts: 0
schema:
$schema: https://json-schema.org/draft/2020-12/schema
type: object
required: [recommendation, confidence]
properties:
recommendation: {type: string}
confidence: {type: number}
additionalProperties: false

View File

@@ -14,4 +14,5 @@ prompts:
output: output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
repair_attempts: 0
schema_path: proposal.schema.json schema_path: proposal.schema.json

View File

@@ -1,10 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["recommendation", "confidence"],
"properties": {
"recommendation": {"type": "string"},
"confidence": {"type": "number"}
},
"additionalProperties": false
}

View File

@@ -1,4 +1,3 @@
version: "2"
config: config:
imports: imports:
- path: fragments/resources.yaml - path: fragments/resources.yaml

View File

@@ -17,6 +17,7 @@ type summary struct {
SelectedProfile string `json:"selected_profile"` SelectedProfile string `json:"selected_profile"`
Model string `json:"model"` Model string `json:"model"`
MessageCount int `json:"message_count"` MessageCount int `json:"message_count"`
MessageContentHash string `json:"message_content_hash"`
ProfileTimeout int `json:"profile_timeout_seconds"` ProfileTimeout int `json:"profile_timeout_seconds"`
Namespaces []string `json:"namespaces"` Namespaces []string `json:"namespaces"`
Collections []collectionSummary `json:"collections"` Collections []collectionSummary `json:"collections"`
@@ -29,6 +30,10 @@ type collectionSummary struct {
ReleaseVersion *string `json:"release_version"` ReleaseVersion *string `json:"release_version"`
} }
type applicationSettings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
func main() { func main() {
if err := run(context.Background(), "examples/go-library/prepare", os.Stdout); err != nil { if err := run(context.Background(), "examples/go-library/prepare", os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
@@ -58,12 +63,18 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
if err != nil { if err != nil {
return err return err
} }
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ definition := promptkit.Configure(configkit.Definition[applicationSettings]{}, func(value *applicationSettings) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[applicationSettings]{
Definition: definition,
Underlays: []configkit.Underlay{ Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{support, review}}, {Inputs: []configkit.Input{support, review}},
{Inputs: []configkit.Input{override}}, {Inputs: []configkit.Input{override}},
}, },
}}, promptkit.WithBundledOpenRouter(false)) // Keep this example's inventory caller-owned. })
if err != nil {
return err
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
if err != nil { if err != nil {
return err return err
} }
@@ -77,7 +88,7 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
if err != nil { if err != nil {
return err return err
} }
if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found { if _, found := view.Explain(configkit.Keys("promptkit", "collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
return fmt.Errorf("collection profile override was not found") return fmt.Errorf("collection profile override was not found")
} }
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{ requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
@@ -156,6 +167,7 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
SelectedProfile: prepared.SelectedProfileID, SelectedProfile: prepared.SelectedProfileID,
Model: prepared.EffectiveModelParams.Model, Model: prepared.EffectiveModelParams.Model,
MessageCount: len(prepared.Messages), MessageCount: len(prepared.Messages),
MessageContentHash: prepared.MessageContentHash,
ProfileTimeout: *profile.TimeoutSeconds, ProfileTimeout: *profile.TimeoutSeconds,
Namespaces: namespaces, Namespaces: namespaces,
Collections: collectionSummaries, Collections: collectionSummaries,

View File

@@ -17,6 +17,7 @@ func TestPrepareExample(t *testing.T) {
Profile string `json:"selected_profile"` Profile string `json:"selected_profile"`
Model string `json:"model"` Model string `json:"model"`
MessageCount int `json:"message_count"` MessageCount int `json:"message_count"`
MessageContentHash string `json:"message_content_hash"`
Timeout int `json:"profile_timeout_seconds"` Timeout int `json:"profile_timeout_seconds"`
Namespaces []string `json:"namespaces"` Namespaces []string `json:"namespaces"`
Collections []map[string]json.RawMessage `json:"collections"` Collections []map[string]json.RawMessage `json:"collections"`
@@ -27,7 +28,7 @@ func TestPrepareExample(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if got.PromptID != "support/prepare" || got.Profile != "support/fast" || got.Model != "offline-model" || if got.PromptID != "support/prepare" || got.Profile != "support/fast" || got.Model != "offline-model" ||
got.MessageCount <= 0 || got.Timeout != 30 || !reflect.DeepEqual(got.Namespaces, []string{"review", "support"}) || got.MessageCount <= 0 || got.MessageContentHash == "" || got.Timeout != 30 || !reflect.DeepEqual(got.Namespaces, []string{"review", "support"}) ||
!reflect.DeepEqual(got.ScopedPrompts, []string{"support/prepare"}) || !reflect.DeepEqual(got.ReviewPrompts, []string{"review/check"}) || len(got.Collections) != 2 { !reflect.DeepEqual(got.ScopedPrompts, []string{"support/prepare"}) || !reflect.DeepEqual(got.ReviewPrompts, []string{"review/check"}) || len(got.Collections) != 2 {
t.Fatalf("unexpected preparation summary: %+v", got) t.Fatalf("unexpected preparation summary: %+v", got)
} }

View File

@@ -50,10 +50,10 @@ func main() {
func run(ctx context.Context, exampleDir string, out io.Writer) error { func run(ctx context.Context, exampleDir string, out io.Writer) error {
endpoint, model := "https://example.invalid/v1", "offline-model" endpoint, model := "https://example.invalid/v1", "offline-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
{ID: "offline-example", Endpoint: &endpoint, Model: &model}, {ID: "offline-example", Endpoint: &endpoint, Model: &model},
})}}}}, }))}}},
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml"))}, Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml")), Destination: configkit.Keys("promptkit")},
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(deterministicClient{})) // Use only this example's offline configuration. }}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(deterministicClient{})) // Use only this example's offline configuration.
if err != nil { if err != nil {
return err return err

View File

@@ -52,7 +52,7 @@ func TestExchangeCapturesInvalidFinalOutput(t *testing.T) {
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}} client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}}
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client)) engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true,
Validation: &promptkit.OutputContract{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: 1}}) Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}})
if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil { if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil {
t.Fatalf("invalid result = %#v, %v", result, err) t.Fatalf("invalid result = %#v, %v", result, err)
} }

View File

@@ -82,7 +82,7 @@ func TestOperationBudgetInterruptsRetryAndRepair(t *testing.T) {
req := simpleGenerationRequest() req := simpleGenerationRequest()
req.OperationTimeout = time.Second req.OperationTimeout = time.Second
if repair { if repair {
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1} req.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
} }
_, err = engine.Run(context.Background(), req) _, err = engine.Run(context.Background(), req)
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {

View File

@@ -5,7 +5,7 @@ import (
"go.maximumdirect.net/promptkit/internal/observation" "go.maximumdirect.net/promptkit/internal/observation"
) )
// ExecutionGroup collects facts from explicitly selected execution events. // ExecutionGroup collects facts from selected execution events.
// Its zero value is ready for concurrent use. Do not copy it after first use. // Its zero value is ready for concurrent use. Do not copy it after first use.
type ExecutionGroup struct{ group observation.Group } type ExecutionGroup struct{ group observation.Group }
@@ -21,7 +21,8 @@ type ExecutionGroupMember struct {
// member has a terminal, independently of usage completeness. An empty group // member has a terminal, independently of usage completeness. An empty group
// has complete coverage and known complete zero usage; a group with members // has complete coverage and known complete zero usage; a group with members
// but no terminals has unknown, incomplete usage. Overlapping groups are not // but no terminals has unknown, incomplete usage. Overlapping groups are not
// generally additive. // generally additive. CostUSD covers included terminals independently of
// terminal coverage, using the same known, complete, and source merge rules.
type ExecutionGroupSummary struct { type ExecutionGroupSummary struct {
Members []ExecutionGroupMember `json:"members"` Members []ExecutionGroupMember `json:"members"`
TerminalCoverageComplete bool `json:"terminal_coverage_complete"` TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
@@ -29,6 +30,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64 `json:"transient_retries"` TransientRetries uint64 `json:"transient_retries"`
RepairCalls uint64 `json:"repair_calls"` RepairCalls uint64 `json:"repair_calls"`
Usage ObservedTokenUsage `json:"usage"` Usage ObservedTokenUsage `json:"usage"`
CostUSD ObservedCost `json:"cost_usd"`
} }
// Observe enrolls an event's run, counting its first terminal event only. // Observe enrolls an event's run, counting its first terminal event only.
@@ -36,8 +38,8 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) {
count := func(c ObservedTokenCount) domain.ObservedTokenCount { count := func(c ObservedTokenCount) domain.ObservedTokenCount {
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete} return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
} }
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, CostUSD: toDomainCost(event.CostUSD),
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}}) Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
} }
// Snapshot returns an internally consistent, caller-owned summary. Coverage // Snapshot returns an internally consistent, caller-owned summary. Coverage
@@ -51,6 +53,6 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
for i, member := range value.Members { for i, member := range value.Members {
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved} members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
} }
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, CostUSD: fromDomainCost(value.CostUSD),
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}} Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
} }

View File

@@ -37,7 +37,7 @@ func TestExecutionGroupExplicitMembershipAndTerminalTotals(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
validation := &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1} validation := &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
callbackCount := 0 callbackCount := 0
var branchID, validatorID string var branchID, validatorID string
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: promptkit.ComposeObservers(all.Observe, selected.Observe), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: promptkit.ComposeObservers(all.Observe, selected.Observe), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {

View File

@@ -3,6 +3,9 @@ package promptkit_test
import ( import (
"context" "context"
"errors" "errors"
"net/http"
"net/http/httptest"
"net/url"
"testing" "testing"
"testing/synctest" "testing/synctest"
"time" "time"
@@ -10,6 +13,70 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestBuiltInGenerationOperationTimeout(t *testing.T) {
for _, split := range []bool{false, true} {
t.Run(map[bool]string{false: "run", true: "prepared"}[split], func(t *testing.T) {
started := make(chan struct{}, 1)
release := make(chan struct{})
provider := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
select {
case started <- struct{}{}:
default:
}
<-release
}))
defer provider.Close()
defer close(release)
config := simpleConfig("message")
zero := 0
config.TransientRetries = &zero
config.HTTPClient = provider.Client()
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatal(err)
}
var terminal promptkit.ExecutionEvent
req := promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second,
Execution: &promptkit.ExecutionTargetOverride{Endpoint: provider.URL + "/v1", TimeoutSeconds: &zero},
Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
terminal = event
}
},
}
parent := context.Background()
var result *promptkit.RunResult
if split {
handle, prepareErr := engine.PrepareExecution(parent, req)
if prepareErr != nil {
t.Fatal(prepareErr)
}
result, err = engine.RunPrepared(parent, handle)
} else {
result, err = engine.Run(parent, req)
}
select {
case <-started:
default:
t.Fatal("operation expired before provider received the request")
}
for _, target := range []error{promptkit.ErrOperationTimeout, context.DeadlineExceeded, promptkit.ErrLLMGenerate, promptkit.ErrTransport} {
if !errors.Is(err, target) {
t.Errorf("error=%v, want identity %v", err, target)
}
}
var transportErr *url.Error
if !errors.As(err, &transportErr) {
t.Errorf("error=%v, want retained HTTP transport cause", err)
}
if result != nil || parent.Err() != nil || terminal.Failure != promptkit.ExecutionFailureDeadline {
t.Fatalf("result=%v parent=%v terminal=%+v", result, parent.Err(), terminal)
}
})
}
}
func TestRunOperationTimeoutAttribution(t *testing.T) { func TestRunOperationTimeoutAttribution(t *testing.T) {
synctest.Test(t, func(t *testing.T) { synctest.Test(t, func(t *testing.T) {
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {

View File

@@ -0,0 +1,204 @@
package promptkit_test
import (
"bytes"
"errors"
"strings"
"sync/atomic"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type focusedInspectionRoot struct {
Sibling string `yaml:"sibling"`
PromptKit *promptkit.Settings `yaml:"promptkit"`
}
func TestFocusedInspectionReturnsDetachedDiagnostics(t *testing.T) {
var resolves, validates atomic.Int64
definition := integrationDefinition(&resolves, &validates)
result, err := configkit.Load(configkit.Bytes("diagnostics", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
first := view.Diagnostics()
if len(first) != 1 || first[0].Primary == nil {
t.Fatalf("diagnostics = %+v", first)
}
original := first[0].Primary.SourceID
first[0].Primary.SourceID = "changed"
again := view.Diagnostics()
if again[0].Primary.SourceID != original {
t.Fatalf("diagnostics alias caller mutation: %+v", again)
}
}
func focusedInspection(t *testing.T, source string, sensitive func(configkit.Path) bool) (promptkit.ConfigurationInspection, *configkit.Result[focusedInspectionRoot]) {
t.Helper()
definition := promptkit.Configure(configkit.Definition[focusedInspectionRoot]{Sensitive: sensitive}, func(value *focusedInspectionRoot) *promptkit.Settings { return value.PromptKit })
result, err := configkit.Load(configkit.Bytes("focused", []byte(source)), configkit.Options[focusedInspectionRoot]{Definition: definition})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
return view, result
}
func TestFocusedInspectionRetainsNativeEvidence(t *testing.T) {
view, result := focusedInspection(t, "sibling: private\npromptkit:\n app_secret: hidden\n app_value: original\n", func(path configkit.Path) bool {
return path.String() == "/sibling" || path.String() == "/promptkit/app_secret"
})
result.Config.PromptKit.Application["app_value"] = "mutated"
var rendered bytes.Buffer
if err := view.WriteEffective(&rendered); err != nil || !strings.Contains(rendered.String(), "original") || strings.Contains(rendered.String(), "hidden") || strings.Contains(rendered.String(), "mutated") || strings.Contains(rendered.String(), "sibling") {
t.Fatalf("focused effective = %q, %v", rendered.String(), err)
}
if _, ok := view.Metadata().Lookup(configkit.Keys("sibling")); !ok {
t.Fatal("unfiltered metadata omitted sibling")
}
sibling, ok := view.Explain(configkit.Keys("sibling"))
if !ok || !sibling.Effective.Redacted {
t.Fatalf("outside-focus explanation = %+v, %t", sibling, ok)
}
if _, ok := view.Explain(configkit.Keys("promptkit")); !ok {
t.Fatal("focus explanation missing")
}
if _, ok := view.Explain(configkit.Path{}); !ok {
t.Fatal("whole-application explanation missing")
}
if len(view.Sources()) == 0 {
t.Fatal("native source catalog missing")
}
for _, path := range view.HistoricalPaths() {
if path.String() != "/promptkit" && !strings.HasPrefix(path.String(), "/promptkit/") {
t.Fatalf("out of scope history: %v", path)
}
}
if len(view.HistoricalPaths()) == 0 {
t.Fatal("focused history empty")
}
}
func TestFocusedInspectionHonorsAncestorSensitivityAndHistory(t *testing.T) {
redacted, _ := focusedInspection(t, "promptkit: {app_value: secret-value}\n", func(path configkit.Path) bool { return path.String() == "/promptkit" })
var rendered bytes.Buffer
if err := redacted.WriteEffective(&rendered); err != nil || strings.Contains(rendered.String(), "secret-value") || !strings.Contains(rendered.String(), "REDACTED") {
t.Fatalf("ancestor redaction = %q, %v", rendered.String(), err)
}
definition := promptkit.Configure(configkit.Definition[focusedInspectionRoot]{}, func(value *focusedInspectionRoot) *promptkit.Settings { return value.PromptKit })
lower := configkit.Input{Source: configkit.Bytes("lower", []byte("promptkit: {historical: old}\n"))}
main := configkit.Input{Source: configkit.Bytes("main", []byte("promptkit: null\n"))}
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[focusedInspectionRoot]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{lower}}}})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
path := configkit.Keys("promptkit", "historical")
if len(view.Metadata().History(path)) == 0 {
t.Fatal("historical-only field has no native history")
}
found := false
for _, item := range view.HistoricalPaths() {
if item.String() == path.String() {
found = true
}
}
if !found {
t.Fatalf("historical path missing: %+v", view.HistoricalPaths())
}
}
func TestFocusedInspectionAbsenceNullAndCrossTypeDiff(t *testing.T) {
type noSection struct {
Sibling string `yaml:"sibling"`
}
missing, err := configkit.Load[noSection](configkit.Bytes("absent", []byte("sibling: before\n")))
if err != nil {
t.Fatal(err)
}
missingEngine, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
absent, err := missingEngine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
if err := absent.WriteEffective(&output); err == nil || output.Len() != 0 {
t.Fatalf("absent output = %q, %v", output.String(), err)
}
nullView, _ := focusedInspection(t, "sibling: before\npromptkit: null\n", nil)
if err := nullView.WriteEffective(&output); err != nil || strings.TrimSpace(output.String()) != "null" {
t.Fatalf("null output = %q, %v", output.String(), err)
}
differences, err := absent.Diff(nullView)
if err != nil || len(differences) == 0 {
t.Fatalf("absent/null diff = %+v, %v", differences, err)
}
changed, _ := focusedInspection(t, "sibling: before\npromptkit: {label: one}\n", nil)
if differences, err := absent.Diff(changed); err != nil || len(differences) == 0 {
t.Fatalf("added scope diff = %+v, %v", differences, err)
}
if differences, err := changed.Diff(absent); err != nil || len(differences) == 0 {
t.Fatalf("removed scope diff = %+v, %v", differences, err)
}
otherSibling, _ := focusedInspection(t, "sibling: after\npromptkit: {label: one}\n", nil)
differences, err = changed.Diff(otherSibling)
if err != nil || len(differences) != 0 {
t.Fatalf("sibling-only diff = %+v, %v", differences, err)
}
type differentRoot struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
Extra bool `yaml:"extra"`
}
definition := promptkit.Configure(configkit.Definition[differentRoot]{}, func(value *differentRoot) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.Load(configkit.Bytes("other", []byte("promptkit: {label: two}\n")), configkit.Options[differentRoot]{Definition: definition})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
if err != nil {
t.Fatal(err)
}
other, err := engine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
differences, err = changed.Diff(other)
if err != nil || len(differences) == 0 {
t.Fatalf("cross-root diff = %+v, %v", differences, err)
}
var zero promptkit.ConfigurationInspection
if _, err := zero.Diff(changed); !errors.Is(err, promptkit.ErrInvalidConfig) || zero.HistoricalPaths() != nil {
t.Fatalf("zero view: %v", err)
}
writeErr := errors.New("writer failure")
if err := changed.WriteEffective(failingInspectionWriter{err: writeErr}); !errors.Is(err, writeErr) {
t.Fatalf("writer error = %v", err)
}
}

View File

@@ -38,8 +38,8 @@ func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
endpoint, model := "https://example.invalid/v1", "offline-model" endpoint, model := "https://example.invalid/v1", "offline-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{ Underlays: []configkit.Underlay{
{Inputs: inputs}, {Inputs: mountSettingsInputs(inputs)},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{ {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
{ID: "offline", Endpoint: &endpoint, Model: &model}, {ID: "offline", Endpoint: &endpoint, Model: &model},
})}}}, })}}},
}, },
@@ -144,10 +144,10 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
endpoint, model := "https://example.invalid/v1", "offline-model" endpoint, model := "https://example.invalid/v1", "offline-model"
schemas := fstest.MapFS{} schemas := fstest.MapFS{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
{ID: "offline", Endpoint: &endpoint, Model: &model}, {ID: "offline", Endpoint: &endpoint, Model: &model},
})}}}}, })}}}},
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{ Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("structured prompt", map[string]any{
"prompts": []any{map[string]any{"id": "structured", "prompts": []any{map[string]any{"id": "structured",
"default_profile": "offline", "default_profile": "offline",
"inputs": []any{map[string]any{"name": "note", "required": true}}, "inputs": []any{map[string]any{"name": "note", "required": true}},
@@ -155,7 +155,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
}}, }},
})}, })},
}}, promptkit.WithSchemaFS(schemas, ".")) }})
if err != nil { if err != nil {
t.Fatalf("construct engine before request resources exist: %v", err) t.Fatalf("construct engine before request resources exist: %v", err)
} }

View File

@@ -79,10 +79,10 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
} }
req := simpleGenerationRequest() req := simpleGenerationRequest()
req.CaptureExchange = true req.CaptureExchange = true
req.Validation = &promptkit.OutputContract{ req.Validation = &promptkit.OutputContractInput{
Format: promptkit.FormatJSON, Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSON, ValidationMode: promptkit.ValidationJSON,
RepairAttempts: 1, RepairAttempts: new(1),
} }
result, err := engine.Run(context.Background(), req) result, err := engine.Run(context.Background(), req)

14
go.mod
View File

@@ -3,16 +3,16 @@ module go.maximumdirect.net/promptkit
go 1.27.0 go 1.27.0
require ( require (
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
go.maximumdirect.net/configkit v1.2.0 go.maximumdirect.net/configkit v1.4.0
go.yaml.in/yaml/v4 v4.0.0-rc.6 go.yaml.in/yaml/v4 v4.0.0
) )
require ( require (
github.com/yuin/goldmark/v2 v2.1.6 // indirect github.com/yuin/goldmark/v2 v2.1.16 // indirect
go.maximumdirect.net/releasekit v0.9.0 // indirect go.maximumdirect.net/releasekit v0.10.0 // indirect
golang.org/x/mod v0.40.0 // indirect golang.org/x/mod v0.43.0 // indirect
golang.org/x/text v0.14.0 // indirect golang.org/x/text v0.44.0 // indirect
) )
tool go.maximumdirect.net/releasekit/cmd/releasekit tool go.maximumdirect.net/releasekit/cmd/releasekit

30
go.sum
View File

@@ -1,16 +1,18 @@
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE= github.com/yuin/goldmark/v2 v2.1.16 h1:MXvCVo+Cb2SVJ5cwJUzXetDjI7HECRJCkcqylEQtcJo=
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ= github.com/yuin/goldmark/v2 v2.1.16/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
go.maximumdirect.net/configkit v1.2.0 h1:c4tDJ6Xeh/Jg90UZplLxsnf500lvrZBVj0E3i1ysir8= go.maximumdirect.net/configkit v1.3.0 h1:X3LZnNJL6T2C6DyUugcvIvHMXX6qhtAF+fQEUBeAwuw=
go.maximumdirect.net/configkit v1.2.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA= go.maximumdirect.net/configkit v1.3.0/go.mod h1:ewu6blSokUDCMovy8JBXAdXRikGmlHXAQpFghnPWRfs=
go.maximumdirect.net/releasekit v0.9.0 h1:sK+Aoj6gS6ATU/1reocjwBekp4YETYENwkcX5ti6mrE= go.maximumdirect.net/configkit v1.4.0 h1:TfOupZbQ23/ZMs+xt5lmyukCF11ekqKep8KbmFQD63o=
go.maximumdirect.net/releasekit v0.9.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44= go.maximumdirect.net/configkit v1.4.0/go.mod h1:ewu6blSokUDCMovy8JBXAdXRikGmlHXAQpFghnPWRfs=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.maximumdirect.net/releasekit v0.10.0 h1:3GOgaIigXTkvYaRkntWZASE//2NTR1+IhkoKmt64aJo=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go.maximumdirect.net/releasekit v0.10.0/go.mod h1:cT5NZNEcf7SKDx4MPG40YNK9V4bNyIxcLLFuNlbGQ90=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= go.yaml.in/yaml/v4 v4.0.0 h1:/LG1pNO84Z4l603j/xMn69CcS5OYhMQ/f7l2LoQUGuw=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= go.yaml.in/yaml/v4 v4.0.0/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/mod v0.43.0 h1:MFuq5J6s35W9D3IjkwyWc9vHh0nENOrt5K8Iof4ofmg=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/mod v0.43.0/go.mod h1:4ec1JnuyBz5+FIVhg86UsA39YG8BpnrZdJYSNHmGzHY=
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=

View File

@@ -13,7 +13,7 @@ import (
const benchmarkDefinitionCount = 128 const benchmarkDefinitionCount = 128
var ( var (
constructionResult *loaded constructionResult *testLoaded
selectedPrompt *domain.PromptDefinition selectedPrompt *domain.PromptDefinition
selectedPromptHash string selectedPromptHash string
selectedPromptInspection *domain.PromptInspection selectedPromptInspection *domain.PromptInspection

View File

@@ -12,5 +12,5 @@ var bundledFiles embed.FS
// BundledOpenRouterUnderlay returns the built-in OpenRouter configuration source. // BundledOpenRouterUnderlay returns the built-in OpenRouter configuration source.
func BundledOpenRouterUnderlay() configkit.Underlay { func BundledOpenRouterUnderlay() configkit.Underlay {
filesystem := configkit.NewFilesystem("Promptkit bundled OpenRouter", bundledFiles) filesystem := configkit.NewFilesystem("Promptkit bundled OpenRouter", bundledFiles)
return configkit.Underlay{Inputs: []configkit.Input{{Source: filesystem.File("bundled/openrouter.yaml")}}} return configkit.Underlay{Inputs: []configkit.Input{{Source: filesystem.File("bundled/openrouter.yaml"), Destination: configkit.Keys("promptkit")}}}
} }

View File

@@ -1,5 +1,6 @@
backends: backends:
- id: openrouter - id: openrouter
provider: openrouter
endpoint: https://openrouter.ai/api/v1 endpoint: https://openrouter.ai/api/v1
api_key_env: OPENROUTER_API_KEY api_key_env: OPENROUTER_API_KEY
smoke_tests: smoke_tests:

View File

@@ -12,12 +12,12 @@ import (
) )
func TestBundledOpenRouterUnderlay(t *testing.T) { func TestBundledOpenRouterUnderlay(t *testing.T) {
state, err := Load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{BundledOpenRouterUnderlay()}) state, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{BundledOpenRouterUnderlay()})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
backend, err := state.GetBackend("openrouter") backend, err := state.GetBackend("openrouter")
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" { if err != nil || backend.Provider != "openrouter" || backend.ZeroCost || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
t.Fatalf("backend = %+v, %v", backend, err) t.Fatalf("backend = %+v, %v", backend, err)
} }
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 { if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
@@ -40,6 +40,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
t.Fatalf("profile %s = %+v, %v", id, profile, err) t.Fatalf("profile %s = %+v, %v", id, profile, err)
} }
want := defaults.ExecutionTargetDefault() want := defaults.ExecutionTargetDefault()
want.Provider = "openrouter"
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY" want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
if tier, ok := strings.CutPrefix(id, "auto-"); ok { if tier, ok := strings.CutPrefix(id, "auto-"); ok {
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}} want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
@@ -48,7 +49,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want) t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want)
} }
} }
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") { if !strings.Contains(fmt.Sprint(state.inspection.Sources()), "Promptkit bundled OpenRouter") {
t.Fatalf("source evidence = %+v", state.result.Sources()) t.Fatalf("source evidence = %+v", state.inspection.Sources())
} }
} }

View File

@@ -0,0 +1,151 @@
package capturetest
import (
"context"
"io/fs"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"go.maximumdirect.net/promptkit/internal/configuration"
)
type application struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
Other string `yaml:"other"`
}
func TestCaptureUsesEffectiveSourceContexts(t *testing.T) {
baseFiles := fstest.MapFS{
"catalog/base.yaml": {Data: []byte("promptkit:\n prompts:\n - id: base\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")},
"catalog/original.txt": {Data: []byte("original")},
"catalog/schema.json": {Data: []byte(`{"type":"object"}`)},
}
overrideFiles := fstest.MapFS{
"operator/main.yaml": {Data: []byte("promptkit:\n prompts:\n - id: child\n extends: {id: base}\n messages: [{role: user, content_file: replacement.txt}]\n")},
"operator/replacement.txt": {Data: []byte("replacement")},
}
base := configkit.NewFilesystem("base", baseFiles)
override := configkit.NewFilesystem("operator", overrideFiles)
definition := promptkit.Configure(configkit.Definition[application]{}, func(value *application) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.LoadLayered(configkit.Input{Source: override.File("operator/main.yaml"), Filesystem: &configkit.FilesystemContext{FS: &override, Root: "."}}, configkit.LayeredOptions[application]{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: base.File("catalog/base.yaml"), Filesystem: &configkit.FilesystemContext{FS: &base, Root: "."}}}}}, Definition: definition,
})
if err != nil {
t.Fatal(err)
}
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
if !ok {
t.Fatal("inspection unavailable")
}
state, err := configuration.Capture(&result.Config.PromptKit, view, nil)
if err != nil {
t.Fatal(err)
}
child, _, err := state.GetPromptSnapshot(context.Background(), "child")
if err != nil || child.Templates[0].Content != "replacement" || child.SchemaFile == nil || child.SchemaFile.Path != "catalog/schema.json" {
t.Fatalf("child = %+v, %v", child, err)
}
data, err := fs.ReadFile(child.SchemaFile.FS, child.SchemaFile.Path)
if err != nil || string(data) != `{"type":"object"}` {
t.Fatalf("schema = %q, %v", data, err)
}
}
func TestCapturedInspectionUsesNativeView(t *testing.T) {
type anotherApplication struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
Sibling int `yaml:"sibling"`
}
definition := promptkit.Configure(configkit.Definition[application]{}, func(value *application) *promptkit.Settings { return &value.PromptKit })
first, err := configkit.Load(configkit.Bytes("first", []byte("promptkit: {application_field: first}\n")), configkit.Options[application]{Definition: definition})
if err != nil {
t.Fatal(err)
}
secondDefinition := promptkit.Configure(configkit.Definition[anotherApplication]{}, func(value *anotherApplication) *promptkit.Settings { return &value.PromptKit })
second, err := configkit.Load(configkit.Bytes("second", []byte("promptkit: {application_field: second}\n")), configkit.Options[anotherApplication]{Definition: secondDefinition})
if err != nil {
t.Fatal(err)
}
leftView, _ := first.InspectSubtree(configkit.Keys("promptkit"))
rightView, _ := second.InspectSubtree(configkit.Keys("promptkit"))
diagnostics := []configkit.Diagnostic{{Message: "warning", Primary: &configkit.Location{SourceID: "first"}, Related: []configkit.Location{{SourceID: "related"}}}}
left, err := configuration.Capture(&first.Config.PromptKit, leftView, diagnostics)
if err != nil {
t.Fatal(err)
}
right, err := configuration.Capture(&second.Config.PromptKit, rightView, nil)
if err != nil {
t.Fatal(err)
}
diagnostics[0].Primary.SourceID = "changed"
diagnostics[0].Related[0].SourceID = "changed"
got := left.Inspection().Diagnostics()
if got[0].Primary.SourceID != "first" || got[0].Related[0].SourceID != "related" {
t.Fatalf("captured diagnostics = %+v", got)
}
got[0].Primary.SourceID = "again"
got[0].Related[0].SourceID = "again"
got = left.Inspection().Diagnostics()
if got[0].Primary.SourceID != "first" || got[0].Related[0].SourceID != "related" {
t.Fatalf("returned diagnostics aliased: %+v", got)
}
differences, err := configuration.Diff(left.Inspection(), right.Inspection())
if err != nil || len(differences) == 0 || len(left.Inspection().HistoricalPaths()) == 0 {
t.Fatalf("diff = %+v, %v", differences, err)
}
}
func TestCaptureOwnsEffectiveResources(t *testing.T) {
definition := promptkit.Configure(configkit.Definition[application]{}, func(value *application) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.Load(configkit.Bytes("application", []byte(`other: application
promptkit:
backends:
- id: local
endpoint: https://example.test/v1
extra_params: {nested: {value: original}}
profiles:
- id: profile
backend: local
model: original
prompts:
- id: prompt
inputs: [{name: document, required: false, description: text}]
messages: [{role: user, content: hello}]
output:
format: json
validation_mode: json_schema
schema: {type: object, properties: {answer: {type: string}}}
`)), configkit.Options[application]{Definition: definition})
if err != nil {
t.Fatal(err)
}
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
if !ok {
t.Fatal("inspection unavailable")
}
state, err := configuration.Capture(&result.Config.PromptKit, view, result.Diagnostics())
if err != nil {
t.Fatal(err)
}
settings := &result.Config.PromptKit
*settings.Backends[0].Endpoint = "https://changed.test/v1"
settings.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
*settings.Profiles[0].Model = "changed"
*settings.Prompts[0].Messages[0].Content = "changed"
settings.Prompts[0].Inputs[0].Required = true
settings.Prompts[0].Output.Schema.(map[string]any)["type"] = "changed"
backend, err := state.InspectBackend(context.Background(), "local")
if err != nil || backend.Endpoint != "https://example.test/v1" || backend.ExtraParams["nested"].(map[string]any)["value"] != "original" {
t.Fatalf("backend = %+v, %v", backend, err)
}
profile, err := state.GetProfile(context.Background(), "profile")
if err != nil || profile.ResolvedTarget.Model != "original" {
t.Fatalf("profile = %+v, %v", profile, err)
}
prompt, err := state.InspectPrompt(context.Background(), "prompt")
if err != nil || prompt.Messages[0].Content != "hello" || prompt.Output.Schema.(map[string]any)["type"] != "object" || prompt.Inputs == nil || (*prompt.Inputs)[0].Required == nil || *(*prompt.Inputs)[0].Required {
t.Fatalf("prompt = %+v, %v", prompt, err)
}
}

View File

@@ -8,24 +8,30 @@ import (
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
) )
// Inspection is the read-only subset of the retained ConfigKit result used by // Inspection is the read-only subset of the native ConfigKit view used by
// the facade. It does not expose typed configuration or resource contexts. // the facade. It does not expose typed configuration or resource contexts.
type Inspection interface { type Inspection interface {
Metadata() configkit.Metadata Metadata() configkit.Metadata
Sources() []configkit.SourceInfo Sources() []configkit.SourceInfo
Diagnostics() []configkit.Diagnostic Diagnostics() []configkit.Diagnostic
HistoricalPaths() []configkit.Path
Explain(configkit.Path) (configkit.Explanation, bool) Explain(configkit.Path) (configkit.Explanation, bool)
WriteEffective(io.Writer) error WriteEffective(io.Writer) error
WriteExplanation(io.Writer, configkit.Path) error WriteExplanation(io.Writer, configkit.Path) error
capturedResult() *configkit.Result[document] nativeView() configkit.Inspection
} }
type inspectionResult struct { type inspectionResult struct {
*configkit.Result[document] configkit.Inspection
diagnostics []configkit.Diagnostic
} }
func (inspection inspectionResult) capturedResult() *configkit.Result[document] { func (inspection inspectionResult) nativeView() configkit.Inspection {
return inspection.Result return inspection.Inspection
}
func (inspection inspectionResult) Diagnostics() []configkit.Diagnostic {
return ownDiagnostics(inspection.diagnostics)
} }
// Inspection returns only the retained result's inspection operations. // Inspection returns only the retained result's inspection operations.
@@ -33,27 +39,25 @@ func (state *loaded) Inspection() Inspection {
if state == nil { if state == nil {
return nil return nil
} }
return inspectionResult{Result: state.result} return inspectionResult{Inspection: state.view, diagnostics: state.diagnostics}
} }
// Diff compares two captured ConfigKit results without reconstructing them. // Diff compares two captured ConfigKit results without reconstructing them.
func Diff(left, right Inspection) ([]configkit.Difference, error) { func Diff(left, right Inspection) ([]configkit.Difference, error) {
var leftResult, rightResult *configkit.Result[document] return left.nativeView().Diff(right.nativeView())
if left != nil {
leftResult = left.capturedResult()
}
if right != nil {
rightResult = right.capturedResult()
}
return configkit.Diff(leftResult, rightResult)
} }
func secretProviderPath(path configkit.Path) bool { // Sensitive marks credential-shaped provider values beneath PromptKit settings.
func Sensitive(path configkit.Path) bool {
pointer := path.String() pointer := path.String()
if pointer == "" { if pointer == "" {
return false return false
} }
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/") parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
if len(parts) == 0 || parts[0] != "promptkit" {
return false
}
parts = parts[1:]
if !providerExtraParamsPath(parts) { if !providerExtraParamsPath(parts) {
return false return false
} }

View File

@@ -12,20 +12,20 @@ func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) {
path configkit.Path path configkit.Path
want bool want bool
}{ }{
{name: "backend nested secret", path: configkit.Keys("backends").Index(0).Key("extra_params").Key("nested").Key("Token"), want: true}, {name: "backend nested secret", path: configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("nested").Key("Token"), want: true},
{name: "profile nested secret", path: configkit.Keys("profiles").Index(1).Key("extra_params").Key("authorization"), want: true}, {name: "profile nested secret", path: configkit.Keys("promptkit", "profiles").Index(1).Key("extra_params").Key("authorization"), want: true},
{name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends").Index(2).Key("extra_params").Key("nested").Key("Token"), want: true}, {name: "collection backend nested secret", path: configkit.Keys("promptkit", "collections", "provider", "backends").Index(2).Key("extra_params").Key("nested").Key("Token"), want: true},
{name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles").Index(0).Key("extra_params").Key("authorization"), want: true}, {name: "collection profile secret", path: configkit.Keys("promptkit", "collections", "assistant", "profiles").Index(0).Key("extra_params").Key("authorization"), want: true},
{name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false}, {name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false},
{name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false}, {name: "extra params outside resource map", path: configkit.Keys("promptkit", "backends", "provider", "options", "extra_params", "token"), want: false},
{name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false}, {name: "collection extra params outside resource map", path: configkit.Keys("promptkit", "collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
{name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false}, {name: "missing resource key", path: configkit.Keys("promptkit", "backends", "", "extra_params", "token"), want: false},
{name: "ordinary provider value", path: configkit.Keys("profiles").Index(0).Key("extra_params").Key("provider").Key("mode"), want: false}, {name: "ordinary provider value", path: configkit.Keys("promptkit", "profiles").Index(0).Key("extra_params").Key("provider").Key("mode"), want: false},
} }
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
if got := secretProviderPath(test.path); got != test.want { if got := Sensitive(test.path); got != test.want {
t.Fatalf("secretProviderPath(%q) = %t, want %t", test.path, got, test.want) t.Fatalf("Sensitive(%q) = %t, want %t", test.path, got, test.want)
} }
}) })
} }

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"strings" "strings"
"testing" "testing"
"testing/fstest"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
) )
@@ -32,59 +33,91 @@ func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
if err != nil || profile.ResolvedTarget.ServiceTier != "standard" || profile.ResolvedTarget.APIKeyEnv != "TOKEN" { if err != nil || profile.ResolvedTarget.ServiceTier != "standard" || profile.ResolvedTarget.APIKeyEnv != "TOKEN" {
t.Fatalf("interpreted profile = %+v, %v", profile, err) t.Fatalf("interpreted profile = %+v, %v", profile, err)
} }
composed := state.result.Config.Backends[0] composed := state.backendRecords["local"]
if *composed.Endpoint != "https://example.invalid/v1" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 || if *composed.Endpoint != "https://example.invalid/v1" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 ||
*composed.SmokeTests.Reachability.Endpoint != "https://example.invalid/health" || *composed.SmokeTests.Reachability.Endpoint != "https://example.invalid/health" ||
*composed.SmokeTests.Authentication.Kind != "none" || *composed.SmokeTests.Authentication.Endpoint != "https://example.invalid/key" || *composed.SmokeTests.Authentication.Kind != "none" || *composed.SmokeTests.Authentication.Endpoint != "https://example.invalid/key" ||
*state.result.Config.Profiles[0].ServiceTier != "standard" || *state.result.Config.Profiles[0].APIKeyEnv != "TOKEN" { *state.profileRecords["partial"].ServiceTier != "standard" || *state.profileRecords["partial"].APIKeyEnv != "TOKEN" {
t.Fatalf("resolved values were not normalized: %+v", state.result.Config) t.Fatalf("resolved values were not normalized: %+v", composed)
} }
path := configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint") path := configkit.Keys("promptkit", "backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint")
field, found := state.result.Metadata().Lookup(path) field, found := state.inspection.Metadata().Lookup(path)
if !found || field.Origin.Operation != configkit.ChangeNormalization { if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("nested field origin changed: %+v", field) t.Fatalf("nested field origin changed: %+v", field)
} }
for _, path := range []configkit.Path{ for _, path := range []configkit.Path{
configkit.Keys("backends").Index(0).Key("endpoint"), configkit.Keys("promptkit", "backends").Index(0).Key("endpoint"),
configkit.Keys("backends").Index(0).Key("smoke_tests").Key("reachability").Key("endpoint"), configkit.Keys("promptkit", "backends").Index(0).Key("smoke_tests").Key("reachability").Key("endpoint"),
configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("kind"), configkit.Keys("promptkit", "backends").Index(0).Key("smoke_tests").Key("authentication").Key("kind"),
configkit.Keys("profiles").Index(0).Key("service_tier"), configkit.Keys("promptkit", "profiles").Index(0).Key("service_tier"),
configkit.Keys("profiles").Index(0).Key("api_key_env"), configkit.Keys("promptkit", "profiles").Index(0).Key("api_key_env"),
} { } {
field, found := state.result.Metadata().Lookup(path) field, found := state.inspection.Metadata().Lookup(path)
if !found || field.Origin.Operation != configkit.ChangeNormalization { if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Errorf("normalization at %v: %+v", path, field) t.Errorf("normalization at %v: %+v", path, field)
} }
} }
} }
func TestProviderPolicyCaptureAndInspection(t *testing.T) {
state, err := load(mapping("provider policy", map[string]any{
"collections": map[string]any{"team": map[string]any{
"backends": []any{map[string]any{"id": "renamed", "endpoint": "https://example.invalid/v1", "provider": "OPENROUTER", "zero_cost": true}, map[string]any{"id": "child", "extends": map[string]any{"id": "renamed"}, "zero_cost": false}},
"profiles": []any{map[string]any{"id": "selected", "backend": "team/child", "endpoint": "https://other.invalid/v1", "model": "m"}},
}},
"profiles": []any{map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1", "model": "m"}},
}), nil)
if err != nil {
t.Fatal(err)
}
backend, err := state.GetBackend("team/child")
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost {
t.Fatalf("backend: %+v, %v", backend, err)
}
profile, err := state.GetProfile(context.Background(), "team/selected")
if err != nil || profile.ResolvedTarget.Provider != "openrouter" || profile.ResolvedTarget.ZeroCost || profile.ResolvedTarget.Endpoint != "https://other.invalid/v1" {
t.Fatalf("profile: %+v, %v", profile, err)
}
profile, err = state.GetProfile(context.Background(), "generic")
if err != nil || profile.ResolvedTarget.Provider != "openai-compatible" || profile.ResolvedTarget.ZeroCost {
t.Fatalf("generic: %+v, %v", profile, err)
}
resource, err := state.InspectBackend(context.Background(), "team/child")
if err != nil || resource.Provider == nil || *resource.Provider != "openrouter" || resource.ZeroCost == nil || *resource.ZeroCost {
t.Fatalf("inspection: %+v, %v", resource, err)
}
}
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) { func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
definition := map[string]any{ definition := map[string]any{
"id": "example", "inputs": []any{map[string]any{"name": " document "}}, "id": "example", "inputs": []any{map[string]any{"name": " document "}},
"messages": []any{map[string]any{"role": "user", "content": "hello"}}, "messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": " schema.json "}, "output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": " schema.json "},
} }
state, err := load(mapping("prompt metadata", map[string]any{"prompts": []any{definition}}), nil) filesystem := configkit.NewFilesystem("schemas", fstest.MapFS{})
input := mapping("prompt metadata", map[string]any{"schema_root": ".", "prompts": []any{definition}})
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
state, err := load(input, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
selected, inspection, err := state.prompt("example") selected, inspection, err := state.prompt("example")
if err != nil || selected.Validation.SchemaPath != "schema.json" || inspection.OutputContract.SchemaPath != "schema.json" || selected.Inputs[0].Name != "document" { if err != nil || selected.Validation.SchemaPath != "schema.json" || selected.Inputs[0].Name != "document" {
t.Fatalf("interpreted prompt = %+v, %+v, %v", selected, inspection, err) t.Fatalf("interpreted prompt = %+v, %+v, %v", selected, inspection, err)
} }
composed := state.result.Config.Prompts[0] composed := state.prompts["example"].resource
if *composed.Output.SchemaPath != "schema.json" || composed.Inputs[0].Name != "document" { if *composed.Output.SchemaPath != "schema.json" || composed.Inputs[0].Name != "document" {
t.Fatalf("prompt was not normalized: %+v", composed) t.Fatalf("prompt was not normalized: %+v", composed)
} }
definition["inputs"] = []any{map[string]any{"name": " document "}, map[string]any{"name": "document"}} definition["inputs"] = []any{map[string]any{"name": " document "}, map[string]any{"name": "document"}}
_, err = load(mapping("duplicate inputs", map[string]any{"prompts": []any{definition}}), nil) _, err = load(mapping("duplicate inputs", map[string]any{"prompts": []any{definition}}), nil)
if err == nil || !strings.Contains(err.Error(), "/prompts/0/inputs/1") || !strings.Contains(err.Error(), "duplicated") { if err == nil || !strings.Contains(err.Error(), "/promptkit/prompts/0/inputs/1") || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("duplicate normalized names were not rejected: %v", err) t.Fatalf("duplicate normalized names were not rejected: %v", err)
} }
} }
func TestPromptResolutionRecordsMessageFields(t *testing.T) { func TestPromptResolutionRecordsMessageFields(t *testing.T) {
loadPrompt := func(role, cacheType, ttl string) *loaded { loadPrompt := func(role, cacheType, ttl string) *testLoaded {
t.Helper() t.Helper()
state, err := load(mapping("prompt", map[string]any{"prompts": []any{map[string]any{ state, err := load(mapping("prompt", map[string]any{"prompts": []any{map[string]any{
"id": "example", "inputs": []any{map[string]any{"name": " input ", "content_type": nil, "description": " text "}}, "id": "example", "inputs": []any{map[string]any{"name": " input ", "content_type": nil, "description": " text "}},
@@ -106,14 +139,15 @@ func TestPromptResolutionRecordsMessageFields(t *testing.T) {
if err != nil || inspection.PromptHash != otherInspection.PromptHash || definition.Templates[0].Role != other.Templates[0].Role || definition.Templates[0].Content != "" { if err != nil || inspection.PromptHash != otherInspection.PromptHash || definition.Templates[0].Role != other.Templates[0].Role || definition.Templates[0].Content != "" {
t.Fatalf("equivalent prompt mismatch: %+v %+v %v", definition, other, err) t.Fatalf("equivalent prompt mismatch: %+v %+v %v", definition, other, err)
} }
base := configkit.Keys("prompts").Index(0) base := configkit.Keys("promptkit", "prompts").Index(0)
for _, path := range []configkit.Path{base.Key("inputs").Index(0).Key("name"), base.Key("inputs").Index(0).Key("description"), base.Key("messages").Index(0).Key("role"), base.Key("messages").Index(0).Key("cache_control").Key("type"), base.Key("messages").Index(0).Key("cache_control").Key("ttl")} { for _, path := range []configkit.Path{base.Key("inputs").Index(0).Key("name"), base.Key("inputs").Index(0).Key("description"), base.Key("messages").Index(0).Key("role"), base.Key("messages").Index(0).Key("cache_control").Key("type"), base.Key("messages").Index(0).Key("cache_control").Key("ttl")} {
field, found := padded.result.Metadata().Lookup(path) field, found := padded.inspection.Metadata().Lookup(path)
if !found || field.Origin.Operation != configkit.ChangeNormalization { if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Errorf("normalization at %v: %+v", path, field) t.Errorf("normalization at %v: %+v", path, field)
} }
} }
if padded.promptMetadata["example"][0].contentType != fieldNull { field, ok := padded.inspection.Metadata().Lookup(base.Key("inputs").Index(0).Key("content_type"))
if !ok || !field.Null {
t.Fatal("null input metadata was not retained") t.Fatal("null input metadata was not retained")
} }
} }
@@ -129,12 +163,12 @@ func TestResolutionRecordsTargetDependencies(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
for _, base := range []configkit.Path{configkit.Keys("backends").Index(0), configkit.Keys("collections", "scoped", "backends").Index(0)} { for _, base := range []configkit.Path{configkit.Keys("promptkit", "backends").Index(0), configkit.Keys("promptkit", "collections", "scoped", "backends").Index(0)} {
endpoint, ok := state.result.Explain(base.Key("endpoint")) endpoint, ok := state.inspection.Explain(base.Key("endpoint"))
if !ok || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") { if !ok || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") {
t.Fatalf("endpoint explanation: %+v", endpoint) t.Fatalf("endpoint explanation: %+v", endpoint)
} }
history := state.result.Metadata().History(base.Key("queue_capacity")) history := state.inspection.Metadata().History(base.Key("queue_capacity"))
found := false found := false
for _, change := range history { for _, change := range history {
if change.Origin.Operation == configkit.ChangeDerivedDefault { if change.Origin.Operation == configkit.ChangeDerivedDefault {

View File

@@ -11,7 +11,7 @@ import (
) )
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) { func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{ state, err := load(mapping("identities", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "http://localhost:8080/v1"}}, "backends": []any{map[string]any{"id": "local", "endpoint": "http://localhost:8080/v1"}},
"collections": map[string]any{ "collections": map[string]any{
"z": map[string]any{"profiles": []any{map[string]any{"id": "profile", "backend": "local", "model": "z-model"}}, "prompts": []any{listingPromptDefinition("prompt")}}, "z": map[string]any{"profiles": []any{map[string]any{"id": "profile", "backend": "local", "model": "z-model"}}, "prompts": []any{listingPromptDefinition("prompt")}},
@@ -56,7 +56,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
} }
func TestLoadedListsAreIndependent(t *testing.T) { func TestLoadedListsAreIndependent(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{ state, err := load(mapping("identities", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.test/v1"}}, "backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.test/v1"}},
"profiles": []any{map[string]any{"id": "profile"}}, "profiles": []any{map[string]any{"id": "profile"}},
"prompts": []any{listingPromptDefinition("example")}, "prompts": []any{listingPromptDefinition("example")},
@@ -151,7 +151,7 @@ func TestLoadedListsSelectNamespaces(t *testing.T) {
} }
func TestLoadedListsEmptyInventory(t *testing.T) { func TestLoadedListsEmptyInventory(t *testing.T) {
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil) state, err := load(configkit.Input{Source: configkit.NoSource()}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -170,7 +170,7 @@ func TestLoadedListsEmptyInventory(t *testing.T) {
} }
func TestLoadedListsHonorCanceledContext(t *testing.T) { func TestLoadedListsHonorCanceledContext(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{ state, err := load(mapping("identities", map[string]any{
"profiles": []any{map[string]any{"id": "partial"}}, "profiles": []any{map[string]any{"id": "partial"}},
"prompts": []any{listingPromptDefinition("example")}, "prompts": []any{listingPromptDefinition("example")},
}), nil) }), nil)

View File

@@ -13,114 +13,22 @@ import (
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
"go.maximumdirect.net/promptkit/internal/settings"
) )
type document struct { type collectionConfig = settings.CollectionSettings
Version *string `yaml:"version"` type backendConfig = settings.BackendSettings
Backends []backendConfig `yaml:"backends"` type profileConfig = settings.ProfileSettings
Profiles []profileConfig `yaml:"profiles"` type promptConfig = settings.PromptSettings
Prompts []promptConfig `yaml:"prompts"` type promptInputConfig = settings.PromptInputSettings
Collections map[string]collectionConfig `yaml:"collections"` type promptMessageConfig = settings.PromptMessageSettings
} type cacheControlConfig = settings.PromptCacheControlSettings
type promptOutputConfig = settings.PromptOutputSettings
type collectionConfig struct {
ReleaseVersion *string `yaml:"release_version"`
Backends []backendConfig `yaml:"backends"`
Profiles []profileConfig `yaml:"profiles"`
Prompts []promptConfig `yaml:"prompts"`
}
type backendConfig struct {
ID string `yaml:"id"`
Endpoint *string `yaml:"endpoint"`
APIKeyEnv *string `yaml:"api_key_env"`
ConcurrencyLimit *int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
ExtraParams map[string]any `yaml:"extra_params"`
SmokeTests *backendSmokeTestConfig `yaml:"smoke_tests"`
}
type backendSmokeTestConfig struct {
DefaultProfile *string `yaml:"default_profile"`
Reachability *reachabilityProbeConfig `yaml:"reachability"`
Authentication *authenticationProbeConfig `yaml:"authentication"`
}
type reachabilityProbeConfig struct {
Endpoint *string `yaml:"endpoint"`
}
type authenticationProbeConfig struct {
Kind *string `yaml:"kind"`
Endpoint *string `yaml:"endpoint"`
}
type profileConfig struct {
ID string `yaml:"id"`
Backend *string `yaml:"backend"`
Endpoint *string `yaml:"endpoint"`
Model *string `yaml:"model"`
Temperature *float64 `yaml:"temperature"`
TopP *float64 `yaml:"top_p"`
MaxTokens *int `yaml:"max_tokens"`
TimeoutSeconds *int `yaml:"timeout_seconds"`
ServiceTier *string `yaml:"service_tier"`
ReasoningEffort *string `yaml:"reasoning_effort"`
APIKeyEnv *string `yaml:"api_key_env"`
APIKeyRequired *bool `yaml:"api_key_required"`
ExtraParams map[string]any `yaml:"extra_params"`
}
type promptConfig struct {
ID string `yaml:"id"`
DefaultProfile *string `yaml:"default_profile"`
Description *string `yaml:"description"`
SessionID *string `yaml:"session_id"`
Inputs []promptInputConfig `yaml:"inputs"`
Messages []promptMessageConfig `yaml:"messages"`
Output *promptOutputConfig `yaml:"output"`
}
type promptInputConfig struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType *string `yaml:"content_type"`
Description *string `yaml:"description"`
}
type promptMessageConfig struct {
Role *string `yaml:"role"`
Content *string `yaml:"content"`
ContentFile *string `yaml:"content_file"`
CacheControl *cacheControlConfig `yaml:"cache_control"`
}
type cacheControlConfig struct {
Type *string `yaml:"type"`
TTL *string `yaml:"ttl"`
}
type promptOutputConfig struct {
Format *string `yaml:"format"`
ValidationMode *string `yaml:"validation_mode"`
SchemaPath *string `yaml:"schema_path"`
RepairAttempts int `yaml:"repair_attempts"`
}
// fieldState distinguishes omission, an explicit value, and an explicit null.
// Inherited values are explicit because ConfigKit has already expanded them.
type fieldState uint8
const (
fieldAbsent fieldState = iota
fieldValue
fieldNull
)
type loaded struct { type loaded struct {
result *configkit.Result[document] view configkit.Inspection
promptMetadata map[string][]promptInputPresence diagnostics []configkit.Diagnostic
promptRepair map[string]bool promptRequired map[string][]bool
prompts map[string]*promptEntry prompts map[string]*promptEntry
registry *backend.Registry registry *backend.Registry
backendRecords map[string]backendConfig backendRecords map[string]backendConfig
@@ -133,6 +41,9 @@ type loaded struct {
collections []CollectionIdentity collections []CollectionIdentity
} }
// Snapshot holds configuration resources captured for engine assembly.
type Snapshot = loaded
type resourceBases struct { type resourceBases struct {
backends configkit.Path backends configkit.Path
profiles configkit.Path profiles configkit.Path
@@ -150,20 +61,20 @@ type resourceSet struct {
func rootResourceBases() resourceBases { func rootResourceBases() resourceBases {
return resourceBases{ return resourceBases{
backends: configkit.Keys("backends"), backends: configkit.Keys("promptkit", "backends"),
profiles: configkit.Keys("profiles"), profiles: configkit.Keys("promptkit", "profiles"),
prompts: configkit.Keys("prompts"), prompts: configkit.Keys("promptkit", "prompts"),
} }
} }
func resourceSets(config *document) []resourceSet { func resourceSets(config *settings.Settings) []resourceSet {
sets := make([]resourceSet, 0, len(config.Collections)+1) sets := make([]resourceSet, 0, len(config.Collections)+1)
sets = append(sets, resourceSet{ sets = append(sets, resourceSet{
bases: rootResourceBases(), backends: config.Backends, profiles: config.Profiles, prompts: config.Prompts, bases: rootResourceBases(), backends: config.Backends, profiles: config.Profiles, prompts: config.Prompts,
}) })
for _, namespace := range sortedKeys(config.Collections) { for _, namespace := range sortedKeys(config.Collections) {
collection := config.Collections[namespace] collection := config.Collections[namespace]
base := configkit.Keys("collections", namespace) base := configkit.Keys("promptkit", "collections", namespace)
sets = append(sets, resourceSet{ sets = append(sets, resourceSet{
isCollection: true, isCollection: true,
namespace: namespace, namespace: namespace,
@@ -191,7 +102,7 @@ func indexedResourceID(namespace, localKey string) string {
return namespace + "/" + localKey return namespace + "/" + localKey
} }
func indexedBackends(config *document) map[string]backendConfig { func indexedBackends(config *settings.Settings) map[string]backendConfig {
backends := make(map[string]backendConfig) backends := make(map[string]backendConfig)
for _, resources := range resourceSets(config) { for _, resources := range resourceSets(config) {
for _, definition := range resources.backends { for _, definition := range resources.backends {
@@ -201,7 +112,7 @@ func indexedBackends(config *document) map[string]backendConfig {
return backends return backends
} }
func indexedProfiles(config *document) map[string]profileConfig { func indexedProfiles(config *settings.Settings) map[string]profileConfig {
profiles := make(map[string]profileConfig) profiles := make(map[string]profileConfig)
for _, resources := range resourceSets(config) { for _, resources := range resourceSets(config) {
for _, definition := range resources.profiles { for _, definition := range resources.profiles {
@@ -213,65 +124,61 @@ func indexedProfiles(config *document) map[string]profileConfig {
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// Load captures one immutable effective configuration for internal consumers. // SequenceIdentities returns the component's native sequence registrations.
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) { func SequenceIdentities() []configkit.SequenceIdentity {
return load(main, underlays) return []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("promptkit", "backends"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("promptkit", "profiles"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("promptkit", "prompts"), "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("promptkit"), configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("backends")}, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("promptkit"), configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("profiles")}, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("promptkit"), configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("prompts")}, "id"),
}
} }
func configurationDefinition(state *loaded) configkit.Definition[document] { // Resolve normalizes the selected effective settings in place.
return configkit.Definition[document]{ func Resolve(config *settings.Settings, resolution *configkit.Resolution) {
SequenceIdentities: []configkit.SequenceIdentity{ if config == nil {
configkit.SequenceIdentityAt(configkit.Keys("backends"), "id"), return
configkit.SequenceIdentityAt(configkit.Keys("profiles"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("backends")}, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("profiles")}, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("prompts")}, "id"),
},
Version: &configkit.VersionRule{
Path: configkit.Keys("version"),
Check: checkConfigurationVersion,
},
Sensitive: secretProviderPath,
Resolve: func(config *document, resolution *configkit.Resolution) error {
for _, resources := range resourceSets(config) {
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata, state.promptRepair)
} }
for _, resources := range resourceSets(config) { for _, resources := range resourceSets(config) {
resolveBackends(resources.backends, resources.bases.backends, resolution) resolveBackends(resources.backends, resources.bases.backends, resolution)
resolveProfiles(resources.profiles, resources.bases.profiles, resolution) resolveProfiles(resources.profiles, resources.bases.profiles, resolution)
resolvePrompts(resources.prompts, resources.bases.prompts, resolution) resolvePrompts(resources.prompts, resources.bases.prompts, resolution)
} }
return nil }
},
Validate: func(config *document, validation *configkit.Validation) { // Validate checks the selected effective settings.
func Validate(config *settings.Settings, validation *configkit.Validation) {
if config != nil {
validateEffective(config, validation) validateEffective(config, validation)
},
} }
} }
// load performs one ConfigKit lifecycle over the selected source tiers. // Capture builds owned runtime resources from effective settings and native evidence.
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) { func Capture(input *settings.Settings, view configkit.Inspection, diagnostics []configkit.Diagnostic) (*loaded, error) {
state := &loaded{ config := ownSettings(input)
promptMetadata: make(map[string][]promptInputPresence), state := &loaded{view: view, diagnostics: ownDiagnostics(diagnostics), promptRequired: make(map[string][]bool)}
promptRepair: make(map[string]bool), metadata := view.Metadata()
for _, resources := range resourceSets(&config) {
for index, entry := range resources.prompts {
presence := make([]bool, len(entry.Inputs))
for i := range entry.Inputs {
path := resources.bases.prompts.Index(index).Key("inputs").Index(i)
// Typed values capture defaults; explicitness preserves source false.
// IsExplicit avoids projecting inheritance origins for a flag.
presence[i] = entry.Inputs[i].Required || metadata.IsExplicit(path.Key("required"))
} }
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{ state.promptRequired[indexedResourceID(resources.namespace, entry.ID)] = presence
Underlays: underlays, }
SourceVersionMode: configkit.SourceVersionModeIfPresent, }
Definition: configurationDefinition(state), if err := state.materializePrompts(&config, metadata); err != nil {
})
if err != nil {
return nil, fmt.Errorf("load configuration: %w", err) return nil, fmt.Errorf("load configuration: %w", err)
} }
state.result = result if err := state.buildExecutionProfiles(&config); err != nil {
if err := state.materializePrompts(); err != nil {
return nil, fmt.Errorf("load configuration: %w", err) return nil, fmt.Errorf("load configuration: %w", err)
} }
if err := state.buildExecutionProfiles(); err != nil { state.backendIDs = sortedKeys(indexedBackends(&config))
return nil, fmt.Errorf("load configuration: %w", err)
}
state.backendIDs = sortedKeys(indexedBackends(&state.result.Config))
state.profileIDs = sortedKeys(state.indexedProfiles) state.profileIDs = sortedKeys(state.indexedProfiles)
for _, id := range sortedKeys(state.prompts) { for _, id := range sortedKeys(state.prompts) {
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id}) state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
@@ -281,8 +188,8 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
"profile": resourceNamespaces(state.profileIDs), "profile": resourceNamespaces(state.profileIDs),
"prompt": promptNamespaces(state.promptIDs), "prompt": promptNamespaces(state.promptIDs),
} }
for _, namespace := range sortedKeys(result.Config.Collections) { for _, namespace := range sortedKeys(config.Collections) {
collection := result.Config.Collections[namespace] collection := config.Collections[namespace]
state.collections = append(state.collections, CollectionIdentity{ state.collections = append(state.collections, CollectionIdentity{
Namespace: namespace, ReleaseVersion: stringPointerValue(collection.ReleaseVersion), Namespace: namespace, ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
}) })
@@ -290,6 +197,18 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
return state, nil return state, nil
} }
func ownDiagnostics(values []configkit.Diagnostic) []configkit.Diagnostic {
if values == nil {
return nil
}
result := append([]configkit.Diagnostic(nil), values...)
for i := range result {
result[i].Primary = copyPointer(result[i].Primary)
result[i].Related = append([]configkit.Location(nil), result[i].Related...)
}
return result
}
func resourceNamespaces(ids []string) []string { func resourceNamespaces(ids []string) []string {
namespaces := make(map[string]struct{}) namespaces := make(map[string]struct{})
for _, id := range ids { for _, id := range ids {
@@ -308,52 +227,6 @@ func promptNamespaces(prompts []domain.PromptIdentity) []string {
return resourceNamespaces(ids) return resourceNamespaces(ids)
} }
func checkConfigurationVersion(version configkit.Version, validation *configkit.Validation) error {
if !version.Present {
return nil
}
if version.Null || version.Text != "2" {
validation.Error(configkit.Keys("version"), "unsupported configuration specification version")
}
return nil
}
type promptInputPresence struct {
required fieldState
contentType fieldState
description fieldState
}
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence, repair map[string]bool) {
for promptIndex, prompt := range prompts {
inputs := prompt.Inputs
states := make([]promptInputPresence, len(inputs))
base := basePath.Index(promptIndex).Key("inputs")
for index := range inputs {
path := base.Index(index)
states[index] = promptInputPresence{
required: captureFieldState(path.Key("required"), resolution),
contentType: captureFieldState(path.Key("content_type"), resolution),
description: captureFieldState(path.Key("description"), resolution),
}
}
destination[indexedResourceID(namespace, prompt.ID)] = states
repair[indexedResourceID(namespace, prompt.ID)] = captureFieldState(basePath.Index(promptIndex).Key("output").Key("repair_attempts"), resolution) == fieldValue
}
}
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
info, exists := resolution.EntryInfo(path)
switch {
case !exists || !info.Explicit:
return fieldAbsent
case info.Null:
return fieldNull
default:
return fieldValue
}
}
// eligibleResourceList leaves invalid identities to the existing validation // eligibleResourceList leaves invalid identities to the existing validation
// boundaries, without mutating a list that cannot be recorded by identity. // boundaries, without mutating a list that cannot be recorded by identity.
func eligibleResourceList[T any](entries []T, identity func(T) string) bool { func eligibleResourceList[T any](entries []T, identity func(T) string) bool {
@@ -386,12 +259,12 @@ func sortedKeys[T any](entries map[string]T) []string {
return keys return keys
} }
func validateEffective(config *document, validation *configkit.Validation) { func validateEffective(config *settings.Settings, validation *configkit.Validation) {
sets := resourceSets(config) sets := resourceSets(config)
backendIDs := make(map[string]struct{}) backendIDs := make(map[string]struct{})
for _, resources := range sets { for _, resources := range sets {
if resources.isCollection && !domain.IsValidResourceNamespace(resources.namespace) { if resources.isCollection && !domain.IsValidResourceNamespace(resources.namespace) {
validation.Error(configkit.Keys("collections", resources.namespace), "collection namespace must be nonblank and contain no slash") validation.Error(configkit.Keys("promptkit", "collections", resources.namespace), "collection namespace must be nonblank and contain no slash")
} }
for _, backend := range resources.backends { for _, backend := range resources.backends {
backendIDs[indexedResourceID(resources.namespace, backend.ID)] = struct{}{} backendIDs[indexedResourceID(resources.namespace, backend.ID)] = struct{}{}
@@ -418,6 +291,9 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
if backend.Endpoint == nil || *backend.Endpoint == "" { if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required") validation.Error(base.Key("endpoint"), "endpoint is required")
} }
if backend.Provider != nil && !llm.SupportedProvider(*backend.Provider) {
validation.Error(base.Key("provider"), "unsupported provider")
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) { if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid") validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
} }
@@ -446,10 +322,8 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
} }
if authentication := smokeTests.Authentication; authentication != nil { if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind") kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil { if authentication.Kind != nil && *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" && *authentication.Kind != "vllm_models" {
validation.Error(kindPath, "authentication kind is required") validation.Error(kindPath, "unsupported authentication kind")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
} }
} }
} }

View File

@@ -28,7 +28,10 @@ func TestLoadIdentifiedResourceLists(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("load identified resources with header %t: %v", header, err) t.Fatalf("load identified resources with header %t: %v", header, err)
} }
if loaded.result.Config.Backends[0].ID != "local" || loaded.result.Config.Collections["team"].Backends[0].ID != "local" { if _, rootErr := loaded.GetBackend("local"); rootErr != nil {
t.Fatal(rootErr)
}
if _, collectionErr := loaded.GetBackend("team/local"); collectionErr != nil {
t.Fatal("identified root and collection resources were not retained") t.Fatal("identified root and collection resources were not retained")
} }
} }
@@ -201,9 +204,10 @@ func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
for _, kind := range []string{"backends", "profiles", "prompts"} { for _, kind := range []string{"backends", "profiles", "prompts"} {
for _, namespace := range []string{"", "*"} { for _, namespace := range []string{"", "*"} {
t.Run(kind+"/"+namespace, func(t *testing.T) { t.Run(kind+"/"+namespace, func(t *testing.T) {
definition := configurationDefinition(&loaded{}) definition := testDefinition()
definition.Resolve = func(config *document, resolution *configkit.Resolution) error { definition.Resolve = func(root *testDocument, resolution *configkit.Resolution) error {
path := configkit.Keys(kind) config := &root.PromptKit
path := configkit.Keys("promptkit", kind)
if namespace == "" { if namespace == "" {
switch kind { switch kind {
case "backends": case "backends":
@@ -224,7 +228,7 @@ func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
collection.Prompts = []promptConfig{{ID: "same"}, {ID: "same"}} collection.Prompts = []promptConfig{{ID: "same"}, {ID: "same"}}
} }
config.Collections = map[string]collectionConfig{namespace: collection} config.Collections = map[string]collectionConfig{namespace: collection}
path = configkit.Keys("collections", namespace, kind) path = configkit.Keys("promptkit", "collections", namespace, kind)
} }
resolution.RecordNormalization(path, "supply resolved records") resolution.RecordNormalization(path, "supply resolved records")
return nil return nil
@@ -235,7 +239,8 @@ func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
namespace: map[string]any{kind: []any{}}, namespace: map[string]any{kind: []any{}},
}}) }})
} }
_, err := configkit.LoadLayered(input, configkit.LayeredOptions[document]{Definition: definition}) input.Destination = configkit.Keys("promptkit")
_, err := configkit.LoadLayered(input, configkit.LayeredOptions[testDocument]{Definition: definition})
var loadErr *configkit.LoadError var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) { if !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want a final identity diagnostic", err) t.Fatalf("load error = %v, want a final identity diagnostic", err)
@@ -272,14 +277,14 @@ func TestIdentifiedResourceLayerOperations(t *testing.T) {
values := map[string]any{"profiles": test.records} values := map[string]any{"profiles": test.records}
if test.mode != "" { if test.mode != "" {
values["config"] = map[string]any{"sequences": []any{map[string]any{ values["config"] = map[string]any{"sequences": []any{map[string]any{
"destination": []any{"profiles"}, "mode": test.mode, "destination": []any{"promptkit", "profiles"}, "mode": test.mode,
}}} }}}
} }
state, err := load(mapping("overlay profiles", values), []configkit.Underlay{{Inputs: []configkit.Input{base}}}) state, err := load(mapping("overlay profiles", values), []configkit.Underlay{{Inputs: []configkit.Input{base}}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
profiles := indexedProfiles(&state.result.Config) profiles := state.profileRecords
if len(profiles) != len(test.models) { if len(profiles) != len(test.models) {
t.Fatalf("profiles = %+v, want IDs and models %+v", profiles, test.models) t.Fatalf("profiles = %+v, want IDs and models %+v", profiles, test.models)
} }
@@ -315,7 +320,7 @@ func TestIdentifiedResourceLayersRejectDisplacedParentAndDuplicateAppend(t *test
{ {
name: "append cannot duplicate an identity", name: "append cannot duplicate an identity",
values: map[string]any{ values: map[string]any{
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"profiles"}, "mode": "append"}}}, "config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "profiles"}, "mode": "append"}}},
"profiles": []any{map[string]any{"id": "shared", "model": "other"}}, "profiles": []any{map[string]any{"id": "shared", "model": "other"}},
}, },
code: configkit.CodeSequenceConflict, code: configkit.CodeSequenceConflict,
@@ -337,59 +342,8 @@ func TestIdentifiedResourceLayersRejectDisplacedParentAndDuplicateAppend(t *test
} }
} }
func TestLoadChecksOptionalConfigurationVersion(t *testing.T) { func TestLoadRejectsLegacyResourceMappings(t *testing.T) {
for _, test := range []struct {
name string
version any
include bool
valid bool
}{
{name: "string two", version: "2", include: true, valid: true},
{name: "integer two", version: int64(2), include: true, valid: true},
{name: "omitted", valid: true},
{name: "explicit null", include: true},
{name: "unsupported string", version: "1", include: true},
} {
t.Run(test.name, func(t *testing.T) {
values := map[string]any{}
if test.include {
values["version"] = test.version
}
loaded, err := load(mapping("main", values), nil)
if test.valid {
if err != nil {
t.Fatalf("load accepted version: %v", err)
}
if test.include && (loaded.result.Config.Version == nil || *loaded.result.Config.Version != "2") {
t.Fatalf("typed version = %v, want 2", loaded.result.Config.Version)
}
if !test.include && loaded.result.Config.Version != nil {
t.Fatalf("omitted version = %q, want nil", *loaded.result.Config.Version)
}
return
}
var loadErr *configkit.LoadError
if err == nil || !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want ConfigKit load error", err)
}
found := false
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Path.String() == "/version" && diagnostic.Message == "unsupported configuration specification version" {
found = true
}
}
if !found {
t.Fatalf("load diagnostics = %+v, want safe version diagnostic", loadErr.Diagnostics())
}
})
}
}
func TestLoadRejectsMalformedHeadersAndLegacyResourceMappings(t *testing.T) {
for name, values := range map[string]map[string]any{ for name, values := range map[string]map[string]any{
"boolean header": {"version": true},
"fractional header": {"version": 2.5},
"sequence header": {"version": []any{2}},
"backend mapping": {"backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}}}, "backend mapping": {"backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}}},
"profile mapping": {"profiles": map[string]any{"local": map[string]any{}}}, "profile mapping": {"profiles": map[string]any{"local": map[string]any{}}},
"prompt mapping": {"prompts": map[string]any{"local": map[string]any{}}}, "prompt mapping": {"prompts": map[string]any{"local": map[string]any{}}},
@@ -420,23 +374,10 @@ func TestNestedConfigRemainsApplicationData(t *testing.T) {
} }
} }
func TestLoadConsumesAcceptedUnderlayVersionAndValidatesHeaderlessPayload(t *testing.T) { func TestLoadValidatesHeaderlessPromptFragment(t *testing.T) {
underlay := mapping("declared underlay", map[string]any{ invalid := mapping("prompt fragment", map[string]any{"prompts": []any{map[string]any{"id": "incomplete", "description": "still validated"}}})
"version": int64(2), if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalid}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
}) t.Fatalf("invalid fragment error = %v", err)
loaded, err := load(mapping("headerless main", map[string]any{}), []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatalf("load accepted non-main declaration: %v", err)
}
if loaded.result.Config.Version != nil {
t.Fatalf("consumed non-main declaration remained in effective payload: %q", *loaded.result.Config.Version)
}
invalidFragment := mapping("headerless prompt fragment", map[string]any{
"prompts": []any{map[string]any{"id": "incomplete", "description": "still validated"}},
})
if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalidFragment}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
t.Fatalf("headerless fragment validation error = %v, want final prompt validation", err)
} }
} }
@@ -484,7 +425,7 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
} }
t.Fatal(err) t.Fatal(err)
} }
profile := loaded.result.Config.Collections["base"].Profiles[1] profile := loaded.profileRecords["base/child.v1"]
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired { if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
t.Fatalf("effective values were not retained: %+v", profile) t.Fatalf("effective values were not retained: %+v", profile)
} }
@@ -492,7 +433,7 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
if provider["first"] != int64(1) || provider["second"] != int64(3) { if provider["first"] != int64(1) || provider["second"] != int64(3) {
t.Fatalf("recursive overlay failed: %+v", provider) t.Fatalf("recursive overlay failed: %+v", provider)
} }
backend := loaded.result.Config.Backends[0] backend := loaded.backendRecords["local.v1"]
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 { if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 {
t.Fatalf("explicit zero queue capacity was lost: %+v", backend) t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
} }
@@ -503,13 +444,11 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
if !exists || collectionProfile.ID != "base/namespaced" || collectionProfile.ResolvedTarget == nil || collectionProfile.ResolvedTarget.BackendID != "base/local.v1" { if !exists || collectionProfile.ID != "base/namespaced" || collectionProfile.ResolvedTarget == nil || collectionProfile.ResolvedTarget.BackendID != "base/local.v1" {
t.Fatalf("collection profile snapshot = %+v, exists %t", collectionProfile, exists) t.Fatalf("collection profile snapshot = %+v, exists %t", collectionProfile, exists)
} }
if loaded.result.Config.Collections["invoice"].Prompts[0].Messages[0].Content == nil { if loaded.prompts["invoice/summary"].definition.Templates[0].Content != "Summarize." {
t.Fatal("literal prompt content was not decoded") t.Fatal("literal prompt content was not decoded")
} }
if loaded.result.Config.Version == nil || *loaded.result.Config.Version != "2" || if len(loaded.collections) != 2 || loaded.collections[0].ReleaseVersion == nil || *loaded.collections[0].ReleaseVersion != "" || loaded.collections[1].ReleaseVersion == nil || *loaded.collections[1].ReleaseVersion != "r7" {
loaded.result.Config.Collections["base"].ReleaseVersion == nil || *loaded.result.Config.Collections["base"].ReleaseVersion != "" || t.Fatalf("collection version metadata was not retained: %+v", loaded.collections)
loaded.result.Config.Collections["invoice"].ReleaseVersion == nil || *loaded.result.Config.Collections["invoice"].ReleaseVersion != "r7" {
t.Fatalf("root or collection version metadata was not retained: %+v", loaded.result.Config)
} }
if _, _, err := loaded.prompt("invoice/version"); err != nil { if _, _, err := loaded.prompt("invoice/version"); err != nil {
t.Fatalf("resource named version was not indexed under its qualified identity: %v", err) t.Fatalf("resource named version was not indexed under its qualified identity: %v", err)
@@ -574,25 +513,73 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
backend.APIKeyEnv != "TOKEN_NAME" || backend.QueueCapacity != 1024 { backend.APIKeyEnv != "TOKEN_NAME" || backend.QueueCapacity != 1024 {
t.Fatalf("backend normalization/default was not applied: %+v", backend) t.Fatalf("backend normalization/default was not applied: %+v", backend)
} }
composed := loaded.result.Config.Backends[0] composed := loaded.backendRecords["local"]
if composed.Endpoint == nil || *composed.Endpoint != "http://localhost:8080/v1" || if composed.Endpoint == nil || *composed.Endpoint != "http://localhost:8080/v1" ||
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 { composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
t.Fatalf("resolved backend was not normalized: %+v", composed) t.Fatalf("resolved backend was not normalized: %+v", composed)
} }
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint")) field, found := loaded.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("endpoint"))
if !found || field.Origin.Operation != configkit.ChangeNormalization { if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("endpoint origin was not preserved: %+v", field) t.Fatalf("endpoint origin was not preserved: %+v", field)
} }
field, found = loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("queue_capacity")) field, found = loaded.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("queue_capacity"))
if !found || field.Origin.Operation != configkit.ChangeDerivedDefault { if !found || field.Origin.Operation != configkit.ChangeDerivedDefault {
t.Fatalf("queue capacity default was not recorded: %+v", field) t.Fatalf("queue capacity default was not recorded: %+v", field)
} }
field, found = loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("concurrency_limit")) field, found = loaded.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("concurrency_limit"))
if !found || field.Origin.Operation == configkit.ChangeNormalization || field.Origin.Operation == configkit.ChangeDerivedDefault { if !found || field.Origin.Operation == configkit.ChangeNormalization || field.Origin.Operation == configkit.ChangeDerivedDefault {
t.Fatalf("unchanged concurrency acquired a resolution record: %+v", field) t.Fatalf("unchanged concurrency acquired a resolution record: %+v", field)
} }
} }
func TestBackendProviderFinalConfiguration(t *testing.T) {
underlays := []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": "bad", "zero_cost": true}},
})}}}
main := mapping("override", map[string]any{"backends": []any{
map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": " OPENROUTER ", "zero_cost": false, "smoke_tests": map[string]any{"authentication": map[string]any{}}},
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}},
map[string]any{"id": "cleared", "extends": map[string]any{"id": "base"}, "provider": nil, "zero_cost": nil},
map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1"},
}})
state, err := load(main, underlays)
if err != nil {
var detail *configkit.LoadError
if errors.As(err, &detail) {
t.Fatalf("%v: %+v", err, detail.Diagnostics())
}
t.Fatal(err)
}
for _, name := range []string{"base", "child"} {
b := state.backendRecords[name]
if b.Provider == nil || *b.Provider != "openrouter" || b.ZeroCost == nil || *b.ZeroCost {
t.Fatalf("%s: %+v", name, b)
}
}
for _, name := range []string{"cleared", "generic"} {
b := state.backendRecords[name]
if b.Provider != nil || b.ZeroCost != nil {
t.Fatalf("%s: %+v", name, b)
}
}
field, found := state.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("provider"))
if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("provider normalization: %+v", field)
}
for _, provider := range []string{"openai-compatible", "openrouter", "vllm", " ", ""} {
_, err := load(mapping("provider", map[string]any{"backends": []any{map[string]any{"id": "x", "endpoint": "https://example.invalid/v1", "provider": provider, "smoke_tests": map[string]any{"authentication": map[string]any{"kind": " VLLM_MODELS "}}}}}), nil)
if err != nil {
t.Errorf("provider %q: %v", provider, err)
}
}
for _, fields := range []map[string]any{{"provider": "unsupported"}, {"zero_cost": "false"}} {
fields["id"], fields["endpoint"] = "x", "https://example.invalid/v1"
if _, err := load(mapping("invalid", map[string]any{"backends": []any{fields}}), nil); err == nil {
t.Fatalf("accepted invalid fields: %v", fields)
}
}
}
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) { func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
name string name string
@@ -624,7 +611,7 @@ func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if got := loaded.result.Config.Backends[0].SmokeTests; got != nil { if got := loaded.backendRecords["local"].SmokeTests; got != nil {
t.Fatalf("smoke_tests = %+v, want nil", got) t.Fatalf("smoke_tests = %+v, want nil", got)
} }
selected, err := loaded.registry.GetBackend("local") selected, err := loaded.registry.GetBackend("local")
@@ -697,32 +684,27 @@ func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
{ {
name: "blank default profile", name: "blank default profile",
data: map[string]any{"default_profile": " \t "}, data: map[string]any{"default_profile": " \t "},
path: "/backends/0/smoke_tests/default_profile", path: "/promptkit/backends/0/smoke_tests/default_profile",
},
{
name: "authentication kind required",
data: map[string]any{"authentication": map[string]any{}},
path: "/backends/0/smoke_tests/authentication/kind",
}, },
{ {
name: "unknown authentication kind", name: "unknown authentication kind",
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}}, data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
path: "/backends/0/smoke_tests/authentication/kind", path: "/promptkit/backends/0/smoke_tests/authentication/kind",
}, },
{ {
name: "non-http reachability URL", name: "non-http reachability URL",
data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}}, data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
path: "/backends/0/smoke_tests/reachability/endpoint", path: "/promptkit/backends/0/smoke_tests/reachability/endpoint",
}, },
{ {
name: "blank authentication URL", name: "blank authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}}, data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}},
path: "/backends/0/smoke_tests/authentication/endpoint", path: "/promptkit/backends/0/smoke_tests/authentication/endpoint",
}, },
{ {
name: "relative authentication URL", name: "relative authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}}, data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
path: "/backends/0/smoke_tests/authentication/endpoint", path: "/promptkit/backends/0/smoke_tests/authentication/endpoint",
}, },
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
@@ -757,7 +739,7 @@ func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
backend, err := loaded.GetBackend("local") backend, err := loaded.GetBackend("local")
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 { if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.backendRecords["local"].QueueCapacity == nil || *loaded.backendRecords["local"].QueueCapacity != 1024 {
t.Fatalf("absent queue capacity did not receive its default: %+v", backend) t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
} }
} }
@@ -769,7 +751,7 @@ func TestLoadCapturesProfilePresence(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
profile := loaded.result.Config.Profiles[0] profile := loaded.profileRecords["local"]
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired { if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
t.Fatalf("profile values were not preserved: %+v", profile) t.Fatalf("profile values were not preserved: %+v", profile)
} }
@@ -799,7 +781,7 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("replaced lower-layer domain value should not fail: %v", err) t.Fatalf("replaced lower-layer domain value should not fail: %v", err)
} }
if loaded.result.Config.Profiles[0].Model != nil { if loaded.profileRecords["partial"].Model != nil {
t.Fatal("unused partial profile unexpectedly gained a model") t.Fatal("unused partial profile unexpectedly gained a model")
} }
@@ -810,7 +792,7 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
if !errors.As(err, &loadError) { if !errors.As(err, &loadError) {
t.Fatalf("expected path-aware effective backend failure, got %v", err) t.Fatalf("expected path-aware effective backend failure, got %v", err)
} }
if diagnostics := loadError.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Path.String() != "/backends/0/endpoint" { if diagnostics := loadError.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Path.String() != "/promptkit/backends/0/endpoint" {
t.Fatalf("expected endpoint diagnostic, got %+v", diagnostics) t.Fatalf("expected endpoint diagnostic, got %+v", diagnostics)
} }
} }

View File

@@ -0,0 +1,147 @@
package configuration
import (
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/settings"
)
func copyPointer[T any](value *T) *T {
if value == nil {
return nil
}
copy := *value
return &copy
}
func copyProviderMap(value map[string]any) map[string]any {
if value == nil {
return nil
}
return jsonvalue.CloneTrusted(value).(map[string]any)
}
func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings {
in.Provider = copyPointer(in.Provider)
in.ZeroCost = copyPointer(in.ZeroCost)
in.Endpoint = copyPointer(in.Endpoint)
in.APIKeyEnv = copyPointer(in.APIKeyEnv)
in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit)
in.QueueCapacity = copyPointer(in.QueueCapacity)
in.ExtraParams = copyProviderMap(in.ExtraParams)
in.Application = nil
if in.SmokeTests != nil {
smoke := *in.SmokeTests
smoke.DefaultProfile = copyPointer(smoke.DefaultProfile)
if smoke.Reachability != nil {
reach := *smoke.Reachability
reach.Endpoint = copyPointer(reach.Endpoint)
smoke.Reachability = &reach
}
if smoke.Authentication != nil {
auth := *smoke.Authentication
auth.Kind = copyPointer(auth.Kind)
auth.Endpoint = copyPointer(auth.Endpoint)
smoke.Authentication = &auth
}
in.SmokeTests = &smoke
}
return in
}
func ownProfileSettings(in settings.ProfileSettings) settings.ProfileSettings {
in.Backend = copyPointer(in.Backend)
in.Endpoint = copyPointer(in.Endpoint)
in.Model = copyPointer(in.Model)
in.Temperature = copyPointer(in.Temperature)
in.TopP = copyPointer(in.TopP)
in.MaxTokens = copyPointer(in.MaxTokens)
in.TimeoutSeconds = copyPointer(in.TimeoutSeconds)
in.ServiceTier = copyPointer(in.ServiceTier)
in.ReasoningEffort = copyPointer(in.ReasoningEffort)
in.APIKeyEnv = copyPointer(in.APIKeyEnv)
in.APIKeyRequired = copyPointer(in.APIKeyRequired)
in.ExtraParams = copyProviderMap(in.ExtraParams)
in.Application = nil
return in
}
func ownPromptSettings(in settings.PromptSettings) settings.PromptSettings {
in.DefaultProfile = copyPointer(in.DefaultProfile)
in.Description = copyPointer(in.Description)
in.SessionID = copyPointer(in.SessionID)
in.Application = nil
if in.Inputs != nil {
in.Inputs = append(make([]settings.PromptInputSettings, 0, len(in.Inputs)), in.Inputs...)
for i := range in.Inputs {
in.Inputs[i].ContentType = copyPointer(in.Inputs[i].ContentType)
in.Inputs[i].Description = copyPointer(in.Inputs[i].Description)
}
}
if in.Messages != nil {
in.Messages = append(make([]settings.PromptMessageSettings, 0, len(in.Messages)), in.Messages...)
for i := range in.Messages {
message := &in.Messages[i]
message.Role = copyPointer(message.Role)
message.Content = copyPointer(message.Content)
message.ContentFile = copyPointer(message.ContentFile)
if message.CacheControl != nil {
cache := *message.CacheControl
cache.Type = copyPointer(cache.Type)
cache.TTL = copyPointer(cache.TTL)
message.CacheControl = &cache
}
}
}
if in.Output != nil {
output := *in.Output
output.Format = copyPointer(output.Format)
output.ValidationMode = copyPointer(output.ValidationMode)
output.Schema = jsonvalue.CloneTrusted(output.Schema)
output.SchemaFile = copyPointer(output.SchemaFile)
output.SchemaPath = copyPointer(output.SchemaPath)
output.RepairAttempts = copyPointer(output.RepairAttempts)
in.Output = &output
}
return in
}
func ownSettings(in *settings.Settings) settings.Settings {
if in == nil {
return settings.Settings{}
}
out := settings.Settings{SchemaRoot: copyPointer(in.SchemaRoot)}
copyResources := func(backends []settings.BackendSettings, profiles []settings.ProfileSettings, prompts []settings.PromptSettings) ([]settings.BackendSettings, []settings.ProfileSettings, []settings.PromptSettings) {
var b []settings.BackendSettings
if backends != nil {
b = make([]settings.BackendSettings, len(backends))
for i := range backends {
b[i] = ownBackendSettings(backends[i])
}
}
var p []settings.ProfileSettings
if profiles != nil {
p = make([]settings.ProfileSettings, len(profiles))
for i := range profiles {
p[i] = ownProfileSettings(profiles[i])
}
}
var q []settings.PromptSettings
if prompts != nil {
q = make([]settings.PromptSettings, len(prompts))
for i := range prompts {
q[i] = ownPromptSettings(prompts[i])
}
}
return b, p, q
}
out.Backends, out.Profiles, out.Prompts = copyResources(in.Backends, in.Profiles, in.Prompts)
if in.Collections != nil {
out.Collections = make(map[string]settings.CollectionSettings, len(in.Collections))
for name, collection := range in.Collections {
copy := settings.CollectionSettings{ReleaseVersion: copyPointer(collection.ReleaseVersion), SchemaRoot: copyPointer(collection.SchemaRoot)}
copy.Backends, copy.Profiles, copy.Prompts = copyResources(collection.Backends, collection.Profiles, collection.Prompts)
out.Collections[name] = copy
}
}
return out
}

View File

@@ -11,8 +11,10 @@ import (
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/filecatalog" "go.maximumdirect.net/promptkit/internal/filecatalog"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/prompt" "go.maximumdirect.net/promptkit/internal/prompt"
"go.maximumdirect.net/promptkit/internal/promptdef" "go.maximumdirect.net/promptkit/internal/promptdef"
"go.maximumdirect.net/promptkit/internal/settings"
) )
var ( var (
@@ -29,24 +31,36 @@ type promptEntry struct {
messages []*template.Template messages []*template.Template
} }
func (state *loaded) materializePrompts() error { func (state *loaded) materializePrompts(config *settings.Settings, metadata configkit.Metadata) error {
state.prompts = make(map[string]*promptEntry) state.prompts = make(map[string]*promptEntry)
var metadata *configkit.Metadata
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) { lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
if metadata == nil {
captured := state.result.Metadata()
metadata = &captured
}
return metadata.Lookup(path) return metadata.Lookup(path)
} }
for _, resources := range resourceSets(&state.result.Config) { applicationRoot, err := state.captureSchemaRoot(config.SchemaRoot, configkit.Keys("promptkit", "schema_root"), lookupOrigin)
if err != nil {
return err
}
roots := make(map[string]*domain.ResourceBinding, len(config.Collections))
for _, name := range sortedKeys(config.Collections) {
collection := config.Collections[name]
root, err := state.captureSchemaRoot(collection.SchemaRoot, configkit.Keys("promptkit", "collections", name, "schema_root"), lookupOrigin)
if err != nil {
return err
}
roots[name] = root
}
for _, resources := range resourceSets(config) {
root := applicationRoot
if selected := roots[resources.namespace]; resources.isCollection && selected != nil {
root = selected
}
for index, raw := range resources.prompts { for index, raw := range resources.prompts {
promptBase := resources.bases.prompts.Index(index) promptBase := resources.bases.prompts.Index(index)
if strings.TrimSpace(raw.ID) == "" { if strings.TrimSpace(raw.ID) == "" {
return fmt.Errorf("%s: prompt ID must not be blank", promptBase) return fmt.Errorf("%s: prompt ID must not be blank", promptBase)
} }
fullID := indexedResourceID(resources.namespace, raw.ID) fullID := indexedResourceID(resources.namespace, raw.ID)
entry, err := state.materializePrompt(fullID, raw, promptBase, lookupOrigin) entry, err := state.materializePrompt(fullID, raw, promptBase, root, lookupOrigin)
if err != nil { if err != nil {
return err return err
} }
@@ -56,11 +70,19 @@ func (state *loaded) materializePrompts() error {
return nil return nil
} }
func (state *loaded) materializePrompt(id string, raw promptConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) { func (state *loaded) captureSchemaRoot(root *string, fieldPath configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*domain.ResourceBinding, error) {
if root == nil {
return nil, nil
}
bound, _, err := state.bindResource(fieldPath, *root, "schema_root", lookupOrigin)
return bound, err
}
func (state *loaded) materializePrompt(id string, raw promptConfig, base configkit.Path, root *domain.ResourceBinding, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
if len(raw.Messages) == 0 { if len(raw.Messages) == 0 {
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages")) return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
} }
definition := domain.PromptDefinition{ID: id} definition := domain.PromptDefinition{ID: id, SchemaRoot: root}
if raw.DefaultProfile != nil { if raw.DefaultProfile != nil {
if strings.TrimSpace(*raw.DefaultProfile) == "" { if strings.TrimSpace(*raw.DefaultProfile) == "" {
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile")) return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
@@ -89,6 +111,22 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
definition.Templates = messages definition.Templates = messages
entry := &promptEntry{resource: raw, requirementInputs: inputMetadata, messages: compiledMessages} entry := &promptEntry{resource: raw, requirementInputs: inputMetadata, messages: compiledMessages}
definition.Validation = contract definition.Validation = contract
if contract.ValidationMode == domain.ValidationJSONSchema {
fieldPath := base.Key("output")
if contract.SchemaFile != nil {
definition.SchemaFile, _, err = state.bindResource(fieldPath.Key("schema_file"), *contract.SchemaFile, "schema_file", lookupOrigin)
if err != nil {
return nil, err
}
} else if contract.SchemaPath != "" {
if root == nil {
return nil, fmt.Errorf("%s: schema_path requires an applicable schema_root", fieldPath.Key("schema_path"))
}
if _, _, err := filecatalog.ResolveFSPath(root.Path, root.Path, contract.SchemaPath); err != nil {
return nil, fmt.Errorf("%s: invalid schema_path: %w", fieldPath.Key("schema_path"), err)
}
}
}
if raw.SessionID != nil { if raw.SessionID != nil {
var err error var err error
entry.session, err = prompt.CompileTemplate("session_id", definition.SessionID) entry.session, err = prompt.CompileTemplate("session_id", definition.SessionID)
@@ -105,7 +143,6 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
entry.inspection = domain.PromptInspection{ entry.inspection = domain.PromptInspection{
PromptID: id, PromptHash: hash, PromptID: id, PromptHash: hash,
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...), DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
OutputContract: contract,
} }
return entry, nil return entry, nil
} }
@@ -134,12 +171,11 @@ func (state *loaded) capturePromptInputs(id string, rawInputs []promptInputConfi
inputs = append(inputs, domain.PromptInput{ inputs = append(inputs, domain.PromptInput{
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description, Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
}) })
presence := state.promptMetadata[id][index]
item := prompt.InputMetadata{Name: input.Name, Required: input.Required} item := prompt.InputMetadata{Name: input.Name, Required: input.Required}
if presence.contentType == fieldValue { if input.ContentType != nil {
item.ContentType = stringPointer(contentType) item.ContentType = stringPointer(contentType)
} }
if presence.description == fieldValue { if input.Description != nil {
item.Description = stringPointer(description) item.Description = stringPointer(description)
} }
metadata = append(metadata, item) metadata = append(metadata, item)
@@ -206,9 +242,14 @@ func capturePromptOutput(output *promptOutputConfig, base configkit.Path) (domai
contract := domain.OutputContract{ contract := domain.OutputContract{
Format: domain.OutputFormat(*output.Format), Format: domain.OutputFormat(*output.Format),
ValidationMode: domain.ValidationMode(*output.ValidationMode), ValidationMode: domain.ValidationMode(*output.ValidationMode),
RepairAttempts: output.RepairAttempts, RepairAttempts: domain.ResolveRepairAttempts(domain.ValidationMode(*output.ValidationMode), output.RepairAttempts),
Schema: jsonvalue.CloneTrusted(output.Schema),
SchemaFile: stringPointerValue(output.SchemaFile),
} }
if output.SchemaPath != nil { if output.SchemaPath != nil {
if contract.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(*output.SchemaPath) == "" {
return domain.OutputContract{}, fmt.Errorf("%s: schema_path must not be blank", base.Key("output").Key("schema_path"))
}
contract.SchemaPath = *output.SchemaPath contract.SchemaPath = *output.SchemaPath
} }
if err := domain.ValidateOutputContract(contract); err != nil { if err := domain.ValidateOutputContract(contract); err != nil {
@@ -229,24 +270,32 @@ func stringPointerValue(value *string) *string {
return stringPointer(*value) return stringPointer(*value)
} }
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) { func (state *loaded) bindResource(fieldPath configkit.Path, name, label string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*domain.ResourceBinding, string, error) {
if strings.TrimSpace(name) == "" { if strings.TrimSpace(name) == "" {
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath) return nil, "", fmt.Errorf("%s: %s path must not be blank", fieldPath, label)
} }
field, found := lookupOrigin(fieldPath) field, found := lookupOrigin(fieldPath)
if !found || len(field.Origin.Locations) != 1 { if !found || len(field.Origin.Locations) != 1 {
return "", fmt.Errorf("%s: content_file needs one contributing source with a filesystem context", fieldPath) return nil, "", fmt.Errorf("%s: %s needs one contributing source with a filesystem context", fieldPath, label)
} }
sourceID := field.Origin.Locations[0].SourceID sourceID := field.Origin.Locations[0].SourceID
context, found := state.result.SourceContext(sourceID) context, found := state.view.SourceContext(sourceID)
if !found || context.FS == nil { if !found || context.FS == nil {
return "", fmt.Errorf("%s: content_file source %s has no filesystem context", fieldPath, sourceID) return nil, "", fmt.Errorf("%s: %s source %s has no filesystem context", fieldPath, label, sourceID)
} }
resolved, _, err := filecatalog.ResolveFSPath(".", context.BaseDir, name) resolved, _, err := filecatalog.ResolveFSPath(".", context.BaseDir, name)
if err != nil { if err != nil {
return "", fmt.Errorf("%s: content_file path is outside its source root or invalid: %w", fieldPath, err) return nil, "", fmt.Errorf("%s: %s path is outside its source root or invalid: %w", fieldPath, label, err)
} }
file, err := context.FS.Open(resolved) return &domain.ResourceBinding{FS: context.FS, Path: resolved}, sourceID, nil
}
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
binding, sourceID, err := state.bindResource(fieldPath, name, "content_file", lookupOrigin)
if err != nil {
return "", err
}
file, err := binding.FS.Open(binding.Path)
if err != nil { if err != nil {
return "", fmt.Errorf("%s: open content_file from source %s: %w", fieldPath, sourceID, err) return "", fmt.Errorf("%s: open content_file from source %s: %w", fieldPath, sourceID, err)
} }
@@ -273,6 +322,15 @@ func (state *loaded) lookupPromptEntry(id string) (*promptEntry, error) {
} }
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition { func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
definition.Validation = ownOutputContract(definition.Validation)
if definition.SchemaRoot != nil {
root := *definition.SchemaRoot
definition.SchemaRoot = &root
}
if definition.SchemaFile != nil {
file := *definition.SchemaFile
definition.SchemaFile = &file
}
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...) definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...) definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
for index := range definition.Templates { for index := range definition.Templates {
@@ -289,6 +347,12 @@ func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspe
return &inspection return &inspection
} }
func ownOutputContract(contract domain.OutputContract) domain.OutputContract {
contract.Schema = jsonvalue.CloneTrusted(contract.Schema)
contract.SchemaFile = stringPointerValue(contract.SchemaFile)
return contract
}
func promptLookupError(err error) error { func promptLookupError(err error) error {
if errors.Is(err, errPromptNotFound) { if errors.Is(err, errPromptNotFound) {
return promptdef.ErrPromptDefinitionNotFound return promptdef.ErrPromptDefinitionNotFound

View File

@@ -3,6 +3,7 @@ package configuration
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
@@ -50,26 +51,18 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" { if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" {
t.Fatalf("message metadata was not normalized: %+v", message) t.Fatalf("message metadata was not normalized: %+v", message)
} }
childIndex := -1 childIndex := 1
for index, item := range state.result.Config.Collections["docs"].Prompts { composed := state.prompts["docs/child"].resource
if item.ID == "child" {
childIndex = index
}
}
if childIndex < 0 {
t.Fatal("inherited prompt missing from effective list")
}
composed := state.result.Config.Collections["docs"].Prompts[childIndex]
if *composed.Description != "description" || *composed.SessionID != "session" || composed.Inputs[0].Name != "document" || if *composed.Description != "description" || *composed.SessionID != "session" || composed.Inputs[0].Name != "document" ||
*composed.Inputs[0].ContentType != "text/plain" || *composed.Inputs[0].Description != "document source" { *composed.Inputs[0].ContentType != "text/plain" || *composed.Inputs[0].Description != "document source" {
t.Fatalf("resolved metadata was not normalized: %+v", composed) t.Fatalf("resolved metadata was not normalized: %+v", composed)
} }
field, ok := state.result.Metadata().Lookup(configkit.Keys("collections", "docs", "prompts").Index(childIndex).Key("messages").Index(0).Key("content_file")) field, ok := state.inspection.Metadata().Lookup(configkit.Keys("promptkit", "collections", "docs", "prompts").Index(childIndex).Key("messages").Index(0).Key("content_file"))
if !ok || len(field.Origin.Locations) != 1 { if !ok || len(field.Origin.Locations) != 1 {
t.Fatalf("content origin was lost during normalization: %+v", field) t.Fatalf("content origin was lost during normalization: %+v", field)
} }
var originPath string var originPath string
for _, source := range state.result.Sources() { for _, source := range state.inspection.Sources() {
if source.ID == field.Origin.Locations[0].SourceID { if source.ID == field.Origin.Locations[0].SourceID {
originPath = source.Path originPath = source.Path
} }
@@ -117,11 +110,93 @@ func TestInspectPromptResourcePresence(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts == nil || *got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil { if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
t.Fatalf("prompt resource = %+v", got) t.Fatalf("prompt resource = %+v", got)
} }
} }
func TestConfiguredRepairBudgets(t *testing.T) {
for _, tc := range []struct {
name, mode string
budget any
include bool
want int
invalid bool
}{
{name: "none omitted", mode: "none", want: 0},
{name: "basic omitted", mode: "basic", want: 1},
{name: "json omitted", mode: "json", want: 1},
{name: "schema omitted", mode: "json_schema", want: 1},
{name: "null", mode: "json", include: true, budget: nil, want: 1},
{name: "zero", mode: "basic", include: true, budget: 0, want: 0},
{name: "two", mode: "basic", include: true, budget: 2, want: 2},
{name: "three", mode: "basic", include: true, budget: 3, want: 3},
{name: "negative", mode: "basic", include: true, budget: -1, invalid: true},
{name: "excess", mode: "basic", include: true, budget: 4, invalid: true},
{name: "none positive", mode: "none", include: true, budget: 1, invalid: true},
{name: "unknown mode", mode: "unknown", invalid: true},
} {
t.Run(tc.name, func(t *testing.T) {
output := map[string]any{"format": "text", "validation_mode": tc.mode}
if tc.mode == "json_schema" {
output["schema"] = true
}
if tc.include {
output["repair_attempts"] = tc.budget
}
state, err := load(mapping("repair", map[string]any{"prompts": []any{map[string]any{
"id": "test", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output,
}}}), nil)
if tc.invalid {
if err == nil {
t.Fatal("expected invalid output contract")
}
return
}
if err != nil {
t.Fatal(err)
}
definition, _, err := state.prompt("test")
if err != nil || definition.Validation.RepairAttempts != tc.want {
t.Fatalf("captured budget: %+v, %v", definition, err)
}
report, err := state.InspectPrompt(context.Background(), "test")
if err != nil || report.Output.RepairAttempts != tc.want {
t.Fatalf("inspection: %+v, %v", report, err)
}
var effective strings.Builder
if err := state.inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), fmt.Sprintf("repair_attempts: %d", tc.want)) {
t.Fatalf("effective inspection = %q, %v", effective.String(), err)
}
})
}
}
func TestNullClearsInheritedRepairBudget(t *testing.T) {
state, err := load(mapping("inherited repair", map[string]any{"prompts": []any{
map[string]any{"id": "parent", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 3}},
map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "output": map[string]any{"repair_attempts": nil}},
}}), nil)
if err != nil {
t.Fatal(err)
}
child, _, err := state.prompt("child")
if err != nil || child.Validation.RepairAttempts != 1 {
t.Fatalf("child = %+v, %v", child, err)
}
}
func TestYAMLRepairBudgetNull(t *testing.T) {
state, err := load(configkit.Input{Source: configkit.Bytes("prompt", []byte("prompts:\n - id: example\n messages:\n - role: user\n content: hello\n output:\n format: text\n validation_mode: basic\n repair_attempts: null\n"))}, nil)
if err != nil {
t.Fatal(err)
}
definition, _, err := state.prompt("example")
if err != nil || definition.Validation.RepairAttempts != 1 {
t.Fatalf("prompt = %+v, %v", definition, err)
}
}
func TestLoadHeaderlessImportedItem(t *testing.T) { func TestLoadHeaderlessImportedItem(t *testing.T) {
files := fstest.MapFS{ files := fstest.MapFS{
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")}, "main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
@@ -157,7 +232,7 @@ func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err) t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err)
} }
_, err = load(mapping("contextless", promptWithFile("body.md")), nil) _, err = load(mapping("contextless", promptWithFile("body.md")), nil)
if err == nil || !strings.Contains(err.Error(), "/prompts/0/messages/0/content_file") { if err == nil || !strings.Contains(err.Error(), "/promptkit/prompts/0/messages/0/content_file") {
t.Fatalf("expected semantic path for contextless resource, got %v", err) t.Fatalf("expected semantic path for contextless resource, got %v", err)
} }
} }
@@ -287,14 +362,17 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
if _, _, err := state.prompt("exact.id"); !errors.Is(err, errPromptNotFound) { if _, _, err := state.prompt("exact.id"); !errors.Is(err, errPromptNotFound) {
t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err) t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err)
} }
_, err = load(mapping("obsolete-field", map[string]any{"prompts": []any{map[string]any{ extended, err := load(mapping("application-field", map[string]any{"prompts": []any{map[string]any{
"version": "v1", "version": "v1",
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}}, "id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"}, "output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}), nil) }}}), nil)
var loadError *configkit.LoadError if err != nil {
if !errors.As(err, &loadError) { t.Fatal(err)
t.Fatalf("obsolete identity field should fail strict decoding: %v", err) }
var effective strings.Builder
if err := extended.inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version: v1") {
t.Fatalf("application field not retained: %q, %v", effective.String(), err)
} }
_, err = load(mapping("null-prompt", map[string]any{"prompts": []any{nil}}), nil) _, err = load(mapping("null-prompt", map[string]any{"prompts": []any{nil}}), nil)
if err == nil { if err == nil {

View File

@@ -40,6 +40,10 @@ func resolvePrompts(entries []promptConfig, base configkit.Path, resolution *con
if entry.Output != nil && entry.Output.SchemaPath != nil { if entry.Output != nil && entry.Output.SchemaPath != nil {
recordStringNormalization(&entry.Output.SchemaPath, path.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*entry.Output.SchemaPath)) recordStringNormalization(&entry.Output.SchemaPath, path.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*entry.Output.SchemaPath))
} }
if output := entry.Output; output != nil && output.ValidationMode != nil && output.RepairAttempts == nil {
attempts := domain.ResolveRepairAttempts(domain.ValidationMode(*output.ValidationMode), nil)
configkit.Normalize(resolution, path.Key("output").Key("repair_attempts"), &output.RepairAttempts, &attempts, "Resolve repair budget")
}
for j := range entry.Messages { for j := range entry.Messages {
message := &entry.Messages[j] message := &entry.Messages[j]
messagePath := path.Key("messages").Index(j) messagePath := path.Key("messages").Index(j)

View File

@@ -15,6 +15,9 @@ func resolveBackends(entries []backendConfig, base configkit.Path, resolution *c
for i := range entries { for i := range entries {
entry := &entries[i] entry := &entries[i]
path := base.Index(i) path := base.Index(i)
if entry.Provider != nil {
recordStringNormalization(&entry.Provider, path.Key("provider"), resolution, strings.ToLower(strings.TrimSpace(*entry.Provider)))
}
if entry.Endpoint != nil { if entry.Endpoint != nil {
if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil { if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil {
recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized) recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized)

View File

@@ -26,6 +26,8 @@ type ProfileResource struct {
// Its fields borrow immutable configuration captured by the adapter. // Its fields borrow immutable configuration captured by the adapter.
type BackendResource struct { type BackendResource struct {
ID, Endpoint string ID, Endpoint string
Provider *string
ZeroCost *bool
APIKeyEnv *string APIKeyEnv *string
ConcurrencyLimit, QueueCapacity *int ConcurrencyLimit, QueueCapacity *int
ExtraParams map[string]any ExtraParams map[string]any
@@ -60,8 +62,10 @@ type PromptCacheControlResource struct {
type PromptOutputResource struct { type PromptOutputResource struct {
Format domain.OutputFormat Format domain.OutputFormat
ValidationMode domain.ValidationMode ValidationMode domain.ValidationMode
Schema any
SchemaFile *string
SchemaPath *string SchemaPath *string
RepairAttempts *int RepairAttempts int
} }
func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) { func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) {
@@ -76,10 +80,10 @@ func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResou
result := &PromptResource{ID: id, Hash: entry.hash, DefaultProfile: raw.DefaultProfile, Description: raw.Description, SessionID: raw.SessionID} result := &PromptResource{ID: id, Hash: entry.hash, DefaultProfile: raw.DefaultProfile, Description: raw.Description, SessionID: raw.SessionID}
if raw.Inputs != nil { if raw.Inputs != nil {
inputs := make([]PromptInputResource, len(raw.Inputs)) inputs := make([]PromptInputResource, len(raw.Inputs))
presence := state.promptMetadata[id] presence := state.promptRequired[id]
for i, input := range raw.Inputs { for i, input := range raw.Inputs {
inputs[i] = PromptInputResource{Name: input.Name, ContentType: input.ContentType, Description: input.Description} inputs[i] = PromptInputResource{Name: input.Name, ContentType: input.ContentType, Description: input.Description}
if presence[i].required == fieldValue { if presence[i] {
required := input.Required required := input.Required
inputs[i].Required = &required inputs[i].Required = &required
} }
@@ -93,11 +97,7 @@ func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResou
result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL} result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL}
} }
} }
result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, SchemaPath: raw.Output.SchemaPath} result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, Schema: entry.definition.Validation.Schema, SchemaFile: raw.Output.SchemaFile, SchemaPath: raw.Output.SchemaPath, RepairAttempts: entry.definition.Validation.RepairAttempts}
if state.promptRepair[id] {
value := raw.Output.RepairAttempts
result.Output.RepairAttempts = &value
}
return result, nil return result, nil
} }
@@ -133,7 +133,7 @@ func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendRes
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id) return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
} }
return &BackendResource{ return &BackendResource{
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, Provider: config.Provider, ZeroCost: config.ZeroCost,
ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity, ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity,
ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests, ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests,
}, nil }, nil

View File

@@ -0,0 +1,154 @@
package configuration
import (
"context"
"io/fs"
"strings"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
)
func schemaPrompt(id string, output map[string]any) map[string]any {
return map[string]any{"id": id, "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output}
}
func TestLightweightPromptSelectionKeepsSchemaInResourceView(t *testing.T) {
schema := map[string]any{"type": "object", "examples": make([]any, 1024)}
definition := schemaPrompt("selected", map[string]any{"format": "json", "validation_mode": "json_schema", "schema": schema})
definition["default_profile"] = "partial"
state, err := load(mapping("selection", map[string]any{
"profiles": []any{map[string]any{"id": "partial"}},
"prompts": []any{definition},
}), nil)
if err != nil {
t.Fatal(err)
}
selection, err := state.GetPromptInspection(context.Background(), "selected")
if err != nil || selection.PromptID != "selected" || selection.DefaultProfileID != "partial" {
t.Fatalf("selection = %+v, %v", selection, err)
}
// The selection type has no output contract; the full resource view still does.
resource, err := state.InspectPrompt(context.Background(), "selected")
if err != nil || resource.Output.Schema.(map[string]any)["type"] != "object" {
t.Fatalf("resource schema = %+v, %v", resource, err)
}
}
func TestSchemaFileCaptureUsesEffectiveSource(t *testing.T) {
first := configkit.NewFilesystem("first", fstest.MapFS{"a/placeholder": {Data: []byte("x")}, "a/schema.json": {Data: []byte("first")}})
second := configkit.NewFilesystem("second", fstest.MapFS{"b/placeholder": {Data: []byte("x")}, "b/schema.json": {Data: []byte("second")}})
base := configkit.Input{Source: configkit.Mapping("base", map[string]any{"prompts": []any{
schemaPrompt("base", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "schema.json"}),
map[string]any{"id": "inherited", "extends": map[string]any{"id": "base"}},
map[string]any{"id": "override", "extends": map[string]any{"id": "base"}},
}}), Filesystem: &configkit.FilesystemContext{FS: &first, Root: ".", BaseDir: "a"}}
overlay := configkit.Input{Source: configkit.Mapping("override", map[string]any{"prompts": []any{
map[string]any{"id": "override", "extends": map[string]any{"id": "base"}, "output": map[string]any{"schema_file": "schema.json"}},
schemaPrompt("missing", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_file": "missing.json"}),
}}), Filesystem: &configkit.FilesystemContext{FS: &second, Root: ".", BaseDir: "b"}}
state, err := load(overlay, []configkit.Underlay{{Inputs: []configkit.Input{base}}})
if err != nil {
t.Fatal(err)
}
for id, want := range map[string]string{"base": "first", "inherited": "first", "override": "second"} {
definition, _, err := state.prompt(id)
if err != nil {
t.Fatal(err)
}
data, err := fs.ReadFile(definition.SchemaFile.FS, definition.SchemaFile.Path)
if err != nil || string(data) != want {
t.Fatalf("%s = %q, %v", id, data, err)
}
}
missing, err := state.InspectPrompt(context.Background(), "missing")
if err != nil || missing.Output.SchemaFile == nil || *missing.Output.SchemaFile != "missing.json" {
t.Fatalf("missing inspection = %+v, %v", missing, err)
}
}
func TestSchemaInlineCompositionAndSelection(t *testing.T) {
base := mapping("base", map[string]any{"prompts": []any{
schemaPrompt("inline-base", map[string]any{"format": "json", "validation_mode": "json_schema", "schema": map[string]any{"type": "object", "properties": map[string]any{"a": map[string]any{"type": "string"}}}}),
schemaPrompt("switch-base", map[string]any{"format": "json", "validation_mode": "none", "schema_file": "unused.json"}),
}})
main := mapping("override", map[string]any{"prompts": []any{
map[string]any{"id": "inline", "extends": map[string]any{"id": "inline-base"}, "output": map[string]any{"schema": map[string]any{"properties": map[string]any{"b": map[string]any{"type": "number"}}}}},
map[string]any{"id": "switch", "extends": map[string]any{"id": "switch-base"}, "output": map[string]any{"validation_mode": "json_schema", "schema_file": nil, "schema": false}},
}})
state, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{base}}})
if err != nil {
t.Fatal(err)
}
definition, inspection, err := state.prompt("inline")
if err != nil {
t.Fatal(err)
}
properties := definition.Validation.Schema.(map[string]any)["properties"].(map[string]any)
if len(properties) != 2 || properties["a"] == nil || properties["b"] == nil {
t.Fatalf("merged properties: %#v", properties)
}
properties["a"] = false
again, _, err := state.prompt("inline")
if err != nil || again.Validation.Schema.(map[string]any)["properties"].(map[string]any)["a"] == false {
t.Fatalf("schema not detached: %+v, %v", again, err)
}
if inspection.PromptHash == "" {
t.Fatal("missing definition hash")
}
switched, _, err := state.prompt("switch")
if err != nil || switched.Validation.Schema != false || switched.Validation.SchemaFile != nil {
t.Fatalf("switch = %+v, %v", switched, err)
}
}
func TestSchemaSelectionMetadataFailures(t *testing.T) {
for _, tc := range []struct {
name string
output map[string]any
want string
}{
{"missing root", map[string]any{"schema_path": "x.json"}, "schema_root"},
{"missing context", map[string]any{"schema_file": "x.json"}, "filesystem context"},
{"blank path", map[string]any{"schema": false, "schema_path": " "}, "schema_path"},
{"conflicting", map[string]any{"schema": false, "schema_file": "x.json"}, "exactly one"},
} {
t.Run(tc.name, func(t *testing.T) {
tc.output["format"], tc.output["validation_mode"] = "json", "json_schema"
_, err := load(mapping("prompt", map[string]any{"prompts": []any{schemaPrompt("p", tc.output)}}), nil)
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("error = %v, want %s", err, tc.want)
}
})
}
_, err := load(mapping("inactive", map[string]any{"prompts": []any{schemaPrompt("p", map[string]any{"format": "text", "validation_mode": "none", "schema_file": "../unused", "schema_path": " "})}}), nil)
if err != nil {
t.Fatalf("inactive references: %v", err)
}
}
func TestSchemaSelectionsContributeToDefinitionHash(t *testing.T) {
filesystem := configkit.NewFilesystem("fixtures", fstest.MapFS{})
hash := func(output map[string]any) string {
t.Helper()
output["format"], output["validation_mode"] = "json", "json_schema"
input := mapping("prompt", map[string]any{"prompts": []any{schemaPrompt("p", output)}})
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
state, err := load(input, nil)
if err != nil {
t.Fatal(err)
}
_, inspection, err := state.prompt("p")
if err != nil {
t.Fatal(err)
}
return inspection.PromptHash
}
if hash(map[string]any{"schema": false}) == hash(map[string]any{"schema": true}) {
t.Fatal("distinct inline values produced the same definition hash")
}
if hash(map[string]any{"schema_file": "one.json"}) == hash(map[string]any{"schema_file": "two.json"}) {
t.Fatal("distinct references produced the same definition hash")
}
}

View File

@@ -0,0 +1,102 @@
package configuration
import (
"errors"
"io/fs"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
)
type countingSchemaFS struct {
opens int
files fstest.MapFS
}
func (f *countingSchemaFS) Open(name string) (fs.File, error) {
if name != "." && name != "app" && name != "operator" {
f.opens++
}
return f.files.Open(name)
}
func TestSchemaRootsCaptureEffectiveContexts(t *testing.T) {
first := &countingSchemaFS{files: fstest.MapFS{"app/placeholder": {Data: []byte("application")}}}
second := &countingSchemaFS{files: fstest.MapFS{"operator/placeholder": {Data: []byte("operator")}}}
firstFS := configkit.NewFilesystem("application", first)
secondFS := configkit.NewFilesystem("override", second)
prompt := map[string]any{"id": "p", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}
base := configkit.Input{Source: configkit.Mapping("base", map[string]any{
"schema_root": "schemas", "prompts": []any{prompt},
"collections": map[string]any{"alpha": map[string]any{"schema_root": "old", "prompts": []any{prompt}}, "beta": map[string]any{"schema_root": "old", "prompts": []any{prompt}}},
}), Filesystem: &configkit.FilesystemContext{FS: &firstFS, Root: ".", BaseDir: "app"}}
override := configkit.Input{Source: configkit.Mapping("override", map[string]any{"collections": map[string]any{"alpha": map[string]any{"schema_root": "new"}, "beta": map[string]any{"schema_root": nil}}}), Filesystem: &configkit.FilesystemContext{FS: &secondFS, Root: ".", BaseDir: "operator"}}
state, err := load(override, []configkit.Underlay{{Inputs: []configkit.Input{base}}})
if err != nil {
var loadErr *configkit.LoadError
if errors.As(err, &loadErr) {
t.Fatalf("%v: %+v", err, loadErr.Diagnostics())
}
t.Fatal(err)
}
for id, want := range map[string]string{"p": "app/schemas", "alpha/p": "operator/new", "beta/p": "app/schemas"} {
definition, _, err := state.prompt(id)
if err != nil || definition.SchemaRoot == nil || definition.SchemaRoot.Path != want {
t.Fatalf("%s root = %+v, %v", id, definition, err)
}
}
if first.opens != 0 || second.opens != 0 {
t.Fatalf("root capture opened files: %d, %d", first.opens, second.opens)
}
for id, resource := range map[string]string{"p": "app/placeholder", "alpha/p": "operator/placeholder", "beta/p": "app/placeholder"} {
definition, _, err := state.prompt(id)
if err != nil {
t.Fatal(err)
}
content, err := fs.ReadFile(definition.SchemaRoot.FS, resource)
want := "application"
if id == "alpha/p" {
want = "operator"
}
if err != nil || string(content) != want {
t.Fatalf("%s source = %q, %v", id, content, err)
}
}
}
func TestSchemaRootRequiresValueAndContext(t *testing.T) {
filesystem := configkit.NewFilesystem("fixture", fstest.MapFS{})
for _, tc := range []struct {
name string
root any
context *configkit.FilesystemContext
valid bool
}{
{"absent", nil, nil, true},
{"blank", " ", &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}, false},
{"missing context", "schemas", nil, false},
{"explicit dot", ".", &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}, true},
} {
t.Run(tc.name, func(t *testing.T) {
input := configkit.Input{Source: configkit.Mapping("root", map[string]any{"schema_root": tc.root}), Filesystem: tc.context}
_, err := load(input, nil)
if (err == nil) != tc.valid {
var loadErr *configkit.LoadError
if errors.As(err, &loadErr) {
t.Fatalf("%v: %+v", err, loadErr.Diagnostics())
}
t.Fatalf("root error = %v", err)
}
})
}
}
func TestSchemaRootValidationUsesEffectiveValue(t *testing.T) {
filesystem := configkit.NewFilesystem("fixture", fstest.MapFS{})
underlay := configkit.Input{Source: configkit.Mapping("old", map[string]any{"schema_root": " "})}
main := configkit.Input{Source: configkit.Mapping("new", map[string]any{"schema_root": "."}), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}}
if _, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}); err != nil {
t.Fatalf("overwritten invalid root was validated: %v", err)
}
}

View File

@@ -8,12 +8,14 @@ import (
"go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm"
profilepkg "go.maximumdirect.net/promptkit/internal/profile" profilepkg "go.maximumdirect.net/promptkit/internal/profile"
"go.maximumdirect.net/promptkit/internal/settings"
) )
func (state *loaded) buildExecutionProfiles() error { func (state *loaded) buildExecutionProfiles(config *settings.Settings) error {
backendConfigs := indexedBackends(&state.result.Config) backendConfigs := indexedBackends(config)
profileConfigs := indexedProfiles(&state.result.Config) profileConfigs := indexedProfiles(config)
state.backendRecords = backendConfigs state.backendRecords = backendConfigs
state.profileRecords = profileConfigs state.profileRecords = profileConfigs
definitions := make(map[string]domain.Backend, len(backendConfigs)) definitions := make(map[string]domain.Backend, len(backendConfigs))
@@ -25,7 +27,7 @@ func (state *loaded) buildExecutionProfiles() error {
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs)) state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
for _, id := range sortedKeys(profileConfigs) { for _, id := range sortedKeys(profileConfigs) {
config := profileConfigs[id] config := profileConfigs[id]
profile, err := captureExecutionProfile(id, config, state.registry) profile, err := captureExecutionProfile(id, config, definitions)
if err != nil { if err != nil {
return err return err
} }
@@ -35,7 +37,7 @@ func (state *loaded) buildExecutionProfiles() error {
} }
func captureBackend(id string, config backendConfig) domain.Backend { func captureBackend(id string, config backendConfig) domain.Backend {
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams} definition := domain.Backend{ID: id, Provider: llm.EffectiveProvider(valueOrZero(config.Provider)), ZeroCost: valueOrZero(config.ZeroCost), ExtraParams: config.ExtraParams}
if config.SmokeTests != nil { if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile} smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil { if config.SmokeTests.Reachability != nil {
@@ -64,14 +66,19 @@ func captureBackend(id string, config backendConfig) domain.Backend {
return definition return definition
} }
func captureExecutionProfile(id string, config profileConfig, registry *backend.Registry) (domain.ExecutionProfile, error) { func captureExecutionProfile(id string, config profileConfig, backends map[string]domain.Backend) (domain.ExecutionProfile, error) {
target := defaults.ExecutionTargetDefault() target := defaults.ExecutionTargetDefault()
target.Provider = llm.EffectiveProvider("")
if config.Backend != nil && *config.Backend != "" { if config.Backend != nil && *config.Backend != "" {
selected, err := registry.GetBackend(*config.Backend) // These defaults are capture-owned and immutable. Public and runtime
if err != nil { // lookups still detach mutable values; profile overlays copy before merging.
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w", id, err) selected, found := backends[*config.Backend]
if !found {
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w: %q", id, backend.ErrBackendNotFound, *config.Backend)
} }
target.BackendID = selected.ID target.BackendID = selected.ID
target.Provider = selected.Provider
target.ZeroCost = selected.ZeroCost
target.Endpoint = selected.Endpoint target.Endpoint = selected.Endpoint
target.APIKeyEnv = selected.APIKeyEnv target.APIKeyEnv = selected.APIKeyEnv
target.ExtraParams = selected.ExtraParams target.ExtraParams = selected.ExtraParams

View File

@@ -64,6 +64,45 @@ func TestOverlayProviderMapOwnsInputsAndReplacesValues(t *testing.T) {
} }
} }
func TestProfilesWithSharedDefaultsRemainIsolatedAtLookup(t *testing.T) {
state, err := load(mapping("shared defaults", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.test/v1",
"extra_params": map[string]any{"nested": map[string]any{"value": "backend"}}}},
"profiles": []any{
map[string]any{"id": "first", "backend": "backend"},
map[string]any{"id": "second", "backend": "backend"},
map[string]any{"id": "overlay", "backend": "backend", "extra_params": map[string]any{"nested": map[string]any{"value": "profile"}}},
},
}), nil)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
for _, id := range []string{"first", "second", "overlay"} {
profile, err := state.GetProfile(ctx, id)
if err != nil {
t.Fatal(err)
}
want := "backend"
if id == "overlay" {
want = "profile"
}
if profile.ResolvedTarget.ExtraParams["nested"].(map[string]any)["value"] != want {
t.Fatalf("%s defaults changed: %+v", id, profile)
}
profile.ResolvedTarget.ExtraParams["nested"].(map[string]any)["value"] = "caller mutation"
}
backend, err := state.GetBackend("backend")
if err != nil {
t.Fatal(err)
}
backend.ExtraParams["nested"].(map[string]any)["value"] = "backend caller mutation"
profile, err := state.GetProfile(ctx, "first")
if err != nil || profile.ResolvedTarget.ExtraParams["nested"].(map[string]any)["value"] != "backend" {
t.Fatalf("lookup mutation changed retained defaults: %+v, %v", profile, err)
}
}
func TestRequestCompletesLoadedPartialProfile(t *testing.T) { func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
state, err := load(mapping("partial profile", map[string]any{ state, err := load(mapping("partial profile", map[string]any{
"profiles": []any{map[string]any{"id": "partial"}}, "profiles": []any{map[string]any{"id": "partial"}},
@@ -229,7 +268,7 @@ func TestProfileNullFallbackAndExplicitValues(t *testing.T) {
t.Errorf("%s target = %+v", id, target) t.Errorf("%s target = %+v", id, target)
} }
} }
child := state.result.Config.Profiles[1] child := state.profileRecords["inherited-null"]
if child.Endpoint != nil || child.ExtraParams != nil || child.APIKeyEnv != nil { if child.Endpoint != nil || child.ExtraParams != nil || child.APIKeyEnv != nil {
t.Fatalf("inherited nulls were not retained in resolved values: %+v", child) t.Fatalf("inherited nulls were not retained in resolved values: %+v", child)
} }

View File

@@ -0,0 +1,63 @@
package configuration
import (
"fmt"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/settings"
)
type testDocument struct {
PromptKit settings.Settings `yaml:"promptkit"`
Application map[string]any `yaml:",inline"`
}
type testLoaded struct {
*loaded
inspection configkit.Inspection
}
func testDefinition() configkit.Definition[testDocument] {
return configkit.Definition[testDocument]{
SequenceIdentities: SequenceIdentities(),
Sensitive: Sensitive,
Resolve: func(value *testDocument, resolution *configkit.Resolution) error {
Resolve(&value.PromptKit, resolution)
return nil
},
Validate: func(value *testDocument, validation *configkit.Validation) { Validate(&value.PromptKit, validation) },
}
}
// The adapter fixtures are wrapperless settings fragments. Mount each with
// ConfigKit's native destination and run the same rules and capture as callers.
func load(main configkit.Input, underlays []configkit.Underlay) (*testLoaded, error) {
mount := func(input configkit.Input) configkit.Input {
if input.Destination.String() == "" {
input.Destination = configkit.Keys("promptkit")
}
return input
}
main = mount(main)
selected := make([]configkit.Underlay, len(underlays))
for i, underlay := range underlays {
selected[i] = underlay
selected[i].Inputs = make([]configkit.Input, len(underlay.Inputs))
for j, input := range underlay.Inputs {
selected[i].Inputs[j] = mount(input)
}
}
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[testDocument]{Definition: testDefinition(), Underlays: selected})
if err != nil {
return nil, fmt.Errorf("load configuration: %w", err)
}
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
if !ok {
return nil, fmt.Errorf("inspection unavailable")
}
state, err := Capture(&result.Config.PromptKit, view, result.Diagnostics())
if err != nil {
return nil, fmt.Errorf("capture configuration: %w", err)
}
return &testLoaded{loaded: state, inspection: view}, nil
}

View File

@@ -7,7 +7,6 @@ import (
) )
const ( const (
SchemaDirDefault = "."
OutputArtifactName = "output" OutputArtifactName = "output"
ContentTypeTextPlain = "text/plain" ContentTypeTextPlain = "text/plain"
ContentTypeTextMarkdown = "text/markdown" ContentTypeTextMarkdown = "text/markdown"

17
internal/domain/cost.go Normal file
View File

@@ -0,0 +1,17 @@
package domain
type CostSource string
const (
CostSourceNone CostSource = ""
CostSourceProvider CostSource = "provider"
CostSourceConfiguredZero CostSource = "configured_zero"
CostSourceMixed CostSource = "mixed"
)
type ObservedCost struct {
Value float64
Known bool
Complete bool
Source CostSource
}

View File

@@ -2,6 +2,7 @@ package domain
import ( import (
"context" "context"
"io/fs"
"time" "time"
) )
@@ -73,7 +74,7 @@ type RunRequest struct {
Inputs map[string]ArtifactRef Inputs map[string]ArtifactRef
Vars map[string]string Vars map[string]string
Execution *ExecutionTargetOverride Execution *ExecutionTargetOverride
Validation *OutputContract Validation *OutputContractInput
AppendedMessages []RenderedMessage AppendedMessages []RenderedMessage
TransientRetries *int TransientRetries *int
Observer ExecutionObserver `json:"-"` Observer ExecutionObserver `json:"-"`
@@ -98,6 +99,7 @@ type RunResult struct {
EffectiveModelParams ExecutionTarget EffectiveModelParams ExecutionTarget
InputHashes map[string]string InputHashes map[string]string
Usage TokenUsage Usage TokenUsage
CostUSD ObservedCost
StartTime time.Time StartTime time.Time
EndTime time.Time EndTime time.Time
Duration time.Duration Duration time.Duration
@@ -118,6 +120,7 @@ type PreparedRun struct {
InputHashes map[string]string InputHashes map[string]string
SessionID string SessionID string
RenderedPromptHash string RenderedPromptHash string
MessageContentHash string
Messages []RenderedMessage Messages []RenderedMessage
StartTime time.Time StartTime time.Time
EndTime time.Time EndTime time.Time
@@ -155,6 +158,8 @@ type PromptDefinition struct {
Inputs []PromptInput `yaml:"inputs"` Inputs []PromptInput `yaml:"inputs"`
Templates []PromptMessageTemplate `yaml:"templates"` Templates []PromptMessageTemplate `yaml:"templates"`
Validation OutputContract `yaml:"validation"` Validation OutputContract `yaml:"validation"`
SchemaRoot *ResourceBinding `json:"-" yaml:"-"`
SchemaFile *ResourceBinding `json:"-" yaml:"-"`
} }
// PromptInspection is lightweight metadata for prompt requirement selection. // PromptInspection is lightweight metadata for prompt requirement selection.
@@ -163,7 +168,6 @@ type PromptInspection struct {
PromptHash string PromptHash string
DefaultProfileID string DefaultProfileID string
Inputs []PromptInput Inputs []PromptInput
OutputContract OutputContract
} }
// PromptInput describes one named input expected by a prompt definition. // PromptInput describes one named input expected by a prompt definition.
@@ -185,6 +189,8 @@ type PromptMessageTemplate struct {
// diagnostic metadata. // diagnostic metadata.
type Backend struct { type Backend struct {
ID string ID string
Provider string
ZeroCost bool
Endpoint string Endpoint string
APIKeyEnv string APIKeyEnv string
ExtraParams map[string]any ExtraParams map[string]any
@@ -233,6 +239,8 @@ type ExecutionTargetPresence struct {
// ExecutionTarget represents effective model runtime settings for a run. // ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct { type ExecutionTarget struct {
Provider string `yaml:"-" json:"-"`
ZeroCost bool `yaml:"-" json:"-"`
BackendID string `yaml:"backend" json:"backend_id,omitempty"` BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"` Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"` Model string `yaml:"model" json:"model"`
@@ -252,10 +260,27 @@ type ExecutionTarget struct {
type OutputContract struct { type OutputContract struct {
Format OutputFormat `yaml:"format"` Format OutputFormat `yaml:"format"`
ValidationMode ValidationMode `yaml:"validation_mode"` ValidationMode ValidationMode `yaml:"validation_mode"`
Schema any `yaml:"schema,omitempty" json:"schema,omitempty"`
SchemaFile *string `yaml:"schema_file,omitempty" json:"schema_file,omitempty"`
SchemaPath string `yaml:"schema_path"` SchemaPath string `yaml:"schema_path"`
RepairAttempts int `yaml:"repair_attempts"` RepairAttempts int `yaml:"repair_attempts"`
} }
// OutputContractInput is a request replacement before repair defaults resolve.
type OutputContractInput struct {
Format OutputFormat
ValidationMode ValidationMode
Schema any
SchemaPath string
RepairAttempts *int
}
// ResourceBinding identifies a resource in its originating filesystem.
type ResourceBinding struct {
FS fs.FS
Path string
}
// RenderedPrompt represents the prompt after template application. // RenderedPrompt represents the prompt after template application.
type RenderedPrompt struct { type RenderedPrompt struct {
SessionID string `json:"session_id,omitempty"` SessionID string `json:"session_id,omitempty"`
@@ -304,12 +329,14 @@ type GenerateResponse struct {
Content string Content string
Usage TokenUsage Usage TokenUsage
UsagePresence *TokenUsagePresence UsagePresence *TokenUsagePresence
CostUSD *float64
} }
// TokenUsagePresence records which token fields a provider reported. // TokenUsagePresence records which token fields a provider reported.
type TokenUsagePresence struct { type TokenUsagePresence struct {
PromptTokens bool PromptTokens bool
CompletionTokens bool CompletionTokens bool
ReasoningTokens bool
TotalTokens bool TotalTokens bool
CachedTokens bool CachedTokens bool
CacheWriteTokens bool CacheWriteTokens bool
@@ -319,6 +346,7 @@ type TokenUsagePresence struct {
type TokenUsage struct { type TokenUsage struct {
PromptTokens int PromptTokens int
CompletionTokens int CompletionTokens int
ReasoningTokens int
TotalTokens int TotalTokens int
CachedTokens int CachedTokens int
CacheWriteTokens int CacheWriteTokens int

View File

@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct { type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount CacheWriteTokens ObservedTokenCount
@@ -75,6 +76,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64 TransientRetries uint64
RepairCalls uint64 RepairCalls uint64
Usage ObservedTokenUsage Usage ObservedTokenUsage
CostUSD ObservedCost
} }
// ExecutionEvent contains metadata only; it carries no mutable containers or // ExecutionEvent contains metadata only; it carries no mutable containers or
@@ -100,6 +102,7 @@ type ExecutionEvent struct {
StatusCode int StatusCode int
RetriesExhausted bool RetriesExhausted bool
Usage ObservedTokenUsage Usage ObservedTokenUsage
CostUSD ObservedCost
Attempts uint64 Attempts uint64
TransientRetries uint64 TransientRetries uint64
RepairCalls uint64 RepairCalls uint64

Some files were not shown because too many files have changed in this diff Show More