Compare commits
275 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5ea60e9b96 | |||
| ba5ac651e2 | |||
| 6f45fd9fd0 | |||
| 79e23ecdba | |||
| 63e2968038 | |||
| 1408001fce | |||
| df8384426d | |||
| 8758c350e9 | |||
| 4a9def2138 | |||
| 40f8a68253 | |||
| 08d3d0ea3b | |||
| 56795bb49a | |||
| 891feb8c27 | |||
| 424d35c798 | |||
| bab3422e62 | |||
| cf03bca753 | |||
| 92e1e225b2 | |||
| 74a77b3c32 | |||
| 3a948a1e02 | |||
| eb66f82b4f | |||
| 4d5841e087 | |||
| 34c0cf3f14 | |||
| 76c8789e65 | |||
| 38a2dcca52 | |||
| dad302f1ff | |||
| 645bff20d7 | |||
| 248d0dbcf5 | |||
| 515649fee0 | |||
| 07aaee8ef6 | |||
| 49ea85680e | |||
| 16f8197a2d | |||
| 2a726d60ff | |||
| d3523000bf | |||
| bb8ec2c792 | |||
| f80ec1c42c | |||
| 649bb7dcf2 | |||
| 55276c9429 | |||
| aae523c916 | |||
| dfe9806228 | |||
| 63c96b7d61 | |||
| e1af2d6d9c | |||
| bd6b2848bb | |||
| eae4d18a9c | |||
| 2c436d14d9 | |||
| 1bfb6eb301 | |||
| 37582e9c30 | |||
| 4120d784a2 | |||
| 3bc16bffe6 | |||
| e9a12d43c2 | |||
| 9bed36282a | |||
| 950d3ccfb2 | |||
| 6fbbaa9cfa | |||
| 2d96dfa696 | |||
| be5a508554 | |||
| e8420d229e | |||
| 596e985cff | |||
| fbe3d427a2 | |||
| e775a3c116 | |||
| adfccb5370 | |||
| 0603a4f0dd | |||
| eaf9a76fd3 | |||
| 3d4c4c940e | |||
| 6668673c8d | |||
| 019e077ba9 | |||
| 718cd05043 | |||
| dd40171c01 | |||
| e415ad43ce | |||
| ae1efc8049 | |||
| 0f8e0e49b1 | |||
| ffce326dd7 | |||
| 804a4d32b2 | |||
| 67d446f611 | |||
| bcdd6fe06c | |||
| 382ccfb14b | |||
| 7406616794 | |||
| 4939fdb283 | |||
| fcbff80ab2 | |||
| d3bc439bb8 | |||
| 3ec68d509c | |||
| 57b62d391e | |||
| 8831ae35f7 | |||
| 3de3e60c0c | |||
| a8698d45a8 | |||
| 2b9badb3a5 | |||
| 79699acbb9 | |||
| 4cc5a55579 | |||
| d42b131de2 | |||
| 2066579e09 | |||
| 0e4f590f5a | |||
| f17e0620a5 | |||
| 5ad1b7aa6e | |||
| 3ace6ddb57 | |||
| 65546d794e | |||
| 892cb74e6e | |||
| 64cf84c9b5 | |||
| c23393b2cc | |||
| 5aae5df5fb | |||
| aaaa2816a4 | |||
| 5c135c5b28 | |||
| 31cc0069fc | |||
| 68daa61867 | |||
| 159d369b9b | |||
| 23fb55f9df | |||
| a07e95fd17 | |||
| c4d7cf8cd4 | |||
| 5a6f73e0bb | |||
| c149f7274c | |||
| e5c42de06d | |||
| 04569e44df | |||
| 26530808e0 | |||
| fcd0485340 | |||
| 5cc3fa0168 | |||
| b423de12c3 | |||
| 72955aaaa3 | |||
| 85dae9b443 | |||
| 70db0ad614 | |||
| 37a48f2e20 | |||
| 6c14379026 | |||
| 45bbf7bb2c | |||
| ee810b8fbe | |||
| bc95d1f9fd | |||
| c948fef8fc | |||
| cb180e8728 | |||
| de2a7fcd6f | |||
| f20020d9f4 | |||
| 9396183bdb | |||
| dba6d7d68c | |||
| d515a7e49b | |||
| 171fb11f4e | |||
| c9cbe7d3b3 | |||
| 7df7e420aa | |||
| 8ff9aa3efd | |||
| 78b78ea00b | |||
| 63e9f0b731 | |||
| 20b0f78af2 | |||
| 101a2840d4 | |||
| f083ad5309 | |||
| 19a0604874 | |||
| ea31e71280 | |||
| 27966c1686 | |||
| ea63a7b0ff | |||
| b5b9b1f209 | |||
| 129959b24c | |||
| dfe9d6566f | |||
| bf3d2e19bd | |||
| 4cbd4e2da0 | |||
| b0a16ccce8 | |||
| 8add7c49e6 | |||
| 8563379216 | |||
| 951739eda0 | |||
| a0f696181f | |||
| 3951dcf1f3 | |||
| 363920deb2 | |||
| 06315062b3 | |||
| c669f14610 | |||
| 52287ab2d1 | |||
| a43e0f17ee | |||
| ffaaf552c8 | |||
| d7007eb072 | |||
| 4f3212bfb6 | |||
| 4faf43cc29 | |||
| c9663ec63c | |||
| d8c98ac575 | |||
| 5ee602cc4b | |||
| 96c619f7d4 | |||
| b3b5ec3820 | |||
| 5dd1418953 | |||
| f5c900cb0a | |||
| c5f3de13f3 | |||
| 88725e79ab | |||
| 6c015a17fa | |||
| 9463c9463a | |||
| 1fbd175e53 | |||
| 4b4ce43796 | |||
| d78474a3bf | |||
| acaf2ccd8b | |||
| 97e5739bfb | |||
| 90f796aef7 | |||
| 0dec3dcec0 | |||
| 448e64dc40 | |||
| de46935a0e | |||
| c69703b2b6 | |||
| f660b6e0fc | |||
| 5f56c65427 | |||
| cfa3a85308 | |||
| 1ff6d778af | |||
| 3f93763c9c | |||
| 4aeb58f913 | |||
| ac1e150168 | |||
| ff57dbf8b3 | |||
| beb18cc9d6 | |||
| 7203e5d6a1 | |||
| 356563a6e5 | |||
| 6673284312 | |||
| 6f42e1e007 | |||
| cee9bbc7b2 | |||
| 7db5572b62 | |||
| a91bc1608b | |||
| e7a3665443 | |||
| 59e8010623 | |||
| ada5377f32 | |||
| df208e132b | |||
| 8c0fb15727 | |||
| 2e2fba4993 | |||
| 68668cee56 | |||
| f745b14b71 | |||
| dbba1485a7 | |||
| 7c435032b6 | |||
| e834e7d806 | |||
| a4901cbb0e | |||
| 5dcfc2e953 | |||
| a02561d40f | |||
| 6112ee8fb4 | |||
| 0aceff8c08 | |||
| cddb2b389c | |||
| e056cb1607 | |||
| 1e122f0473 | |||
| 2f0f5fa607 | |||
| 7d500ee62f | |||
| 7be677c80b | |||
| 1c075bd510 | |||
| b8dc8a49a9 | |||
| cd373ee46b | |||
| e881bbdcf4 | |||
| 106c808442 | |||
| 4ec35f5ca6 | |||
| 6a1a32ae49 | |||
| e85ea2db35 | |||
| 3ef2a9e18e | |||
| a85a6788b4 | |||
| 21fea46fb3 | |||
| 62bf82a56b | |||
| bd3dfa992c | |||
| 425f53a62e | |||
| fcc94eb9e9 | |||
| 95730a3c11 | |||
| c542b81666 | |||
| e8e112186e | |||
| aa08d46ef6 | |||
| 668d09997a | |||
| 69203dcb12 | |||
| 3b760310a6 | |||
| b39460a507 | |||
| 5177db0611 | |||
| 31f1e6a89d | |||
| f7085d76f6 | |||
| cf0bfd4d76 | |||
| 6b1323d1fe | |||
| fb87fabc2e | |||
| 5adc1c1e64 | |||
| 0d27252999 | |||
| d9f6aad305 | |||
| fda0338134 | |||
| 2b3d165ce2 | |||
| 43f55edb98 | |||
| 4950e1dee2 | |||
| d240364776 | |||
| 6f57f2f30a | |||
| 2c50eca033 | |||
| ac44a4f325 | |||
| 9e05c3ac8b | |||
| 2c83925c88 | |||
| e0cd63923d | |||
| 75204d6134 | |||
| 5ce6e01f27 | |||
| 5cec25a40c | |||
| 855aae6daa | |||
| 43abbcb414 | |||
| 9f33fed823 | |||
| 7a98feb399 | |||
| 626134e5e1 | |||
| e7ed315671 | |||
| d7d9a7dd65 | |||
| 18f6a474e0 | |||
| fb7ebe6a6c |
43
README.md
43
README.md
@@ -26,6 +26,8 @@ source-relative prompt content, supplies an inline input, and prepares through
|
||||
a scoped view. The example also demonstrates collection inventories and
|
||||
namespace-filtered discovery, without contacting a provider or requiring
|
||||
credentials.
|
||||
The example loads application-owned settings once and selects no bundled
|
||||
underlay, keeping inventories limited to its own offline fixtures.
|
||||
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
||||
ordered file/embedded sources, programmatic mappings, and how to replace an
|
||||
engine when source files change. Applications adapting the prior keyed
|
||||
@@ -43,6 +45,47 @@ boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
The [v0.24.0 release notes](docs/releases/v0.24.0.md) cover application-owned
|
||||
configuration loading, prompt-owned schemas, default output repair, and the
|
||||
prepared message-content hash. Upgrading consumers should review the breaking
|
||||
configuration and request-API changes in the
|
||||
[migration guide](docs/consumers/configuration-migration.md#move-settings-under-promptkit).
|
||||
|
||||
The [v0.23.3 release notes](docs/releases/v0.23.3.md) cover HTTP operation-timeout
|
||||
attribution, custom caller cancellation causes, and response-read error identity.
|
||||
|
||||
The [v0.23.2 release notes](docs/releases/v0.23.2.md) cover configuration,
|
||||
validation, and release-tool dependency updates without an API migration or
|
||||
Go minimum increase.
|
||||
|
||||
The [v0.23.1 release notes](docs/releases/v0.23.1.md) cover corrected constructor
|
||||
error wrapping for downstream classification, with loading diagnostics and
|
||||
underlying causes preserved.
|
||||
|
||||
The [v0.23.0 release notes](docs/releases/v0.23.0.md) cover per-operation
|
||||
timeouts and opt-in prepared-request notifications for ordinary and coordinated
|
||||
execution.
|
||||
|
||||
The [v0.22.0 release notes](docs/releases/v0.22.0.md) cover captured conversation
|
||||
exchanges, application-directed turn coordination, and explicit execution groups.
|
||||
|
||||
The [v0.21.0 release notes](docs/releases/v0.21.0.md) cover ConfigKit v1.2.0
|
||||
and main-relative companion imports through the existing configuration-source
|
||||
API.
|
||||
|
||||
The [v0.20.0 release notes](docs/releases/v0.20.0.md) cover resolved resource
|
||||
inspection, generic JSON/YAML reports, typed prompt sources, and backend
|
||||
inheritance. Upgrading consumers should review the changed inspection result
|
||||
types and profile null-fallback behavior.
|
||||
|
||||
The [v0.19.0 release notes](docs/releases/v0.19.0.md) cover audit remediation
|
||||
and compatibility changes to artifact ownership, programmatic sources, and schemas.
|
||||
|
||||
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
||||
OpenRouter backend and profiles, recorded configuration normalization, and
|
||||
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
||||
the Go 1.27 minimum remains unchanged.
|
||||
|
||||
The [v0.17.0 release notes](docs/releases/v0.17.0.md) cover the vanity module
|
||||
path, ConfigKit v1, identified resource lists, removal of prompt-version
|
||||
metadata, and the Go 1.27 minimum. This is a breaking pre-v1 migration.
|
||||
|
||||
325
application_handoff_integration_test.go
Normal file
325
application_handoff_integration_test.go
Normal file
@@ -0,0 +1,325 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type integrationComponent struct {
|
||||
ID string `yaml:"id"`
|
||||
}
|
||||
|
||||
type integrationApplication struct {
|
||||
Version string `yaml:"version"`
|
||||
Sibling string `yaml:"sibling"`
|
||||
Components []integrationComponent `yaml:"components"`
|
||||
Prompts []string `yaml:"prompts"`
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
func integrationDefinition(resolves, validates *atomic.Int64) configkit.Definition[integrationApplication] {
|
||||
definition := configkit.Definition[integrationApplication]{
|
||||
Defaults: func() integrationApplication { return integrationApplication{Sibling: "default"} },
|
||||
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(version configkit.Version, validation *configkit.Validation) error {
|
||||
if version.Present && version.Text != "app-v1" {
|
||||
validation.Error(configkit.Keys("version"), "invalid application version")
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("components"), "id")},
|
||||
Sensitive: func(path configkit.Path) bool { return path.String() == "/sibling" },
|
||||
Resolve: func(_ *integrationApplication, _ *configkit.Resolution) error { resolves.Add(1); return nil },
|
||||
Validate: func(_ *integrationApplication, validation *configkit.Validation) {
|
||||
validates.Add(1)
|
||||
validation.Warning(configkit.Keys("sibling"), "application warning")
|
||||
},
|
||||
}
|
||||
configured := promptkit.Configure(definition, func(value *integrationApplication) *promptkit.Settings { return &value.PromptKit })
|
||||
previous := configured.Validate
|
||||
configured.Validate = func(value *integrationApplication, validation *configkit.Validation) {
|
||||
previous(value, validation)
|
||||
if len(value.PromptKit.Backends) != 1 || value.PromptKit.Backends[0].Endpoint == nil || strings.Contains(*value.PromptKit.Backends[0].Endpoint, " ") {
|
||||
validation.Error(configkit.Keys("promptkit"), "backend normalization missing")
|
||||
}
|
||||
if value.PromptKit.Application["section_extra"] != "section" || value.PromptKit.Backends[0].Application["backend_extra"] != "backend" || value.PromptKit.Profiles[0].Application["profile_extra"] != "profile" || value.PromptKit.Prompts[0].Application["prompt_extra"] != "prompt" || value.PromptKit.Collections["group"].Application["collection_extra"] != "collection" {
|
||||
validation.Error(configkit.Keys("promptkit"), "application fields missing")
|
||||
}
|
||||
}
|
||||
return configured
|
||||
}
|
||||
|
||||
const integrationDocument = `version: app-v1
|
||||
prompts: [application-only]
|
||||
components: [{id: second-component}]
|
||||
promptkit:
|
||||
section_extra: section
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: ' https://example.test/v1 '
|
||||
extra_params: {provider: {mode: original}}
|
||||
backend_extra: backend
|
||||
profiles:
|
||||
- id: profile
|
||||
backend: local
|
||||
model: original-model
|
||||
profile_extra: profile
|
||||
prompts:
|
||||
- id: example
|
||||
default_profile: profile
|
||||
inputs: [{name: document, required: false}]
|
||||
messages: [{role: user, content: hello}]
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema: {type: object, required: [ok], properties: {ok: {type: boolean}}}
|
||||
prompt_extra: prompt
|
||||
collections:
|
||||
group: {collection_extra: collection}
|
||||
`
|
||||
|
||||
func TestApplicationOwnedLoadAndCapture(t *testing.T) {
|
||||
var resolves, validates atomic.Int64
|
||||
definition := integrationDefinition(&resolves, &validates)
|
||||
result, err := configkit.Load(configkit.Bytes("application", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Config.Sibling != "default" || result.Config.Prompts[0] != "application-only" || len(result.Diagnostics()) != 1 || result.Config.PromptKit.Prompts[0].Output.RepairAttempts == nil || *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
|
||||
t.Fatalf("loaded application = %+v, diagnostics = %+v", result.Config, result.Diagnostics())
|
||||
}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"ok":true}`}}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolves.Load() != 1 || validates.Load() != 1 {
|
||||
t.Fatalf("callbacks rerun during handoff: %d %d", resolves.Load(), validates.Load())
|
||||
}
|
||||
before, err := engine.InspectPrompt(context.Background(), "example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app := &result.Config.PromptKit
|
||||
*app.Backends[0].Endpoint = "https://changed.test/v1"
|
||||
app.Backends[0].ExtraParams["provider"].(map[string]any)["mode"] = "changed"
|
||||
*app.Profiles[0].Model = "changed-model"
|
||||
*app.Prompts[0].Messages[0].Content = "changed"
|
||||
app.Prompts[0].Inputs[0].Name = "changed"
|
||||
app.Prompts[0].Output.Schema.(map[string]any)["required"].([]any)[0] = "changed"
|
||||
app.Prompts = nil
|
||||
app.Collections = nil
|
||||
after, err := engine.InspectPrompt(context.Background(), "example")
|
||||
if err != nil || !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("captured prompt changed: %+v, %+v, %v", before, after, err)
|
||||
}
|
||||
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "example"})
|
||||
if err != nil || run.Validation.Status != promptkit.ValidationPassed || len(client.requests) != 1 || client.requests[0].Prompt.Messages[0].Content != "hello" || client.requests[0].Target.Model != "original-model" {
|
||||
t.Fatalf("run = %+v, requests = %+v, error = %v", run, client.requests, err)
|
||||
}
|
||||
if strings.Contains(fmt.Sprintf("%+v", client.requests[0].Target.ExtraParams), "backend_extra") || strings.Contains(fmt.Sprintf("%+v", after), "prompt_extra") {
|
||||
t.Fatal("application fields leaked into resources or provider parameters")
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var report bytes.Buffer
|
||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "section_extra: section") || !strings.Contains(report.String(), "prompt_extra: prompt") {
|
||||
t.Fatalf("inspection = %q, %v", report.String(), err)
|
||||
}
|
||||
if len(view.Diagnostics()) != 1 {
|
||||
t.Fatalf("diagnostics = %+v", view.Diagnostics())
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for range 4 {
|
||||
wg.Go(func() {
|
||||
_, err := configkit.Load(configkit.Bytes("repeat", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if resolves.Load() != 5 || validates.Load() != 5 {
|
||||
t.Fatalf("reused callbacks = %d %d", resolves.Load(), validates.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandoffMountedSourcesAndExecution(t *testing.T) {
|
||||
baseFS := &countingMountFS{FS: fstest.MapFS{
|
||||
"base/collection.yaml": {Data: []byte("prompts:\n - id: parent\n default_profile: profile\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")},
|
||||
"base/original.txt": {Data: []byte("original")}, "base/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}}
|
||||
operatorFS := &countingMountFS{FS: fstest.MapFS{
|
||||
"operator/item.yaml": {Data: []byte("id: child\nextends: {id: parent}\nmessages: [{role: user, content_file: replacement.txt}]\n")},
|
||||
"operator/replacement.txt": {Data: []byte("replacement")},
|
||||
}}
|
||||
base := configkit.NewFilesystem("embedded", baseFS)
|
||||
operator := configkit.NewFilesystem("operator", operatorFS)
|
||||
collection, err := promptkit.CollectionInput("team", base.File("base/collection.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collection.Filesystem = &configkit.FilesystemContext{FS: &base, Root: "."}
|
||||
item := configkit.Input{Source: operator.File("operator/item.yaml"), Destination: configkit.Keys("promptkit", "collections", "team", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem, Filesystem: &configkit.FilesystemContext{FS: &operator, Root: "."}}
|
||||
profile := promptkit.SettingsInput(configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.test/v1", "model": "model"}}}))
|
||||
type root struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||
layers := []configkit.Underlay{{Inputs: []configkit.Input{profile, collection}}, {Inputs: []configkit.Input{item}}}
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: layers})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
beforeBase, beforeOperator := baseFS.opens, operatorFS.opens
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{}`}}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if baseFS.opens-beforeBase != 1 || operatorFS.opens-beforeOperator != 1 {
|
||||
t.Fatalf("unexpected construction reads: base=%d operator=%d", baseFS.opens-beforeBase, operatorFS.opens-beforeOperator)
|
||||
}
|
||||
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: layers}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompt, err := standalone.InspectPrompt(context.Background(), "team/child")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotPrompt, err := engine.InspectPrompt(context.Background(), "team/child")
|
||||
if err != nil || !reflect.DeepEqual(gotPrompt, wantPrompt) {
|
||||
t.Fatalf("resource reports = %+v, %+v, %v", gotPrompt, wantPrompt, err)
|
||||
}
|
||||
wantInventory, err := standalone.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotInventory, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || !reflect.DeepEqual(gotInventory, wantInventory) {
|
||||
t.Fatalf("inventories = %+v, %+v, %v", gotInventory, wantInventory, err)
|
||||
}
|
||||
collectionSettings := result.Config.PromptKit.Collections["team"]
|
||||
*collectionSettings.Prompts[1].Output.SchemaFile = "changed.json"
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||
if err != nil || prepared.Messages[0].Content != "replacement" {
|
||||
t.Fatalf("prepared = %+v, %v", prepared, err)
|
||||
}
|
||||
standalonePrepared, err := standalone.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||
if err != nil || !reflect.DeepEqual(prepared.StructuredOutput, standalonePrepared.StructuredOutput) || !reflect.DeepEqual(prepared.OutputContract, standalonePrepared.OutputContract) || !reflect.DeepEqual(prepared.Messages, standalonePrepared.Messages) {
|
||||
t.Fatalf("prepared details differ: %+v, %+v, %v", prepared, standalonePrepared, err)
|
||||
}
|
||||
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||
if err != nil || run.Validation.Status != promptkit.ValidationPassed {
|
||||
t.Fatalf("run = %+v, %v", run, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandoffUsesOnlyBundlesSelectedBeforeLoad(t *testing.T) {
|
||||
type root struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{promptkit.BundledOpenRouterUnderlay()}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends, err := engine.ListBackends(context.Background())
|
||||
if err != nil || len(backends) != 1 || backends[0] != "openrouter" {
|
||||
t.Fatalf("bundles = %+v, %v", backends, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationFieldsComposeAndInherit(t *testing.T) {
|
||||
type root struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||
underlay := promptkit.SettingsInput(configkit.Mapping("defaults", map[string]any{
|
||||
"section_data": map[string]any{"first": "section"},
|
||||
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.test/v1", "app_data": map[string]any{"first": "backend"}}},
|
||||
"profiles": []any{map[string]any{"id": "base", "model": "model", "app_data": map[string]any{"first": "profile"}}},
|
||||
"prompts": []any{map[string]any{"id": "base", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "app_data": map[string]any{"first": "prompt"}}},
|
||||
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"first": "collection"}}},
|
||||
}))
|
||||
main := promptkit.SettingsInput(configkit.Mapping("operator", map[string]any{
|
||||
"section_data": map[string]any{"second": "section"},
|
||||
"backends": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "backend"}}},
|
||||
"profiles": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "profile"}}},
|
||||
"prompts": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "prompt"}}},
|
||||
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"second": "collection"}}},
|
||||
}))
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := result.Config.PromptKit
|
||||
for name, fields := range map[string]map[string]any{"section": settings.Application, "backend": settings.Backends[1].Application, "profile": settings.Profiles[1].Application, "prompt": settings.Prompts[1].Application, "collection": settings.Collections["team"].Application} {
|
||||
key := "app_data"
|
||||
if name == "section" {
|
||||
key = "section_data"
|
||||
}
|
||||
data, ok := fields[key].(map[string]any)
|
||||
if !ok || data["first"] != name || data["second"] != name {
|
||||
t.Errorf("%s extension = %#v", name, fields[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandoffMissingAndNullSections(t *testing.T) {
|
||||
type noSection struct {
|
||||
Prompts []string `yaml:"prompts"`
|
||||
}
|
||||
missing, err := configkit.Load[noSection](configkit.Bytes("application-only", []byte("prompts: [application]\n")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
empty, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prompts, err := empty.ListPrompts(context.Background()); err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("application root resources were consumed: %+v, %v", prompts, err)
|
||||
}
|
||||
for _, source := range []string{"{}", "promptkit: null", "promptkit: {}"} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
type root struct {
|
||||
PromptKit *promptkit.Settings `yaml:"promptkit"`
|
||||
Prompts []string `yaml:"prompts"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return value.PromptKit })
|
||||
result, err := configkit.Load(configkit.Bytes("missing", []byte(source)), configkit.Options[root]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("prompts = %+v, %v", prompts, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := promptkit.NewFromConfig(nil, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil result = %v", err)
|
||||
}
|
||||
}
|
||||
90
application_input_defaults_test.go
Normal file
90
application_input_defaults_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestApplicationDefaultInputInspection(t *testing.T) {
|
||||
for _, derived := range []bool{false, true} {
|
||||
name := "Go defaults"
|
||||
if derived {
|
||||
name = "derived collection defaults"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
settings := promptkit.Settings{
|
||||
Profiles: []promptkit.ProfileSettings{{ID: "profile", Endpoint: new("https://example.test/v1"), Model: new("model")}},
|
||||
Prompts: []promptkit.PromptSettings{{ID: "prompt", DefaultProfile: new("profile"),
|
||||
Inputs: []promptkit.PromptInputSettings{{Name: "doc", Required: true, ContentType: new("text/plain"), Description: new("")}},
|
||||
Messages: []promptkit.PromptMessageSettings{{Role: new("user"), Content: new("hello")}},
|
||||
Output: &promptkit.PromptOutputSettings{Format: new("text"), ValidationMode: new("none")},
|
||||
}},
|
||||
}
|
||||
definition := configkit.Definition[handoffApplication]{Defaults: func() handoffApplication { return handoffApplication{PromptKit: settings} }}
|
||||
source := configkit.NoSource()
|
||||
id := "prompt"
|
||||
if derived {
|
||||
definition.Defaults = nil
|
||||
source = configkit.Bytes("derived inputs", []byte(`promptkit:
|
||||
profiles: [{id: profile, endpoint: https://example.test/v1, model: model}]
|
||||
collections:
|
||||
team:
|
||||
prompts:
|
||||
- id: prompt
|
||||
default_profile: profile
|
||||
inputs: [{name: doc}]
|
||||
messages: [{role: user, content: hello}]
|
||||
output: {format: text, validation_mode: none}
|
||||
`))
|
||||
id = "team/prompt"
|
||||
definition.Resolve = func(value *handoffApplication, resolution *configkit.Resolution) error {
|
||||
input := &value.PromptKit.Collections["team"].Prompts[0].Inputs[0]
|
||||
input.Required, input.ContentType, input.Description = true, new("text/plain"), new("")
|
||||
base := configkit.Keys("promptkit", "collections", "team", "prompts").Index(0).Key("inputs").Index(0)
|
||||
for _, field := range []string{"required", "content_type", "description"} {
|
||||
resolution.RecordDerivedDefault(base.Key(field), "Application input default", base.Key("name"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
definition = promptkit.Configure(definition, func(value *handoffApplication) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.Load(source, configkit.Options[handoffApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
resource, err := engine.InspectPrompt(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := (*resource.Inputs)[0]
|
||||
if input.Required == nil || !*input.Required || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil || *input.Description != "" {
|
||||
t.Fatalf("resource input = %+v", input)
|
||||
}
|
||||
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
required := requirements.Inputs[0]
|
||||
if required.Kind != promptkit.RequirementRequired || required.ContentType == nil || *required.ContentType != "text/plain" || required.Description == nil || *required.Description != "" {
|
||||
t.Fatalf("requirement = %+v", required)
|
||||
}
|
||||
request := promptkit.RunRequest{PromptID: id}
|
||||
if err := engine.ValidateRequest(ctx, request); !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("missing input: %v", err)
|
||||
}
|
||||
request.Inputs = map[string]promptkit.ArtifactRef{"doc": promptkit.Inline("document")}
|
||||
if err := engine.ValidateRequest(ctx, request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
43
artifact_ownership_contract_test.go
Normal file
43
artifact_ownership_contract_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type independentArtifactReader struct{}
|
||||
|
||||
func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
return &promptkit.Artifact{Body: []byte(ref.Body), Hash: ref.Body}, nil
|
||||
}
|
||||
|
||||
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("ownership", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithArtifactReader(independentArtifactReader{}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var group sync.WaitGroup
|
||||
for i := range 24 {
|
||||
group.Go(func() {
|
||||
value := fmt.Sprintf("body-%d", i)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"document": promptkit.Inline(value)}})
|
||||
if err != nil {
|
||||
t.Errorf("prepare: %v", err)
|
||||
return
|
||||
}
|
||||
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != value || prepared.InputHashes["document"] != value {
|
||||
t.Errorf("prepared = %+v", prepared)
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
}
|
||||
@@ -73,6 +73,7 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||
var underlays []configkit.Underlay
|
||||
var expectedMessage string
|
||||
|
||||
switch name {
|
||||
@@ -91,9 +92,9 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||
}
|
||||
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{
|
||||
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||
}, "."))
|
||||
}, ".")}})
|
||||
default:
|
||||
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||
}
|
||||
@@ -106,8 +107,8 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
||||
"output": output,
|
||||
}},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
|
||||
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||
}}}, options...)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
|
||||
48
backends.go
48
backends.go
@@ -19,10 +19,13 @@ const BackendLocal = "local"
|
||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
||||
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
||||
// inputs for explicit null. BackendConfig has no stable JSON representation.
|
||||
// inputs for explicit null. Extends names a sibling backend in the same list.
|
||||
// BackendConfig has no stable JSON representation.
|
||||
type BackendConfig struct {
|
||||
// ID is the local backend identity within its resource list.
|
||||
ID string
|
||||
// Extends names a sibling backend to inherit through ConfigKit.
|
||||
Extends *string
|
||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||
Endpoint *string
|
||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||
@@ -38,51 +41,20 @@ type BackendConfig struct {
|
||||
SmokeTests *BackendSmokeTestConfig
|
||||
}
|
||||
|
||||
// BackendSmokeTestConfig contains optional diagnostic settings associated with
|
||||
// a backend. Nil nested fields are omitted by BackendSource.
|
||||
type BackendSmokeTestConfig struct {
|
||||
// DefaultProfile is the exact profile ID selected as the backend's
|
||||
// diagnostic inference default. A supplied value must not be blank; the ID
|
||||
// is resolved only when that default is selected.
|
||||
DefaultProfile *string
|
||||
// Reachability optionally selects a full HTTP destination for reachability
|
||||
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
||||
Reachability *ReachabilityProbeConfig
|
||||
// Authentication optionally declares a named authentication diagnostic.
|
||||
// A supplied object requires Kind.
|
||||
Authentication *AuthenticationProbeConfig
|
||||
}
|
||||
|
||||
// ReachabilityProbeConfig configures an optional reachability destination.
|
||||
type ReachabilityProbeConfig struct {
|
||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||
// Nil selects the effective backend endpoint; an explicit blank value is
|
||||
// invalid.
|
||||
Endpoint *string
|
||||
}
|
||||
|
||||
// AuthenticationProbeConfig configures an optional named authentication
|
||||
// diagnostic.
|
||||
type AuthenticationProbeConfig struct {
|
||||
// Kind selects the authentication adapter. Supported values are "none" and
|
||||
// "openrouter_key"; surrounding whitespace and case are normalized.
|
||||
Kind *string
|
||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||
// Nil selects the adapter's backend-relative destination; an explicit blank
|
||||
// value is invalid.
|
||||
Endpoint *string
|
||||
}
|
||||
|
||||
// BackendSource creates a labeled ConfigKit mapping containing backend records
|
||||
// in caller order. IDs are explicit and duplicates are left for loading to reject.
|
||||
// It encodes fields without validating them; NewEngine validates the final
|
||||
// effective configuration after all selected source layers are combined.
|
||||
// Mapping captures a copy of supported input values at creation; unsupported
|
||||
// values fail when the source is loaded.
|
||||
// values fail when the source is loaded. ExtraParams must use ConfigKit Mapping
|
||||
// input forms; callers must explicitly convert numbers or containers as needed.
|
||||
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||
definitions := make([]any, 0, len(backends))
|
||||
for _, config := range backends {
|
||||
fields := map[string]any{"id": config.ID}
|
||||
if config.Extends != nil {
|
||||
fields["extends"] = map[string]any{"id": *config.Extends}
|
||||
}
|
||||
if config.Endpoint != nil {
|
||||
fields["endpoint"] = *config.Endpoint
|
||||
}
|
||||
@@ -96,7 +68,7 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||
fields["queue_capacity"] = *config.QueueCapacity
|
||||
}
|
||||
if config.ExtraParams != nil {
|
||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
||||
fields["extra_params"] = config.ExtraParams
|
||||
}
|
||||
if config.SmokeTests != nil {
|
||||
smokeTests := make(map[string]any)
|
||||
|
||||
114
bundled_composition_contract_test.go
Normal file
114
bundled_composition_contract_test.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
||||
underlay := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller underlay", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}},
|
||||
})}
|
||||
underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
||||
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
|
||||
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
||||
}
|
||||
main := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller main", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||
"profiles": []any{
|
||||
map[string]any{"id": "gpt-sol-latest", "model": "main-model"},
|
||||
map[string]any{"id": "custom", "extends": map[string]any{"id": "gpt-luna-latest"}, "model": "custom-model"},
|
||||
map[string]any{"id": "auto", "backend": "other", "model": "other-model"},
|
||||
},
|
||||
"collections": map[string]any{"openrouter": map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://collection.invalid/v1"}},
|
||||
"profiles": []any{map[string]any{"id": "profile", "backend": "openrouter/local", "model": "collection-model"}},
|
||||
}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}, Main: &main}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspect := func(id string) *promptkit.ProfileInspection {
|
||||
t.Helper()
|
||||
profile, err := engine.InspectProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return profile
|
||||
}
|
||||
if target := inspect("gpt-sol-latest"); target.Model == nil || *target.Model != "main-model" || target.Backend != nil {
|
||||
t.Fatalf("replacement = %+v", target)
|
||||
}
|
||||
if target := inspect("custom"); target.Model == nil || *target.Model != "custom-model" || target.Backend == nil || *target.Backend != "openrouter" || target.Endpoint != nil {
|
||||
t.Fatalf("inherited = %+v", target)
|
||||
}
|
||||
if backend, err := engine.InspectBackend(context.Background(), "openrouter"); err != nil || backend.Endpoint != "https://local.invalid/v1" || backend.APIKeyEnv != nil {
|
||||
t.Fatalf("backend replacement = %+v, %v", backend, err)
|
||||
}
|
||||
if target := inspect("openrouter/profile"); target.Backend == nil || *target.Backend != "openrouter/local" || target.Model == nil || *target.Model != "collection-model" {
|
||||
t.Fatalf("collection = %+v", target)
|
||||
}
|
||||
target := inspect("auto-low")
|
||||
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
||||
if target.Backend == nil || *target.Backend != "other" || target.Model == nil || *target.Model != "other-model" || target.ExtraParams == nil || !reflect.DeepEqual(*target.ExtraParams, wantPlugins) {
|
||||
t.Errorf("inherited tier = %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledCompositionSequenceModes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name, mode, id string
|
||||
wantErr bool
|
||||
hasAuto bool
|
||||
}{
|
||||
{"duplicate append", "append", "auto", true, false},
|
||||
{"distinct append", "append", "custom", false, true},
|
||||
{"replace", "replace", "custom", false, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("sequence", map[string]any{
|
||||
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "profiles"}, "mode": test.mode}}},
|
||||
"profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate rejection")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil || !slices.Contains(profiles, "custom") || slices.Contains(profiles, "auto") != test.hasAuto {
|
||||
t.Fatalf("profiles = %v, %v", profiles, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) {
|
||||
profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})}
|
||||
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}})
|
||||
if err == nil {
|
||||
t.Fatal("strict group accepted duplicate records")
|
||||
}
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replace backends", map[string]any{
|
||||
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "backends"}, "mode": "replace"}}},
|
||||
"backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||
})}
|
||||
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil {
|
||||
t.Fatal("missing bundled backend was accepted")
|
||||
}
|
||||
}
|
||||
89
bundled_execution_contract_test.go
Normal file
89
bundled_execution_contract_test.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func bundledPromptConfig(client *http.Client) promptkit.Config {
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}}})}
|
||||
return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}
|
||||
}
|
||||
|
||||
func TestBundledProfilePreparationAndExecution(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
engine, err := promptkit.NewEngine(bundledPromptConfig(nil), promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileRequired) {
|
||||
t.Fatalf("missing profile = %v", err)
|
||||
}
|
||||
temperature := 0.3
|
||||
request := promptkit.RunRequest{PromptID: "prompt", ProfileID: "gpt-luna-latest", Execution: &promptkit.ExecutionTargetOverride{Temperature: &temperature}}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
details := prepared.Details()
|
||||
if details.SelectedProfileID != "gpt-luna-latest" || details.EffectiveModelParams.Model != "~openai/gpt-luna-latest" || details.EffectiveModelParams.BackendID != "openrouter" || details.EffectiveModelParams.Temperature != temperature {
|
||||
t.Fatalf("prepared target = %+v", details.EffectiveModelParams)
|
||||
}
|
||||
if _, err := engine.Run(context.Background(), request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Target.Model != "~openai/gpt-luna-latest" || client.requests[0].Target.BackendID != "openrouter" {
|
||||
t.Fatalf("requests = %+v", client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
||||
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||
for _, test := range []struct{ id, model, tier string }{
|
||||
{"auto-low", "openrouter/auto", "low"}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
||||
} {
|
||||
t.Run(test.id, func(t *testing.T) {
|
||||
var body map[string]any
|
||||
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
if request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||
t.Errorf("authorization = %q", request.Header.Get("Authorization"))
|
||||
}
|
||||
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||
})
|
||||
engine, err := promptkit.NewEngine(bundledPromptConfig(&http.Client{Transport: transport}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", ProfileID: test.id}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body["model"] != test.model || body["extra_params"] != nil {
|
||||
t.Fatalf("request = %+v", body)
|
||||
}
|
||||
if test.tier == "" {
|
||||
if _, exists := body["plugins"]; exists {
|
||||
t.Fatalf("unexpected plugins = %+v", body)
|
||||
}
|
||||
return
|
||||
}
|
||||
want := []any{map[string]any{"id": "auto-router", "cost_tier": test.tier}}
|
||||
if !reflect.DeepEqual(body["plugins"], want) {
|
||||
t.Fatalf("plugins = %#v, want %#v", body["plugins"], want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
117
bundled_openrouter_contract_test.go
Normal file
117
bundled_openrouter_contract_test.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type rejectBundleTransport struct{}
|
||||
|
||||
func (rejectBundleTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||
return nil, errors.New("unexpected HTTP request")
|
||||
}
|
||||
|
||||
func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
||||
t.Setenv("OPENROUTER_API_KEY", "")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
options []promptkit.Option
|
||||
enabled bool
|
||||
}{
|
||||
{"default", nil, true}, {"enabled", []promptkit.Option{promptkit.WithBundledOpenRouter(true)}, true},
|
||||
{"disabled", []promptkit.Option{promptkit.WithBundledOpenRouter(false)}, false},
|
||||
{"last disables", []promptkit.Option{promptkit.WithBundledOpenRouter(true), promptkit.WithBundledOpenRouter(false)}, false},
|
||||
{"last enables", []promptkit.Option{promptkit.WithBundledOpenRouter(false), promptkit.WithBundledOpenRouter(true)}, true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: rejectBundleTransport{}}}, test.options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := engine.ListProfiles(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends, err := engine.ListBackends(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collections, err := engine.ListCollections(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if (len(profiles) > 0) != test.enabled || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
||||
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
||||
}
|
||||
if test.enabled {
|
||||
if !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
||||
}
|
||||
profile, err := engine.InspectProfile(context.Background(), "auto")
|
||||
if err != nil || profile.Backend == nil || *profile.Backend != "openrouter" {
|
||||
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") {
|
||||
t.Fatal("bundle source missing")
|
||||
}
|
||||
if explanation, found := view.Explain(configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 {
|
||||
t.Fatalf("bundle explanation = %+v", explanation)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func boolCount(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
func fmtSources(sources []configkit.SourceInfo) string {
|
||||
var b strings.Builder
|
||||
for _, source := range sources {
|
||||
b.WriteString(source.Label)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
|
||||
source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}})
|
||||
groups := make([]configkit.Underlay, 1, 2)
|
||||
groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}
|
||||
before := append([]configkit.Underlay(nil), groups...)
|
||||
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(groups, before) || !reflect.DeepEqual(groups[:2][1], configkit.Underlay{}) {
|
||||
t.Fatal("caller underlays were modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) {
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends, err := engine.ListBackends(context.Background())
|
||||
if err != nil || !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||
t.Fatalf("backends = %v, %v", backends, err)
|
||||
}
|
||||
}
|
||||
181
bundled_smoke_contract_test.go
Normal file
181
bundled_smoke_contract_test.go
Normal file
@@ -0,0 +1,181 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type cancelingAuthenticationBody struct {
|
||||
remaining int
|
||||
cancel context.CancelFunc
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (b *cancelingAuthenticationBody) Read(p []byte) (int, error) {
|
||||
n := min(len(p), b.remaining)
|
||||
for i := range p[:n] {
|
||||
p[i] = 'x'
|
||||
}
|
||||
b.remaining -= n
|
||||
if b.remaining == 0 {
|
||||
b.cancel()
|
||||
return n, context.Canceled
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
func (b *cancelingAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
func TestAuthenticationReadCancellationWinsOverOverflow(t *testing.T) {
|
||||
for _, size := range []int{1024, (1 << 20) + 1} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
body := &cancelingAuthenticationBody{remaining: size, cancel: cancel}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body}, nil
|
||||
})}})
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter"})
|
||||
cancel()
|
||||
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != promptkit.ExecutionFailureCanceled || report.Facts.CredentialsAccepted != nil || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || !body.closed {
|
||||
t.Fatalf("size %d: report=%+v error=%v closed=%v", size, report, err, body.closed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
||||
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||
var urls []string
|
||||
var models []string
|
||||
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
urls = append(urls, request.URL.String())
|
||||
body := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
|
||||
if request.Method == http.MethodPost {
|
||||
var payload map[string]any
|
||||
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
models = append(models, payload["model"].(string))
|
||||
content := "PROMPTKIT_OK"
|
||||
if _, structured := payload["response_format"]; structured {
|
||||
content = `{"status":"ok"}`
|
||||
}
|
||||
encoded, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body = string(encoded)
|
||||
} else if strings.HasSuffix(request.URL.Path, "/key") && request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||
t.Errorf("authentication header missing")
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := engine.ListProfiles(context.Background())
|
||||
inspectionBefore, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput} {
|
||||
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: kind, BackendID: "openrouter"})
|
||||
if err != nil || report == nil || !report.Completed || report.Failure != "" {
|
||||
t.Fatalf("%s report = %+v, %v", kind, report, err)
|
||||
}
|
||||
if kind == promptkit.SmokeTestAuthentication && (report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted) {
|
||||
t.Fatalf("authentication facts = %+v", report.Facts)
|
||||
}
|
||||
if kind == promptkit.SmokeTestTextInference || kind == promptkit.SmokeTestStructuredOutput {
|
||||
if report.Target.ProfileID != "gpt-luna-latest" || report.Target.EffectiveSettings.Model != "~openai/gpt-luna-latest" {
|
||||
t.Fatalf("target = %+v", report.Target)
|
||||
}
|
||||
}
|
||||
}
|
||||
wantURLs := []string{"https://openrouter.ai/api/v1", "https://openrouter.ai/api/v1/key", "https://openrouter.ai/api/v1/chat/completions", "https://openrouter.ai/api/v1/chat/completions"}
|
||||
if !reflect.DeepEqual(urls, wantURLs) || !reflect.DeepEqual(models, []string{"~openai/gpt-luna-latest", "~openai/gpt-luna-latest"}) {
|
||||
t.Fatalf("urls %v, models %v", urls, models)
|
||||
}
|
||||
after, _ := engine.ListProfiles(context.Background())
|
||||
if !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("inventory changed: %v %v", before, after)
|
||||
}
|
||||
inspectionAfter, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
diff, err := inspectionBefore.Diff(inspectionAfter)
|
||||
if err != nil || len(diff) != 0 {
|
||||
t.Fatalf("inspection changed: %+v, %v", diff, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
||||
replacement := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
|
||||
replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||
replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replacementReport, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||
if err != nil || replacementReport.Target.ProfileID != "gpt-luna-latest" || replacementReport.Target.EffectiveSettings.Model != "replacement" {
|
||||
t.Fatalf("replaced default = %+v, %v", replacementReport, err)
|
||||
}
|
||||
replacementClient.response.Content = "unexpected"
|
||||
mismatch, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||
if err != nil || mismatch == nil || !mismatch.Completed {
|
||||
t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
|
||||
}
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("application", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
|
||||
"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
request promptkit.SmokeTestRequest
|
||||
profile, model string
|
||||
}{
|
||||
{promptkit.SmokeTestRequest{BackendID: "openrouter"}, "custom", "custom-model"},
|
||||
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt"}, "free", "openrouter/free"},
|
||||
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt", ProfileID: "gpt-luna-latest"}, "gpt-luna-latest", "replacement"},
|
||||
} {
|
||||
test.request.Kind = promptkit.SmokeTestTextInference
|
||||
report, err := engine.SmokeTest(context.Background(), test.request)
|
||||
if err != nil || report.Target.ProfileID != test.profile || report.Target.EffectiveSettings.Model != test.model {
|
||||
t.Fatalf("report = %+v, %v", report, err)
|
||||
}
|
||||
if strings.Contains(client.requests[len(client.requests)-1].Prompt.Messages[0].Content, "private application marker") {
|
||||
t.Fatal("application content used by diagnostic")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledSmokeAuthenticationResponseOverflow(t *testing.T) {
|
||||
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(strings.Repeat("x", (1<<20)+1)))}, nil
|
||||
})
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter", APIKey: "private-key-marker"})
|
||||
if err != nil || report == nil || !report.Completed || report.Reason != promptkit.SmokeTestReasonAuthenticationResponseLimitExceeded || report.Failure != promptkit.ExecutionFailureOther || report.Facts.CredentialsAccepted != nil {
|
||||
t.Fatalf("overflow = %+v, %v", report, err)
|
||||
}
|
||||
}
|
||||
@@ -265,7 +265,7 @@ func newBackendCapacityEngine(
|
||||
options = append(options, promptkit.WithArtifactReader(reader))
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||
"backends": []any{backend},
|
||||
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
|
||||
"prompts": []any{definition},
|
||||
|
||||
38
configuration.go
Normal file
38
configuration.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||
)
|
||||
|
||||
// Configure composes PromptKit's settings rules with an application definition.
|
||||
// selectSettings identifies the application's PromptKit section and must be
|
||||
// non-nil. A nil selected section is left untouched. Existing resolution and
|
||||
// validation callbacks run first; callers may wrap the returned callbacks to
|
||||
// run additional work afterward. Defaults and version policy are preserved.
|
||||
func Configure[T any](definition configkit.Definition[T], selectSettings func(*T) *Settings) configkit.Definition[T] {
|
||||
previousResolve := definition.Resolve
|
||||
previousValidate := definition.Validate
|
||||
previousSensitive := definition.Sensitive
|
||||
identities := append([]configkit.SequenceIdentity(nil), definition.SequenceIdentities...)
|
||||
definition.SequenceIdentities = append(identities, configuration.SequenceIdentities()...)
|
||||
definition.Resolve = func(value *T, resolution *configkit.Resolution) error {
|
||||
if previousResolve != nil {
|
||||
if err := previousResolve(value, resolution); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
configuration.Resolve(selectSettings(value), resolution)
|
||||
return nil
|
||||
}
|
||||
definition.Validate = func(value *T, validation *configkit.Validation) {
|
||||
if previousValidate != nil {
|
||||
previousValidate(value, validation)
|
||||
}
|
||||
configuration.Validate(selectSettings(value), validation)
|
||||
}
|
||||
definition.Sensitive = func(path configkit.Path) bool {
|
||||
return (previousSensitive != nil && previousSensitive(path)) || configuration.Sensitive(path)
|
||||
}
|
||||
return definition
|
||||
}
|
||||
@@ -21,7 +21,7 @@ func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
if got, want := difference.Path.String(), "/profiles"; got != want {
|
||||
if got, want := difference.Path.String(), "/promptkit/profiles"; got != want {
|
||||
t.Fatalf("difference path = %q, want %q", got, want)
|
||||
}
|
||||
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||
@@ -81,6 +81,34 @@ func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
|
||||
inspect := func(endpoint string) promptkit.ConfigurationInspection {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("backend", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": endpoint}},
|
||||
})}}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return view
|
||||
}
|
||||
left := inspect(" HTTPS://example.invalid/v1 ")
|
||||
equal := inspect("https://example.invalid/v1")
|
||||
differences, err := left.Diff(equal)
|
||||
if err != nil || len(differences) != 0 {
|
||||
t.Fatalf("normalized diff = %+v, %v", differences, err)
|
||||
}
|
||||
changed := inspect("https://other.invalid/v1")
|
||||
differences, err = left.Diff(changed)
|
||||
if err != nil || len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||
t.Fatalf("changed diff = %+v, %v", differences, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||
"sequence": []any{"earlier", "unchanged"},
|
||||
@@ -94,7 +122,7 @@ func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("compare presence and sequence values: %v", err)
|
||||
}
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||
t.Fatalf("atomic backend list difference = %+v", differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
@@ -119,7 +147,7 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive values: %v", err)
|
||||
}
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||
t.Fatalf("sensitive backend list difference = %+v", differences)
|
||||
}
|
||||
secret := differences[0]
|
||||
@@ -148,7 +176,7 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
|
||||
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||
t.Fatal("mutating returned path reached retained inspection")
|
||||
}
|
||||
if _, exists := againByPath["/backends"]; !exists {
|
||||
if _, exists := againByPath["/promptkit/backends"]; !exists {
|
||||
t.Fatalf("original backend path was lost after mutation: %+v", again)
|
||||
}
|
||||
for _, difference := range again {
|
||||
@@ -173,7 +201,7 @@ func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||
}
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||
t.Fatalf("shape replacement differences = %+v", differences)
|
||||
}
|
||||
difference := differences[0]
|
||||
@@ -197,7 +225,7 @@ func comparisonInspectionFrom(t *testing.T, source, model string, extraParams ma
|
||||
backend["extra_params"] = extraParams
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping(source, map[string]any{
|
||||
"backends": []any{backend},
|
||||
"profiles": []any{map[string]any{
|
||||
"id": "offline", "backend": "local",
|
||||
|
||||
76
configuration_handoff_test.go
Normal file
76
configuration_handoff_test.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type handoffApplication struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
type handoffWrapper struct {
|
||||
result *configkit.Result[handoffApplication]
|
||||
override any
|
||||
useOverride bool
|
||||
inspection bool
|
||||
}
|
||||
|
||||
func (w *handoffWrapper) Value(path configkit.Path) (any, bool) {
|
||||
if w.useOverride {
|
||||
return w.override, true
|
||||
}
|
||||
return w.result.Value(path)
|
||||
}
|
||||
func (w *handoffWrapper) InspectSubtree(path configkit.Path) (configkit.Inspection, bool) {
|
||||
if !w.inspection {
|
||||
return configkit.Inspection{}, false
|
||||
}
|
||||
return w.result.InspectSubtree(path)
|
||||
}
|
||||
func (w *handoffWrapper) Diagnostics() []configkit.Diagnostic { return w.result.Diagnostics() }
|
||||
|
||||
func TestConstructorsShareCapturedResources(t *testing.T) {
|
||||
source := configkit.Bytes("prompt", []byte("promptkit:\n prompts:\n - id: example\n messages: [{role: user, content: hello}]\n output: {format: text, validation_mode: basic}\n"))
|
||||
definition := promptkit.Configure(configkit.Definition[handoffApplication]{}, func(value *handoffApplication) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.Load(source, configkit.Options[handoffApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wrapped := &handoffWrapper{result: result, inspection: true}
|
||||
applicationEngine, err := promptkit.NewFromConfig(wrapped, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(true))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := configkit.Input{Source: source}
|
||||
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, engine := range []*promptkit.Engine{applicationEngine, standalone} {
|
||||
prompt, err := engine.InspectPrompt(context.Background(), "example")
|
||||
if err != nil || prompt.Messages[0].Content != "hello" || prompt.OutputContract.RepairAttempts != 1 {
|
||||
t.Fatalf("prompt = %+v, %v", prompt, err)
|
||||
}
|
||||
backends, err := engine.ListBackends(context.Background())
|
||||
if err != nil || len(backends) != 0 {
|
||||
t.Fatalf("backends = %+v, %v", backends, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFromConfigRejectsInvalidHandoff(t *testing.T) {
|
||||
var nilResult *handoffWrapper
|
||||
if _, err := promptkit.NewFromConfig(nilResult, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil result: %v", err)
|
||||
}
|
||||
for _, wrapper := range []*handoffWrapper{{inspection: false}, {inspection: true, useOverride: true, override: "wrong"}} {
|
||||
if _, err := promptkit.NewFromConfig(wrapper, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("invalid handoff: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,8 +9,10 @@ import (
|
||||
)
|
||||
|
||||
// ConfigurationInspection is a read-only view of one Engine's captured
|
||||
// effective configuration and source history. It does not expose typed
|
||||
// configuration values, filesystem contexts, or runtime credentials.
|
||||
// effective configuration and source history focused on /promptkit. Metadata,
|
||||
// sources, and explicit path queries retain native whole-load evidence; the
|
||||
// focus is not an isolation boundary. It does not expose typed configuration
|
||||
// values, filesystem contexts, or runtime credentials.
|
||||
// Report content may include application prompt and provider data, so callers
|
||||
// choose where and to whom to disclose it.
|
||||
type ConfigurationInspection struct {
|
||||
@@ -30,7 +32,9 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
||||
// Diff compares this view's captured effective configuration with other. This
|
||||
// view is the earlier (left) value and other is the later (right) value. The
|
||||
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||
// source or resolution history alone does not create differences. It does not
|
||||
// source or resolution history alone does not create differences. Native
|
||||
// ancestor differences are retained, while sibling-only records are excluded.
|
||||
// It does not
|
||||
// reread external sources or compare referenced file contents, resolved
|
||||
// credential contents, later execution preparation, or request-specific
|
||||
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||
@@ -52,6 +56,14 @@ func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||
return v.source.Metadata()
|
||||
}
|
||||
|
||||
// HistoricalPaths lists native historical paths within the PromptKit focus.
|
||||
func (v ConfigurationInspection) HistoricalPaths() []configkit.Path {
|
||||
if v.source == nil {
|
||||
return nil
|
||||
}
|
||||
return v.source.HistoricalPaths()
|
||||
}
|
||||
|
||||
// Sources returns a detached list of loaded source descriptions.
|
||||
func (v ConfigurationInspection) Sources() []configkit.SourceInfo {
|
||||
if v.source == nil {
|
||||
@@ -78,7 +90,8 @@ func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explana
|
||||
return v.source.Explain(path)
|
||||
}
|
||||
|
||||
// WriteEffective writes ConfigKit's redacted effective YAML. Writer errors
|
||||
// WriteEffective writes the focused ConfigKit-redacted effective YAML. An absent
|
||||
// effective focus fails without output; explicit null renders normally. Writer errors
|
||||
// remain available through errors.Is and errors.As. The zero view returns
|
||||
// ErrInvalidConfig.
|
||||
func (v ConfigurationInspection) WriteEffective(w io.Writer) error {
|
||||
|
||||
@@ -19,9 +19,9 @@ import (
|
||||
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
||||
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("defaults", map[string]any{
|
||||
"backends": []any{map[string]any{
|
||||
"id": "local", "endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
||||
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY",
|
||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||
}},
|
||||
"collections": map[string]any{"literal": map[string]any{
|
||||
@@ -35,14 +35,14 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
||||
},
|
||||
}},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
||||
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("operator", map[string]any{
|
||||
"collections": map[string]any{"literal": map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
|
||||
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
|
||||
}},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatalf("construct layered engine: %v", err)
|
||||
}
|
||||
@@ -50,30 +50,35 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model")
|
||||
modelPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(0).Key("model")
|
||||
model, found := view.Explain(modelPath)
|
||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
|
||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||
}
|
||||
inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model")
|
||||
inheritedPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(1).Key("model")
|
||||
inherited, found := view.Explain(inheritedPath)
|
||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||
}
|
||||
if inheritedPath.String() != "/collections/literal/profiles/1/model" {
|
||||
if inheritedPath.String() != "/promptkit/collections/literal/profiles/1/model" {
|
||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||
}
|
||||
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token")
|
||||
secretPath := configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("token")
|
||||
endpointPath := configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")
|
||||
endpoint, found := view.Explain(endpointPath)
|
||||
if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") {
|
||||
t.Fatalf("normalized endpoint explanation = %+v", endpoint)
|
||||
}
|
||||
secret, found := view.Explain(secretPath)
|
||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||
}
|
||||
namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
|
||||
namespacedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
|
||||
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
||||
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
||||
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
||||
}
|
||||
inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
|
||||
inheritedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
|
||||
inheritedSecret, found := view.Explain(inheritedSecretPath)
|
||||
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
|
||||
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
|
||||
@@ -119,7 +124,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
||||
|
||||
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
|
||||
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("invalid backend", map[string]any{
|
||||
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid backend", map[string]any{
|
||||
"collections": map[string]any{"private": map[string]any{"backends": []any{
|
||||
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
|
||||
}}},
|
||||
@@ -134,9 +139,9 @@ func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
|
||||
func TestConfigurationInspectionRetainsApplicationVersionField(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("main", map[string]any{"version": "2"}),
|
||||
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("main", map[string]any{"version": "2"}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with accepted declaration: %v", err)
|
||||
@@ -145,7 +150,7 @@ func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
path := configkit.Keys("version")
|
||||
path := configkit.Keys("promptkit", "version")
|
||||
field, found := inspection.Explain(path)
|
||||
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
|
||||
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
||||
@@ -166,7 +171,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
||||
t.Fatalf("nil engine inspection = %v", err)
|
||||
}
|
||||
var empty promptkit.ConfigurationInspection
|
||||
path := configkit.Keys("profiles", "missing")
|
||||
path := configkit.Keys("promptkit", "profiles", "missing")
|
||||
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
||||
t.Fatal("zero inspection reported configuration")
|
||||
}
|
||||
@@ -180,7 +185,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
||||
t.Fatalf("zero explanation writer = %v", err)
|
||||
}
|
||||
|
||||
profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{
|
||||
profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("inspection test profile", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
|
||||
})}
|
||||
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
@@ -198,7 +203,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("effective writer error = %v", err)
|
||||
}
|
||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
|
||||
if err := view.WriteExplanation(writer, configkit.Keys("promptkit", "profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("explanation writer error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -225,7 +230,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
client := countingLLMClient{calls: &modelCalls}
|
||||
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{
|
||||
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(inputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("offline profile", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
@@ -233,7 +238,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{
|
||||
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(otherInputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("comparison profile", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
@@ -256,7 +261,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err != nil || len(differences) != 1 {
|
||||
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||
}
|
||||
if differences[0].Path.String() != "/profiles" ||
|
||||
if differences[0].Path.String() != "/promptkit/profiles" ||
|
||||
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||
t.Fatalf("captured difference = %+v", differences[0])
|
||||
@@ -268,7 +273,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||
}
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
|
||||
if _, found := view.Explain(configkit.Keys("promptkit", "prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
|
||||
t.Fatal("captured prompt content path was not inspectable")
|
||||
}
|
||||
var report bytes.Buffer
|
||||
|
||||
60
configuration_mounts_test.go
Normal file
60
configuration_mounts_test.go
Normal file
@@ -0,0 +1,60 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type mountApplication struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
type countingMountFS struct {
|
||||
fs.FS
|
||||
opens int
|
||||
}
|
||||
|
||||
func (c *countingMountFS) Open(name string) (fs.File, error) { c.opens++; return c.FS.Open(name) }
|
||||
|
||||
func TestSettingsAndCollectionMounts(t *testing.T) {
|
||||
files := &countingMountFS{FS: fstest.MapFS{
|
||||
"fragments/root.yaml": {Data: []byte("profiles:\n - id: main\n model: model\n")},
|
||||
"fragments/collection.yaml": {Data: []byte("prompts:\n - id: child\n messages: [{role: user, content_file: body.txt}]\n output: {format: text, validation_mode: basic}\n")},
|
||||
"fragments/item.yaml": {Data: []byte("id: item\nmessages: [{role: user, content: hello}]\noutput: {format: text, validation_mode: basic}\n")},
|
||||
"fragments/body.txt": {Data: []byte("message")},
|
||||
}}
|
||||
namespace := configkit.NewFilesystem("mounts", files)
|
||||
root := promptkit.SettingsInput(namespace.File("fragments/root.yaml"))
|
||||
collection, err := promptkit.CollectionInput("docs", namespace.File("fragments/collection.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := configkit.Input{Source: namespace.File("fragments/item.yaml"), Destination: configkit.Keys("promptkit", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem}
|
||||
for _, input := range []*configkit.Input{&root, &collection, &item} {
|
||||
input.Filesystem = &configkit.FilesystemContext{FS: &namespace, Root: "."}
|
||||
}
|
||||
bundled := promptkit.BundledOpenRouterUnderlay()
|
||||
if files.opens != 0 || root.Imports || !collection.Imports || root.Destination.String() != "/promptkit" || collection.Destination.String() != "/promptkit/collections/docs" {
|
||||
t.Fatalf("mount construction read files or chose wrong defaults: opens=%d", files.opens)
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[mountApplication]{}, func(value *mountApplication) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[mountApplication]{Definition: definition, Underlays: []configkit.Underlay{bundled, {Inputs: []configkit.Input{root, collection, item}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.Config.PromptKit.Profiles) < 1 || len(result.Config.PromptKit.Prompts) != 1 || len(result.Config.PromptKit.Collections["docs"].Prompts) != 1 {
|
||||
t.Fatalf("mounted settings = %+v", result.Config.PromptKit)
|
||||
}
|
||||
info, ok := result.Metadata().Lookup(configkit.Keys("promptkit", "collections", "docs", "prompts").Index(0).Key("messages").Index(0).Key("content_file"))
|
||||
if !ok || len(info.Origin.Locations) != 1 {
|
||||
t.Fatalf("origin = %+v", info)
|
||||
}
|
||||
context, ok := result.SourceContext(info.Origin.Locations[0].SourceID)
|
||||
if !ok || context.FS == nil || context.BaseDir != "fragments" {
|
||||
t.Fatalf("context = %+v", context)
|
||||
}
|
||||
}
|
||||
99
configuration_rules_test.go
Normal file
99
configuration_rules_test.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type rulesApplication struct {
|
||||
Version string `yaml:"version"`
|
||||
Other []rulesOther `yaml:"other"`
|
||||
PromptKit *promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
type rulesOther struct {
|
||||
ID string `yaml:"id"`
|
||||
}
|
||||
|
||||
func TestConfigureComposesApplicationRules(t *testing.T) {
|
||||
var events []string
|
||||
definition := configkit.Definition[rulesApplication]{
|
||||
Defaults: func() rulesApplication { return rulesApplication{Version: "app"} },
|
||||
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(v configkit.Version, validation *configkit.Validation) error {
|
||||
events = append(events, "version")
|
||||
return nil
|
||||
}},
|
||||
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("other"), "id")},
|
||||
Resolve: func(_ *rulesApplication, _ *configkit.Resolution) error {
|
||||
events = append(events, "application resolve")
|
||||
return nil
|
||||
},
|
||||
Validate: func(_ *rulesApplication, _ *configkit.Validation) { events = append(events, "application validate") },
|
||||
Sensitive: func(path configkit.Path) bool { return path.String() == "/application_secret" },
|
||||
}
|
||||
configured := promptkit.Configure(definition, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
|
||||
if len(definition.SequenceIdentities) != 1 || len(configured.SequenceIdentities) != 7 {
|
||||
t.Fatalf("registrations: %d, %d", len(definition.SequenceIdentities), len(configured.SequenceIdentities))
|
||||
}
|
||||
priorValidate := configured.Validate
|
||||
configured.Validate = func(value *rulesApplication, validation *configkit.Validation) {
|
||||
priorValidate(value, validation)
|
||||
if value.PromptKit == nil || value.PromptKit.Backends[0].Endpoint == nil || strings.TrimSpace(*value.PromptKit.Backends[0].Endpoint) != *value.PromptKit.Backends[0].Endpoint {
|
||||
validation.Error(configkit.Keys("promptkit"), "normalization did not run")
|
||||
}
|
||||
events = append(events, "after validate")
|
||||
}
|
||||
source := configkit.Bytes("rules", []byte("promptkit:\n backends:\n - id: local\n endpoint: ' https://example.test/v1/ '\n extra_params: {api_key: hidden}\nother: [{id: one}]\n"))
|
||||
result, err := configkit.Load(source, configkit.Options[rulesApplication]{Definition: configured})
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Config.Version != "app" || strings.Join(events, ",") != "version,application resolve,application validate,after validate" {
|
||||
t.Fatalf("callbacks: %+v, %+v", result.Config, events)
|
||||
}
|
||||
if !configured.Sensitive(configkit.Keys("application_secret")) || !configured.Sensitive(configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("api_key")) || configured.Sensitive(configkit.Keys("backends").Index(0).Key("extra_params").Key("api_key")) {
|
||||
t.Fatal("sensitivity composition failed")
|
||||
}
|
||||
var rendered strings.Builder
|
||||
if err := result.WriteEffective(&rendered); err != nil || strings.Contains(rendered.String(), "hidden") {
|
||||
t.Fatalf("redaction: %q, %v", rendered.String(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurePropagatesResolveError(t *testing.T) {
|
||||
want := errors.New("resolve failed")
|
||||
definition := promptkit.Configure(configkit.Definition[rulesApplication]{Resolve: func(*rulesApplication, *configkit.Resolution) error { return want }}, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
|
||||
err := definition.Resolve(&rulesApplication{}, nil)
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("resolve error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureReusableAcrossConcurrentLoads(t *testing.T) {
|
||||
definition := promptkit.Configure(configkit.Definition[rulesApplication]{}, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
|
||||
var group sync.WaitGroup
|
||||
for range 8 {
|
||||
group.Go(func() {
|
||||
for _, source := range []string{"{}", "promptkit: {prompts: [{id: test, output: {format: text, validation_mode: basic}}]}"} {
|
||||
result, err := configkit.Load(configkit.Bytes("settings", []byte(source)), configkit.Options[rulesApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
continue
|
||||
}
|
||||
if result.Config.PromptKit != nil && *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
|
||||
t.Error("default was not resolved")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
}
|
||||
@@ -8,12 +8,13 @@ import (
|
||||
"path/filepath"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/filecatalog"
|
||||
)
|
||||
|
||||
// CollectionInput mounts source at the selected collection's configuration
|
||||
// destination, collections/<namespace>, and enables ConfigKit imports for that
|
||||
// destination, /promptkit/collections/<namespace>, and enables ConfigKit imports for that
|
||||
// input. An invalid namespace returns an error matching ErrInvalidConfig. The
|
||||
// source is not read or validated until ConfigKit loads it. Callers may adjust
|
||||
// the returned input, including its filesystem context or imports setting.
|
||||
@@ -23,11 +24,23 @@ func CollectionInput(namespace string, source configkit.Source) (configkit.Input
|
||||
}
|
||||
return configkit.Input{
|
||||
Source: source,
|
||||
Destination: configkit.Keys("collections", namespace),
|
||||
Destination: configkit.Keys("promptkit", "collections", namespace),
|
||||
Imports: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SettingsInput mounts a wrapperless PromptKit settings fragment at /promptkit.
|
||||
// It does not acquire the source; callers may adjust the returned input.
|
||||
func SettingsInput(source configkit.Source) configkit.Input {
|
||||
return configkit.Input{Source: source, Destination: configkit.Keys("promptkit")}
|
||||
}
|
||||
|
||||
// BundledOpenRouterUnderlay selects the bundled OpenRouter settings fragment.
|
||||
// It does not acquire its source until ConfigKit loads it.
|
||||
func BundledOpenRouterUnderlay() configkit.Underlay {
|
||||
return configuration.BundledOpenRouterUnderlay()
|
||||
}
|
||||
|
||||
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
||||
// directory as one strict-composition group. The returned order is deterministic;
|
||||
// it does not assign precedence among those files. Discovery is recursive over
|
||||
|
||||
@@ -23,8 +23,8 @@ func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if input.Destination.String() != "/collections/docs" || !input.Imports {
|
||||
t.Fatalf("collection input = %+v, want destination /collections/docs with imports enabled", input)
|
||||
if input.Destination.String() != "/promptkit/collections/docs" || !input.Imports {
|
||||
t.Fatalf("collection input = %+v, want destination /promptkit/collections/docs with imports enabled", input)
|
||||
}
|
||||
for _, namespace := range []string{"", " ", "docs/api"} {
|
||||
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
@@ -51,18 +51,59 @@ func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||
Main: &configkit.Input{
|
||||
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
|
||||
Destination: configkit.Keys("collections", "docs"),
|
||||
Destination: configkit.Keys("promptkit", "collections", "docs"),
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with native collection import: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
|
||||
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionInputUsesMainRelativeContext(t *testing.T) {
|
||||
filesystem := configkit.NewFilesystem("application", fstest.MapFS{
|
||||
"site/main.yaml": {Data: []byte("version: '2'\n")},
|
||||
"site/prompts/greeting.yaml": {Data: []byte(`id: greeting
|
||||
messages:
|
||||
- role: user
|
||||
content_file: greeting.txt
|
||||
output: {format: text, validation_mode: none}
|
||||
`)},
|
||||
"site/prompts/greeting.txt": {Data: []byte("Hello from the imported prompt's directory.")},
|
||||
})
|
||||
companion, err := promptkit.CollectionInput("support", configkit.Bytes("companion prompts", []byte(`
|
||||
config:
|
||||
imports:
|
||||
- dir: prompts
|
||||
pattern: ["*.yaml", "*.yml"]
|
||||
destination: [prompts]
|
||||
placement: sequence-item
|
||||
required: true
|
||||
`)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
companion.MainRelative = configkit.MainRelative()
|
||||
main := configkit.Input{Source: filesystem.File("site/main.yaml")}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Main: &main,
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{companion}}},
|
||||
}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with main-relative imports: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(t.Context(), "support/greeting")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(inspection.Messages) != 1 || inspection.Messages[0].Content != "Hello from the imported prompt's directory." {
|
||||
t.Fatalf("unexpected imported messages: %+v", inspection.Messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"collection.yaml": {Data: []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n")},
|
||||
@@ -78,14 +119,14 @@ func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||
Main: &configkit.Input{
|
||||
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
|
||||
Destination: configkit.Keys("collections", "docs"),
|
||||
Destination: configkit.Keys("promptkit", "collections", "docs"),
|
||||
},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with embedded collection import: %v", err)
|
||||
}
|
||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
|
||||
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||
endpoint, model := "http://example.test/v1", "model"
|
||||
lowerName := name
|
||||
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
|
||||
Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
|
||||
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
|
||||
}),
|
||||
}}})
|
||||
@@ -49,7 +49,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||
if test.clear {
|
||||
env = ""
|
||||
}
|
||||
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
|
||||
cfg.Sources.Main = &configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
|
||||
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
|
||||
})}
|
||||
var calls int
|
||||
@@ -73,7 +73,7 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||
|
||||
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
||||
backendID, model := "workstation", "offline-model"
|
||||
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
||||
prompt := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{
|
||||
"prompts": []any{map[string]any{
|
||||
"id": "hello",
|
||||
"default_profile": "offline",
|
||||
@@ -86,10 +86,10 @@ func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T)
|
||||
localBackend.ID = backendID
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
{Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
|
||||
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
|
||||
localBackend,
|
||||
})},
|
||||
{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||
{ID: "offline", Backend: &backendID, Model: &model},
|
||||
})},
|
||||
}}},
|
||||
@@ -116,11 +116,11 @@ func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
||||
model := "configured-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
{Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
|
||||
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
|
||||
{ID: openRouterID, Endpoint: &openRouterEndpoint},
|
||||
{ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint},
|
||||
})},
|
||||
{Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
|
||||
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
|
||||
{ID: "openrouter-profile", Backend: &openRouterID, Model: &model},
|
||||
{ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model},
|
||||
})},
|
||||
@@ -134,7 +134,7 @@ func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
||||
"rakestrawhome-profile": rakestrawhomeID,
|
||||
} {
|
||||
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||
if err != nil || inspection.EffectiveModelParams.BackendID != backendID {
|
||||
if err != nil || inspection.Backend == nil || *inspection.Backend != backendID {
|
||||
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
||||
}
|
||||
}
|
||||
@@ -170,7 +170,7 @@ prompts:
|
||||
t.Fatalf("select operator inputs: %v", err)
|
||||
}
|
||||
programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1"
|
||||
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
|
||||
programmatic := configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
|
||||
{ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel},
|
||||
})}
|
||||
for _, test := range []struct {
|
||||
@@ -183,9 +183,9 @@ prompts:
|
||||
{name: "programmatic", apply: true, want: "programmatic-model"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
underlays := []configkit.Underlay{{Inputs: embedded}}
|
||||
underlays := []configkit.Underlay{{Inputs: mountSettingsInputs(embedded)}}
|
||||
if test.apply {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: operator})
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: mountSettingsInputs(operator)})
|
||||
}
|
||||
if test.name == "programmatic" {
|
||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
|
||||
@@ -207,7 +207,7 @@ prompts:
|
||||
|
||||
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("invalid", map[string]any{"profiles": []any{map[string]any{"id": "broken", "unknown": true}}}),
|
||||
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid", map[string]any{"prompts": []any{map[string]any{"id": "broken", "output": map[string]any{"unknown": true}}}}),
|
||||
}}})
|
||||
var loadErr *configkit.LoadError
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||
@@ -215,6 +215,36 @@ func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicConstructionPreservesMalformedYAMLCause(t *testing.T) {
|
||||
const sourceLabel = "prompts.yml"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Bytes(sourceLabel, []byte("profiles: [\n")),
|
||||
}}})
|
||||
var loadErr *configkit.LoadError
|
||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||
t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err)
|
||||
}
|
||||
cause := errors.Unwrap(err)
|
||||
var causeLoadErr *configkit.LoadError
|
||||
if cause == nil || errors.Is(cause, promptkit.ErrInvalidConfig) || !errors.As(cause, &causeLoadErr) || causeLoadErr != loadErr {
|
||||
t.Fatalf("construction cause = %v, want the original loading failure without a category branch", cause)
|
||||
}
|
||||
for _, diagnostic := range loadErr.Diagnostics() {
|
||||
if diagnostic.Code != "yaml.invalid" {
|
||||
continue
|
||||
}
|
||||
if diagnostic.Primary == nil {
|
||||
t.Fatal("YAML diagnostic has no source location")
|
||||
}
|
||||
source, ok := loadErr.Source(diagnostic.Primary.SourceID)
|
||||
if !ok || source.Label != sourceLabel {
|
||||
t.Fatalf("YAML diagnostic source = %+v, found=%v, want %q", source, ok, sourceLabel)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics())
|
||||
}
|
||||
|
||||
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
||||
main := configkit.Input{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||
@@ -227,10 +257,10 @@ func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T
|
||||
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||
{Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
|
||||
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
|
||||
{ID: backendID, Endpoint: &endpoint},
|
||||
})},
|
||||
{Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
|
||||
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
|
||||
{ID: "application-profile", Backend: &backendID, Model: &model},
|
||||
})},
|
||||
}}},
|
||||
@@ -242,8 +272,9 @@ func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configured profile: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
||||
backendInspection, err := engine.InspectBackend(context.Background(), backendID)
|
||||
if err != nil || inspection.Endpoint != nil || backendInspection.Endpoint != endpoint {
|
||||
t.Fatalf("profile/backend = %+v / %+v, %v", inspection, backendInspection, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -273,7 +304,7 @@ func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
||||
authenticationEndpoint = "https://mutated.example/key"
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}},
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
|
||||
@@ -291,7 +322,7 @@ func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
||||
"https://backend.example/v1",
|
||||
"not-yet-listed",
|
||||
"https://probe.example/ready?check=1",
|
||||
" OPENROUTER_KEY ",
|
||||
"openrouter_key",
|
||||
"https://probe.example/key?scope=current",
|
||||
} {
|
||||
if !strings.Contains(contents, want) {
|
||||
|
||||
@@ -11,6 +11,14 @@ import (
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func mountSettingsInputs(inputs []configkit.Input) []configkit.Input {
|
||||
result := append([]configkit.Input(nil), inputs...)
|
||||
for i := range result {
|
||||
result[i].Destination = configkit.Keys("promptkit")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type fakeLLMClient struct {
|
||||
response *promptkit.GenerateResponse
|
||||
responses []*promptkit.GenerateResponse
|
||||
@@ -59,7 +67,7 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
|
||||
|
||||
func simpleConfig(content string) promptkit.Config {
|
||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
||||
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("consumer test", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "profile",
|
||||
"endpoint": "http://example.test/v1", "model": "model",
|
||||
}},
|
||||
|
||||
97
context_identity_contract_test.go
Normal file
97
context_identity_contract_test.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type contextIdentityResponseBody struct {
|
||||
ctx context.Context
|
||||
cause error
|
||||
}
|
||||
|
||||
func (b contextIdentityResponseBody) Read([]byte) (int, error) { <-b.ctx.Done(); return 0, b.cause }
|
||||
func (contextIdentityResponseBody) Close() error { return nil }
|
||||
|
||||
func TestExecutionContextAndOriginalCauseIdentities(t *testing.T) {
|
||||
for _, split := range []bool{false, true} {
|
||||
for _, kind := range []string{"caller cancel", "caller deadline", "body operation deadline"} {
|
||||
t.Run(map[bool]string{false: "run/", true: "prepared/"}[split]+kind, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
cause := errors.New("private original cause")
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
defer cancel(nil)
|
||||
want := error(context.Canceled)
|
||||
failure := promptkit.ExecutionFailureCanceled
|
||||
if kind == "caller deadline" {
|
||||
var stop context.CancelFunc
|
||||
ctx, stop = context.WithTimeoutCause(context.Background(), time.Second, cause)
|
||||
defer stop()
|
||||
}
|
||||
if kind != "caller cancel" {
|
||||
want = context.DeadlineExceeded
|
||||
failure = promptkit.ExecutionFailureDeadline
|
||||
}
|
||||
cfg := simpleConfig("message")
|
||||
cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if kind == "body operation deadline" {
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: contextIdentityResponseBody{ctx: r.Context(), cause: cause}}, nil
|
||||
}
|
||||
if kind == "caller cancel" {
|
||||
cancel(cause)
|
||||
} else {
|
||||
<-ctx.Done()
|
||||
}
|
||||
return nil, context.Cause(ctx)
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var events []promptkit.ExecutionEvent
|
||||
req := simpleGenerationRequest()
|
||||
req.OperationTimeout = time.Minute
|
||||
if kind == "body operation deadline" {
|
||||
req.OperationTimeout = time.Second
|
||||
}
|
||||
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
|
||||
var result *promptkit.RunResult
|
||||
if split {
|
||||
handle, prepareErr := engine.PrepareExecution(ctx, req)
|
||||
if prepareErr != nil {
|
||||
t.Fatal(prepareErr)
|
||||
}
|
||||
result, err = engine.RunPrepared(ctx, handle)
|
||||
} else {
|
||||
result, err = engine.Run(ctx, req)
|
||||
}
|
||||
for _, identity := range []error{want, cause, promptkit.ErrLLMGenerate, promptkit.ErrTransport} {
|
||||
if !errors.Is(err, identity) {
|
||||
t.Errorf("error=%v, want identity %v", err, identity)
|
||||
}
|
||||
}
|
||||
if kind != "body operation deadline" {
|
||||
var transportErr *url.Error
|
||||
if !errors.As(err, &transportErr) {
|
||||
t.Errorf("lost HTTP transport error: %v", err)
|
||||
}
|
||||
}
|
||||
if result != nil || errors.Is(err, promptkit.ErrOperationTimeout) != (kind == "body operation deadline") || strings.Contains(err.Error(), cause.Error()) {
|
||||
t.Fatalf("incorrect outcome or disclosure: result=%v error=%v", result, err)
|
||||
}
|
||||
if len(events) != 3 || events[1].Failure != failure || events[2].Kind != promptkit.ExecutionFinished || events[2].Failure != failure {
|
||||
t.Fatalf("attempt/terminal classification = %+v", events)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
141
continuation_contract_test.go
Normal file
141
continuation_contract_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestContinuationBranchesAcrossEngines(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
firstClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "first answer"}}
|
||||
firstEngine := newSimpleEngine(t, "first question", promptkit.WithLLMClient(firstClient))
|
||||
initial, err := firstEngine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, APIKey: "synthetic-first-key"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := initial.Exchange.Details()
|
||||
secondClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "follow-up answer"}}
|
||||
secondEngine := newNamespacedContractEngine(t, secondClient)
|
||||
scoped, err := secondEngine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: initial.Exchange, SessionID: "new-session", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON}, AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "branch one"}}}
|
||||
branch, err := scoped.Run(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request.AppendedMessages[0].Content = "branch two"
|
||||
sibling, err := scoped.Run(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next, err := scoped.Run(ctx, promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: branch.Exchange})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if branch.ParentRunID != initial.RunID || sibling.ParentRunID != initial.RunID || next.ParentRunID != branch.RunID || branch.PromptID != "alpha/shared" || branch.SelectedProfileID != "alpha/shared" || branch.SessionID != "new-session" {
|
||||
t.Fatalf("ancestry or selections incorrect: %#v %#v %#v", branch, sibling, next)
|
||||
}
|
||||
if !reflect.DeepEqual(initial.Exchange.Details(), original) {
|
||||
t.Fatal("branching changed source")
|
||||
}
|
||||
if branch.Validation.Status != promptkit.ValidationFailed || branch.ModelName != "alpha-model" {
|
||||
t.Fatalf("follow-up settings: %#v", branch)
|
||||
}
|
||||
branchMessages := branch.Exchange.Details().Messages
|
||||
siblingMessages := sibling.Exchange.Details().Messages
|
||||
if len(branchMessages) != 5 || len(siblingMessages) != 5 || branchMessages[3].Content != "branch one" || siblingMessages[3].Content != "branch two" || len(next.Exchange.Details().Messages) != 7 {
|
||||
t.Fatalf("branch histories = %#v / %#v", branchMessages, siblingMessages)
|
||||
}
|
||||
encoded, err := json.Marshal(branch.Exchange.Details())
|
||||
if err != nil || !json.Valid(encoded) {
|
||||
t.Fatalf("details JSON = %s, %v", encoded, err)
|
||||
}
|
||||
branchMessages[0].Content = "changed details"
|
||||
branch.RawOutput = "changed result"
|
||||
if initial.Exchange.Details().Messages[0].Content != "first question" || branch.Exchange.Details().Messages[4].Content != "follow-up answer" {
|
||||
t.Fatal("caller mutation changed captured history")
|
||||
}
|
||||
for _, request := range secondClient.requests {
|
||||
if request.APIKey == "synthetic-first-key" || request.Prompt.Messages[0].Content != "first question" || request.Prompt.Messages[1].Content != "first answer" {
|
||||
t.Fatalf("unexpected follow-up request: %#v", request)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinuationDoesNotReloadCapturedInputs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "input.txt")
|
||||
if err := os.WriteFile(path, []byte("original input"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: ""}}
|
||||
engine := newSimpleEngine(t, `{{input "source"}}`, promptkit.WithLLMClient(client))
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(path)}, CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("new input"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(path)}, ContinueFrom: initial.Exchange}
|
||||
prepared, err := engine.PrepareExecution(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := engine.RunPrepared(ctx, prepared)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "original input"}, {Role: promptkit.RoleAssistant, Content: ""}, {Role: promptkit.RoleUser, Content: "new input"}, {Role: promptkit.RoleAssistant, Content: ""}}
|
||||
if !reflect.DeepEqual(result.Exchange.Details().Messages, want) {
|
||||
t.Fatalf("history = %#v", result.Exchange.Details().Messages)
|
||||
}
|
||||
if result, err := engine.Run(ctx, request); err == nil || result != nil {
|
||||
t.Fatalf("missing new input = %#v, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinuationSourceSupportsConcurrentRuns(t *testing.T) {
|
||||
client := &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const count = 8
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, count)
|
||||
for i := 0; i < count; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||
if err == nil && (result.Exchange == nil || result.ParentRunID != initial.RunID) {
|
||||
t.Error("missing continuation ancestry")
|
||||
}
|
||||
errors <- err
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(initial.Exchange.Details().Messages) != 2 {
|
||||
t.Fatal("source changed")
|
||||
}
|
||||
}
|
||||
192
convert.go
192
convert.go
@@ -1,15 +1,23 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"unicode/utf8"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
var history *domain.Exchange
|
||||
if req.ContinueFrom != nil && req.ContinueFrom.internal == nil {
|
||||
return domain.RunRequest{}, fmt.Errorf("continuation exchange is empty")
|
||||
}
|
||||
if req.ContinueFrom != nil {
|
||||
history = req.ContinueFrom.internal
|
||||
}
|
||||
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
||||
if err != nil {
|
||||
return domain.RunRequest{}, err
|
||||
@@ -23,7 +31,16 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
callback := req.Observer
|
||||
observer = func(event domain.ExecutionEvent) { callback(fromDomainExecutionEvent(event)) }
|
||||
}
|
||||
var onPrepared func(context.Context, *domain.PreparedRun)
|
||||
if callback := req.OnPrepared; callback != nil {
|
||||
onPrepared = func(ctx context.Context, prepared *domain.PreparedRun) {
|
||||
callback(ctx, *fromDomainPreparedRun(prepared))
|
||||
}
|
||||
}
|
||||
return domain.RunRequest{
|
||||
OperationTimeout: req.OperationTimeout,
|
||||
OnPrepared: onPrepared,
|
||||
CaptureExchange: req.CaptureExchange || req.ContinueFrom != nil,
|
||||
PromptID: req.PromptID,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
@@ -35,6 +52,7 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
AppendedMessages: appendedMessages,
|
||||
TransientRetries: copyIntPtr(req.TransientRetries),
|
||||
Observer: observer,
|
||||
ContinueFrom: history,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -71,6 +89,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
return nil
|
||||
}
|
||||
return &PreparedRun{
|
||||
ParentRunID: prepared.ParentRunID,
|
||||
PromptID: prepared.PromptID,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
@@ -81,6 +100,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
InputHashes: copyStringMap(prepared.InputHashes),
|
||||
SessionID: prepared.SessionID,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
MessageContentHash: prepared.MessageContentHash,
|
||||
Messages: fromDomainRenderedMessages(prepared.Messages),
|
||||
StartTime: prepared.StartTime,
|
||||
EndTime: prepared.EndTime,
|
||||
@@ -92,7 +112,13 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
var exchange *Exchange
|
||||
if result.Exchange != nil {
|
||||
exchange = &Exchange{internal: result.Exchange}
|
||||
}
|
||||
return &RunResult{
|
||||
Exchange: exchange,
|
||||
ParentRunID: result.ParentRunID,
|
||||
RunID: result.RunID,
|
||||
Artifact: fromDomainArtifact(result.Artifact),
|
||||
RawOutput: result.RawOutput,
|
||||
@@ -229,37 +255,58 @@ func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, ext
|
||||
}
|
||||
}
|
||||
|
||||
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
||||
if inspection == nil {
|
||||
return nil
|
||||
}
|
||||
return &ProfileInspection{
|
||||
ProfileID: inspection.ProfileID,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
||||
APIKeyRequired: inspection.APIKeyRequired,
|
||||
func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
|
||||
out := &ProfileInspection{ProfileID: v.ID, Backend: copyStringPtr(v.Backend), Endpoint: copyStringPtr(v.Endpoint), Model: copyStringPtr(v.Model), Temperature: copyFloat64Ptr(v.Temperature), TopP: copyFloat64Ptr(v.TopP), MaxTokens: copyIntPtr(v.MaxTokens), TimeoutSeconds: copyIntPtr(v.TimeoutSeconds), ServiceTier: copyStringPtr(v.ServiceTier), ReasoningEffort: copyStringPtr(v.ReasoningEffort), APIKeyEnv: copyStringPtr(v.APIKeyEnv), APIKeyRequired: copyBoolPtr(v.APIKeyRequired)}
|
||||
if v.ExtraParams != nil {
|
||||
data := copyAnyMap(v.ExtraParams)
|
||||
out.ExtraParams = &data
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
||||
if inspection == nil {
|
||||
return nil
|
||||
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
||||
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||
if v.ExtraParams != nil {
|
||||
data := copyAnyMap(v.ExtraParams)
|
||||
out.ExtraParams = &data
|
||||
}
|
||||
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
||||
for i, input := range inspection.Inputs {
|
||||
inputs[i] = PromptInputDefinition{
|
||||
Name: input.Name,
|
||||
Required: input.Required,
|
||||
ContentType: input.ContentType,
|
||||
Description: input.Description,
|
||||
if v.SmokeTests != nil {
|
||||
out.SmokeTests = &BackendSmokeTestConfig{DefaultProfile: copyStringPtr(v.SmokeTests.DefaultProfile)}
|
||||
if v.SmokeTests.Reachability != nil {
|
||||
out.SmokeTests.Reachability = &ReachabilityProbeConfig{Endpoint: copyStringPtr(v.SmokeTests.Reachability.Endpoint)}
|
||||
}
|
||||
if v.SmokeTests.Authentication != nil {
|
||||
out.SmokeTests.Authentication = &AuthenticationProbeConfig{Kind: copyStringPtr(v.SmokeTests.Authentication.Kind), Endpoint: copyStringPtr(v.SmokeTests.Authentication.Endpoint)}
|
||||
}
|
||||
}
|
||||
return &PromptInspection{
|
||||
PromptID: inspection.PromptID,
|
||||
PromptHash: inspection.PromptHash,
|
||||
DefaultProfileID: inspection.DefaultProfileID,
|
||||
Inputs: inputs,
|
||||
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
||||
return out
|
||||
}
|
||||
|
||||
func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
|
||||
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), Schema: jsonvalue.CloneTrusted(v.Output.Schema), SchemaFile: copyStringPtr(v.Output.SchemaFile), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: v.Output.RepairAttempts}}
|
||||
if v.Inputs != nil {
|
||||
inputs := make([]PromptInputInspection, len(*v.Inputs))
|
||||
for i, input := range *v.Inputs {
|
||||
inputs[i] = PromptInputInspection{Name: input.Name, Required: copyBoolPtr(input.Required), ContentType: copyStringPtr(input.ContentType), Description: copyStringPtr(input.Description)}
|
||||
}
|
||||
out.Inputs = &inputs
|
||||
}
|
||||
out.Messages = make([]PromptMessageInspection, len(v.Messages))
|
||||
for i, message := range v.Messages {
|
||||
out.Messages[i] = PromptMessageInspection{Role: message.Role, Content: message.Content}
|
||||
if message.CacheControl != nil {
|
||||
out.Messages[i].CacheControl = &PromptCacheControlInspection{Type: CacheControlType(message.CacheControl.Type), TTL: copyStringPtr(message.CacheControl.TTL)}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyBoolPtr(src *bool) *bool {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
value := *src
|
||||
return &value
|
||||
}
|
||||
|
||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||
@@ -271,20 +318,16 @@ func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence)
|
||||
}
|
||||
}
|
||||
|
||||
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
||||
func toDomainOutputContractPtr(contract *OutputContractInput) *domain.OutputContractInput {
|
||||
if contract == nil {
|
||||
return nil
|
||||
}
|
||||
out := toDomainOutputContract(*contract)
|
||||
return &out
|
||||
}
|
||||
|
||||
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
||||
return domain.OutputContract{
|
||||
return &domain.OutputContractInput{
|
||||
Format: domain.OutputFormat(contract.Format),
|
||||
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||
Schema: contract.Schema,
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
RepairAttempts: copyIntPtr(contract.RepairAttempts),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -292,6 +335,7 @@ func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
||||
return OutputContract{
|
||||
Format: OutputFormat(contract.Format),
|
||||
ValidationMode: ValidationMode(contract.ValidationMode),
|
||||
Schema: jsonvalue.CloneTrusted(contract.Schema),
|
||||
SchemaPath: contract.SchemaPath,
|
||||
RepairAttempts: contract.RepairAttempts,
|
||||
}
|
||||
@@ -395,88 +439,10 @@ func copyAnyMap(src map[string]any) map[string]any {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = copyAny(v)
|
||||
}
|
||||
return out
|
||||
return jsonvalue.CloneTrusted(src).(map[string]any)
|
||||
}
|
||||
|
||||
func copyAny(value any) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
switch v := value.(type) {
|
||||
case map[string]any:
|
||||
return copyAnyMap(v)
|
||||
case []any:
|
||||
out := make([]any, len(v))
|
||||
for i, item := range v {
|
||||
out[i] = copyAny(item)
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
return copyStringSlice(v)
|
||||
case []byte:
|
||||
return copyBytes(v)
|
||||
default:
|
||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
||||
}
|
||||
}
|
||||
|
||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
||||
if !value.IsValid() {
|
||||
return value
|
||||
}
|
||||
|
||||
switch value.Kind() {
|
||||
case reflect.Interface:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
copied := copyReflectValue(value.Elem())
|
||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||
return copied
|
||||
}
|
||||
out := reflect.New(value.Type()).Elem()
|
||||
out.Set(copied)
|
||||
return out
|
||||
case reflect.Pointer:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.New(value.Type().Elem())
|
||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
||||
return out
|
||||
case reflect.Map:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||
iter := value.MapRange()
|
||||
for iter.Next() {
|
||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
||||
}
|
||||
return out
|
||||
case reflect.Slice:
|
||||
if value.IsNil() {
|
||||
return reflect.Zero(value.Type())
|
||||
}
|
||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||
}
|
||||
return out
|
||||
case reflect.Array:
|
||||
out := reflect.New(value.Type()).Elem()
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
func copyAny(value any) any { return jsonvalue.CloneTrusted(value) }
|
||||
|
||||
func copyStringSlice(src []string) []string {
|
||||
if src == nil {
|
||||
|
||||
82
coordinate.go
Normal file
82
coordinate.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
// ConversationTurn is delivered before each requested continuation. Result is
|
||||
// nil on the initial callback; RemainingTurns counts additional runs allowed.
|
||||
type ConversationTurn struct {
|
||||
Exchange *Exchange
|
||||
Result *RunResult
|
||||
RemainingTurns int
|
||||
}
|
||||
|
||||
// CoordinateRequest selects a starting exchange, turn budget and callback.
|
||||
// Observer selects continuation runs only, independently of callback calls.
|
||||
type CoordinateRequest struct {
|
||||
Start *Exchange
|
||||
AdditionalTurns int
|
||||
Next func(context.Context, ConversationTurn) (*RunRequest, error)
|
||||
Observer ExecutionObserver
|
||||
}
|
||||
|
||||
// CoordinationOutcome distinguishes caller-directed stopping from exhaustion.
|
||||
type CoordinationOutcome string
|
||||
|
||||
const (
|
||||
// CoordinationStopped means the callback chose not to continue.
|
||||
CoordinationStopped CoordinationOutcome = "stopped"
|
||||
// CoordinationExhausted means another run was requested without budget.
|
||||
CoordinationExhausted CoordinationOutcome = "exhausted"
|
||||
)
|
||||
|
||||
// CoordinationResult contains the normal outcome or a response withheld by
|
||||
// cancellation before delivery to the callback.
|
||||
type CoordinationResult struct {
|
||||
Outcome CoordinationOutcome `json:"outcome,omitempty"`
|
||||
Undelivered *RunResult `json:"undelivered,omitempty"`
|
||||
}
|
||||
|
||||
// Coordinate advances one branch from Start using ordinary runs. Next is called
|
||||
// initially even for a zero budget and receives each completed result once,
|
||||
// including the final permitted response. The initial exchange is not itself a
|
||||
// newly delivered result. Each started continuation consumes one turn; retries
|
||||
// and structural repairs do not. Callback work runs after admission is released.
|
||||
// A nil request stops; requesting a run without remaining budget exhausts it.
|
||||
// Callback and execution errors retain their identities. Cancellation observed
|
||||
// before delivery returns the single completed response as Undelivered;
|
||||
// cancellation after callback invocation does not replay that response. Context
|
||||
// cancellation takes precedence over a callback's returned action or error.
|
||||
// Start must be a captured exchange, Next non-nil, and AdditionalTurns
|
||||
// nonnegative; invalid values match ErrInvalidRequest. A nil engine returns
|
||||
// ErrInvalidConfig. The selected request is copied and its ContinueFrom is
|
||||
// replaced by the coordinator's current exchange. The request observer runs
|
||||
// before the coordinator observer for each continuation.
|
||||
func (e *Engine) Coordinate(ctx context.Context, req CoordinateRequest) (CoordinationResult, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return CoordinationResult{}, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
if req.Start == nil || req.Start.internal == nil || req.Next == nil || req.AdditionalTurns < 0 {
|
||||
return CoordinationResult{}, fmt.Errorf("%w: invalid coordination request", ErrInvalidRequest)
|
||||
}
|
||||
outcome, err := usecase.Coordinate(ctx, req.Start.internal, req.AdditionalTurns,
|
||||
func(ctx context.Context, turn usecase.ConversationTurn[RunResult]) (*RunRequest, error) {
|
||||
return req.Next(ctx, ConversationTurn{Exchange: &Exchange{internal: turn.Exchange}, Result: turn.Result, RemainingTurns: turn.RemainingTurns})
|
||||
},
|
||||
func(ctx context.Context, current *domain.Exchange, selected *RunRequest) (*RunResult, *domain.Exchange, error) {
|
||||
request := *selected
|
||||
request.ContinueFrom = &Exchange{internal: current}
|
||||
request.Observer = ComposeObservers(request.Observer, req.Observer)
|
||||
result, runErr := e.Run(ctx, request)
|
||||
if runErr != nil {
|
||||
return nil, nil, runErr
|
||||
}
|
||||
return result, result.Exchange.internal, nil
|
||||
})
|
||||
return CoordinationResult{Outcome: CoordinationOutcome(outcome.Outcome), Undelivered: outcome.Undelivered}, err
|
||||
}
|
||||
61
coordinate_contract_test.go
Normal file
61
coordinate_contract_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestCoordinatePublicBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, request := range []promptkit.CoordinateRequest{{}, {Start: &promptkit.Exchange{}, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }}, {Start: initial.Exchange, AdditionalTurns: -1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }}} {
|
||||
if _, err := engine.Coordinate(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("invalid request error = %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := (*promptkit.Engine)(nil).Coordinate(ctx, promptkit.CoordinateRequest{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil engine error = %v", err)
|
||||
}
|
||||
callbacks := 0
|
||||
selected := &promptkit.RunRequest{PromptID: "prompt", ContinueFrom: &promptkit.Exchange{}}
|
||||
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 2, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
callbacks++
|
||||
if callbacks == 1 {
|
||||
if turn.Result != nil || turn.RemainingTurns != 2 {
|
||||
t.Fatalf("initial turn = %#v", turn)
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
if turn.Result == nil || turn.Exchange.Details().ParentRunID == "" {
|
||||
t.Fatalf("delivered turn = %#v", turn)
|
||||
}
|
||||
if callbacks == 2 {
|
||||
turn.Result.Exchange = nil
|
||||
*turn.Exchange = promptkit.Exchange{}
|
||||
return selected, nil
|
||||
}
|
||||
if turn.Result.ParentRunID == initial.RunID {
|
||||
t.Fatal("callback mutation redirected branch")
|
||||
}
|
||||
return nil, nil
|
||||
}})
|
||||
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbacks != 3 || selected.ContinueFrom == nil {
|
||||
t.Fatalf("result=%#v error=%v callbacks=%d", result, err, callbacks)
|
||||
}
|
||||
stop, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }})
|
||||
if err != nil || stop.Outcome != promptkit.CoordinationStopped {
|
||||
t.Fatalf("zero budget = %#v, %v", stop, err)
|
||||
}
|
||||
sentinel := errors.New("callback")
|
||||
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, sentinel }})
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("callback error = %v", err)
|
||||
}
|
||||
}
|
||||
86
coordinate_controls_contract_test.go
Normal file
86
coordinate_controls_contract_test.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if req.Prompt.SessionID == "continuation" {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "answer", Usage: promptkit.TokenUsage{PromptTokens: 2}}, nil
|
||||
}), 1, capacityInt(0), nil)
|
||||
parent := context.Background()
|
||||
initial, err := engine.Run(parent, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var group promptkit.ExecutionGroup
|
||||
var child context.Context
|
||||
var details []promptkit.PreparedRun
|
||||
var requestEvents, coordinatorEvents []promptkit.ExecutionEvent
|
||||
start := time.Now()
|
||||
result, err := engine.Coordinate(parent, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 2, Observer: promptkit.ComposeObservers(group.Observe, func(e promptkit.ExecutionEvent) { coordinatorEvents = append(coordinatorEvents, e) }), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result != nil {
|
||||
if ctx.Err() != nil || child == nil || !errors.Is(child.Err(), context.Canceled) {
|
||||
t.Fatal("child was not cleaned up before callback")
|
||||
}
|
||||
if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second}); err != nil {
|
||||
t.Fatalf("assessment: %v", err)
|
||||
}
|
||||
}
|
||||
if turn.RemainingTurns == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &promptkit.RunRequest{PromptID: "prompt", SessionID: "continuation", OperationTimeout: 3 * time.Second, Observer: func(e promptkit.ExecutionEvent) { requestEvents = append(requestEvents, e) }, OnPrepared: func(ctx context.Context, prepared promptkit.PreparedRun) {
|
||||
child = ctx
|
||||
details = append(details, prepared)
|
||||
}}, nil
|
||||
}})
|
||||
if err != nil || result.Outcome != promptkit.CoordinationStopped || time.Since(start) != 4*time.Second || len(details) != 2 || len(details[0].Messages) != 3 || len(details[1].Messages) != 5 {
|
||||
t.Fatalf("result=%+v error=%v details=%v elapsed=%v", result, err, len(details), time.Since(start))
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if len(requestEvents) != 6 || len(coordinatorEvents) != 6 || len(summary.Members) != 2 || summary.Attempts != 2 || summary.Usage.PromptTokens.Value != 4 || !summary.TerminalCoverageComplete {
|
||||
t.Fatalf("summary=%+v request=%d coordinator=%d", summary, len(requestEvents), len(coordinatorEvents))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCoordinateTimeoutLeavesParentAndSiblingUsable(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := newBackendCapacityEngine(t, observationClientFunc(func(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if req.Prompt.SessionID == "slow" {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "answer"}, nil
|
||||
}), 1, capacityInt(0), nil)
|
||||
parent := context.Background()
|
||||
initial, err := engine.Run(parent, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var details promptkit.PreparedRun
|
||||
calls := 0
|
||||
result, err := engine.Coordinate(parent, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
calls++
|
||||
return &promptkit.RunRequest{PromptID: "prompt", SessionID: "slow", OperationTimeout: time.Second, OnPrepared: func(_ context.Context, p promptkit.PreparedRun) { details = p }}, nil
|
||||
}})
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || calls != 1 || result.Outcome != "" || result.Undelivered != nil || details.ParentRunID != initial.RunID || parent.Err() != nil {
|
||||
t.Fatalf("result=%+v error=%v callbacks=%d", result, err, calls)
|
||||
}
|
||||
sibling, err := engine.Run(parent, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||
if err != nil || sibling.Exchange == nil {
|
||||
t.Fatalf("sibling=%v error=%v", sibling, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
170
coordinate_lifecycle_contract_test.go
Normal file
170
coordinate_lifecycle_contract_test.go
Normal file
@@ -0,0 +1,170 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestCoordinateReleasesCapacityBeforeCallback(t *testing.T) {
|
||||
client := &preparedRecordingClient{responses: []*promptkit.GenerateResponse{{Content: "initial"}, {Content: ""}, {Content: " "}, {Content: "validator"}}}
|
||||
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
calls++
|
||||
if turn.Result == nil {
|
||||
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}}, nil
|
||||
}
|
||||
if turn.Result.Validation.Status != promptkit.ValidationFailed || turn.Result.Validation.RepairAttempts != 1 || turn.RemainingTurns != 0 {
|
||||
t.Fatalf("delivered turn = %#v", turn)
|
||||
}
|
||||
_, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
return nil, err
|
||||
}})
|
||||
if err != nil || result.Outcome != promptkit.CoordinationStopped || calls != 2 || len(client.snapshot()) != 4 {
|
||||
t.Fatalf("outcome=%#v error=%v callbacks=%d calls=%d", result, err, calls, len(client.snapshot()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatePreservesCapacityAndProviderErrors(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
sourceEngine := newSimpleEngine(t, "source", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||
initial, err := sourceEngine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := make(chan struct{})
|
||||
client := newCapacityGateClient(release, 1)
|
||||
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||
done := make(chan capacityRunResult, 1)
|
||||
go runCapacityRequest(engine, ctx, promptkit.RunRequest{PromptID: "prompt"}, done)
|
||||
select {
|
||||
case <-client.started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
request := promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||
}}
|
||||
_, err = engine.Coordinate(ctx, request)
|
||||
var capacity *promptkit.CapacityError
|
||||
if !errors.As(err, &capacity) || capacity.BackendID != "limited" {
|
||||
t.Fatalf("capacity error = %v", err)
|
||||
}
|
||||
close(release)
|
||||
if finished := awaitCapacityRun(t, done); finished.err != nil {
|
||||
t.Fatal(finished.err)
|
||||
}
|
||||
providerEngine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, `{"error":{"message":"synthetic"}}`)
|
||||
_, err = providerEngine.Coordinate(ctx, request)
|
||||
var provider *promptkit.GenerationError
|
||||
if !errors.As(err, &provider) || provider.StatusCode() != http.StatusUnprocessableEntity {
|
||||
t.Fatalf("provider error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinateRetriesAndRepairsDeliverOnce(t *testing.T) {
|
||||
initialEngine := newSimpleEngine(t, "source", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||
initial, err := initialEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
responses := []*http.Response{observedHTTPResponse(503, `{}`), observedHTTPResponse(200, `{"choices":[{"message":{"content":"invalid"}}]}`), observedHTTPResponse(200, `{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)}
|
||||
calls := 0
|
||||
one := 1
|
||||
config := simpleConfig("follow-up")
|
||||
config.TransientRetries = &one
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
response := responses[calls]
|
||||
calls++
|
||||
return response, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callbacks := 0
|
||||
result, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
callbacks++
|
||||
if turn.Result == nil {
|
||||
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}}, nil
|
||||
}
|
||||
if !turn.Result.Validation.IsValid || turn.RemainingTurns != 0 {
|
||||
t.Fatalf("turn = %#v", turn)
|
||||
}
|
||||
return nil, nil
|
||||
}})
|
||||
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbacks != 2 || calls != 3 {
|
||||
t.Fatalf("result=%#v error=%v callbacks=%d attempts=%d", result, err, callbacks, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinateCancellationDelivery(t *testing.T) {
|
||||
for _, cancelInside := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "before delivery", true: "inside callback"}[cancelInside], func(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
calls := 0
|
||||
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
calls++
|
||||
if turn.Result != nil {
|
||||
cancel()
|
||||
return nil, nil
|
||||
}
|
||||
return &promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) {
|
||||
if !cancelInside && event.Kind == promptkit.ExecutionFinished {
|
||||
cancel()
|
||||
}
|
||||
}}, nil
|
||||
}})
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome != "" || (result.Undelivered != nil) == cancelInside {
|
||||
t.Fatalf("result=%#v error=%v callbacks=%d", result, err, calls)
|
||||
}
|
||||
if !cancelInside && (calls != 1 || result.Undelivered.Exchange == nil) {
|
||||
t.Fatalf("undelivered result=%#v calls=%d", result, calls)
|
||||
}
|
||||
if cancelInside && calls != 2 {
|
||||
t.Fatalf("callbacks=%d", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinateFailureLeavesSiblingUsable(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||
ctx := context.Background()
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentinel := errors.New("provider")
|
||||
client.err = sentinel
|
||||
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||
}})
|
||||
if !errors.Is(err, sentinel) || !errors.Is(err, promptkit.ErrLLMGenerate) || result.Outcome != "" || result.Undelivered != nil {
|
||||
t.Fatalf("failure=%#v, %v", result, err)
|
||||
}
|
||||
client.err = nil
|
||||
if sibling, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange}); err != nil || sibling.Exchange == nil {
|
||||
t.Fatalf("sibling=%#v, %v", sibling, err)
|
||||
}
|
||||
}
|
||||
39
doc.go
39
doc.go
@@ -1,29 +1,33 @@
|
||||
// Package promptkit provides an embeddable engine for preparing and executing
|
||||
// prompt-defined LLM workflows.
|
||||
//
|
||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||
// sources for prompts, profiles, and backends, and
|
||||
// Applications construct an [Engine] with [NewEngine] or register [Configure]
|
||||
// during their own ConfigKit load and pass its result to [NewFromConfig], then
|
||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
||||
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.InspectBackend],
|
||||
// [Engine.ListProfiles], [Engine.ListPrompts], [Engine.Prepare],
|
||||
// [Engine.PrepareExecution], [Engine.Run], [Engine.RunPrepared], or
|
||||
// [Engine.Coordinate]. Captured [Exchange] values support independent
|
||||
// continuations, and [ExecutionGroup] collects explicitly observed runs.
|
||||
// Concrete registries, repositories,
|
||||
// validators, and the built-in OpenAI-compatible client remain internal
|
||||
// implementation details.
|
||||
//
|
||||
// # Concurrency and ownership
|
||||
//
|
||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||
// InspectRequirements, InspectProfile, InspectBackend, ListProfiles, ListPrompts,
|
||||
// Prepare, PrepareExecution, Run, RunPrepared, and Coordinate calls. Engine-local backend policies
|
||||
// bound admitted Run and RunPrepared calls and model generations where
|
||||
// configured, while different backend pools and unlimited backends continue
|
||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||
// still receive concurrent calls and must be safe for that use.
|
||||
//
|
||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||
// An Engine captures selected configuration resources during construction.
|
||||
// Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||
// InspectRequirements, and InspectProfile return copied inspection values.
|
||||
// InspectRequirements, InspectProfile, and InspectBackend return copied
|
||||
// inspection values.
|
||||
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||
// Returned values and values passed to extension interfaces are likewise
|
||||
// isolated from engine state.
|
||||
@@ -42,7 +46,7 @@
|
||||
// that enforces their filesystem, authorization, and size policies.
|
||||
//
|
||||
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
||||
// and [ValidationResult.Errors] may contain sensitive data. Credential
|
||||
// [ValidationResult.Errors], and [ExchangeDetails.Messages] may contain sensitive data. Credential
|
||||
// exclusion and redaction do not sanitize those values. Applications and
|
||||
// injected collaborators are responsible for access control, retention,
|
||||
// logging, and secret handling appropriate to their data.
|
||||
@@ -56,17 +60,22 @@
|
||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||
// used by those values.
|
||||
//
|
||||
// Construction, inspection, handle, and error values, including [Config],
|
||||
// Resource inspection results support direct JSON and YAML encoding.
|
||||
// [PromptInspection], [ProfileInspection], [BackendInspection], and their nested
|
||||
// values define field names, nesting, and omission of absent optional fields.
|
||||
// Formatting belongs to the selected encoder.
|
||||
//
|
||||
// Other construction, inspection, handle, and error values, including [Config],
|
||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||
// [ProfileConfig], [RunRequest],
|
||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
||||
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
||||
// [ArtifactRef], [ExecutionTargetOverride],
|
||||
// [PromptIdentity], [PromptInputDefinition],
|
||||
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||
// [ExecutionEvent], do not have stable JSON
|
||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||
// every public value.
|
||||
|
||||
//
|
||||
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||
// contain sensitive request or schema fragments. Applications must apply their
|
||||
// own disclosure policy before logging, displaying, or returning them.
|
||||
|
||||
63
docs/adr/0002-shared-configuration-data-model.md
Normal file
63
docs/adr/0002-shared-configuration-data-model.md
Normal file
@@ -0,0 +1,63 @@
|
||||
# 0002: Share Configuration Data Types Across The Public Boundary
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Date
|
||||
|
||||
2026-11-21
|
||||
|
||||
## Context
|
||||
|
||||
PromptKit is a Go library that applications use to configure and run prompts.
|
||||
Applications use its public package; the code that loads settings and runs
|
||||
prompts lives in internal packages. This separation normally lets the internal
|
||||
code change without changing the application's interface.
|
||||
|
||||
The planned configuration API lets applications load settings once and pass them to
|
||||
PromptKit. Keeping separate public and internal definitions of those same
|
||||
settings would require duplicate declarations and code to copy each field back
|
||||
and forth. That adds maintenance work and opportunities to miss a field without
|
||||
changing what the settings mean.
|
||||
|
||||
## Decision
|
||||
|
||||
Define the public configuration settings and their nested data types once in a
|
||||
small, data-only internal package. Expose them through type aliases in the public
|
||||
`promptkit` package. An alias gives the same Go type a public name; consumers
|
||||
continue to use names such as `promptkit.Settings` without importing an internal
|
||||
package. The internal configuration code uses the same definitions directly.
|
||||
|
||||
This is a narrow exception to the rule against exposing internal types. These
|
||||
settings are intentional public contracts, including their fields and
|
||||
configuration tags. Their location does not make them private implementation
|
||||
details that can be changed independently of the public API.
|
||||
|
||||
The rest of the public boundary stays in place. Loading, validation, and
|
||||
execution logic remain internal. Source-building helpers, execution requests,
|
||||
runtime objects, and inspection reports keep separate representations where
|
||||
they serve different purposes. Internal packages still do not import the public
|
||||
package. Sharing declarations does not remove the copies needed to keep a
|
||||
running engine independent of later application changes.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- Keep separate public and internal settings. This allows independent changes,
|
||||
but provides little benefit when both describe the same configuration and
|
||||
must stay synchronized.
|
||||
- Put the shared types in a new public settings package. This also removes
|
||||
duplication, but adds a public package when the existing `promptkit` package
|
||||
already provides a suitable consumer interface.
|
||||
|
||||
## Consequences
|
||||
|
||||
Configuration fields have one declaration, and registration and resource
|
||||
capture no longer need conversions between equivalent settings types. The
|
||||
primary benefit is simpler maintenance; performance improvements are secondary.
|
||||
Conversion into execution-ready objects and necessary ownership copies remain.
|
||||
|
||||
This decision leaves the [ConfigKit ownership boundary](0001-configuration-boundary.md)
|
||||
unchanged. The [shared-loading guide](../consumers/pkg-promptkit.md#reuse-an-application-owned-load)
|
||||
describes the implemented consumer workflow; the
|
||||
[internal overview](../internal/overview.md) describes its implementation.
|
||||
@@ -1,15 +1,43 @@
|
||||
# Configuration And API Migration
|
||||
|
||||
This breaking pre-v1 minor migration adopts Promptkit's vanity module path,
|
||||
changes the v0.16-era keyed resources to identified lists, adopts ConfigKit v1,
|
||||
and removes prompt-version labels.
|
||||
Promptkit does not install a provider catalog; applications choose configuration
|
||||
This guide covers the current configuration and repair-input migration, as well
|
||||
as earlier module-path and keyed-resource migrations.
|
||||
Promptkit installs a default-on bundled OpenRouter catalog; applications choose configuration
|
||||
inputs explicitly, and execution selects one prompt by its exact ID. This guide
|
||||
explains the shape changes and decisions an application must make; it is not an
|
||||
automatic migration tool. The [framework format reference](../formats.md) owns
|
||||
the current configuration contract, and the root package GoDoc owns exact API
|
||||
signatures and errors.
|
||||
|
||||
## Move Settings Under Promptkit
|
||||
|
||||
Place complete-document resources under `promptkit`, including `schema_root`,
|
||||
`backends`, `profiles`, `prompts`, and `collections`. Old root-level resource
|
||||
keys are application data and are not loaded as Promptkit resources; selecting
|
||||
an unmigrated resource can therefore fail with a not-found error. IDs and
|
||||
references do not acquire a `promptkit` prefix. `SettingsInput` mounts a
|
||||
wrapperless fragment at `/promptkit`, and `CollectionInput` mounts one at
|
||||
`/promptkit/collections/<namespace>`. Discovery helpers still return root-mounted
|
||||
inputs; set destinations explicitly for discovered fragments. Standalone
|
||||
id-first files use native sequence-item placement. The [format reference](../formats.md)
|
||||
describes extensions and strict nested objects.
|
||||
|
||||
Applications can instead embed `promptkit.Settings`, call `Configure` on their
|
||||
ConfigKit definition, load once, and pass the whole result to `NewFromConfig`.
|
||||
The [preparation example](../../examples/go-library/prepare/main.go) shows the
|
||||
workflow. Select `BundledOpenRouterUnderlay` before an application-owned load;
|
||||
the default bundle option applies only to `NewEngine`. The application owns
|
||||
version policy; Promptkit no longer validates resource-version fields.
|
||||
|
||||
Replace `RunRequest.Validation` values of type `*OutputContract` with
|
||||
`*OutputContractInput` and use a pointer for `RepairAttempts`. Nil or null
|
||||
selects zero for `none` and one for `basic`, `json`, and `json_schema`;
|
||||
explicit zero disables correction. Runtime replacements resolve independently
|
||||
of configured budgets. Inspection reports resolved integer budgets, including
|
||||
zero. A failed validation can cause an additional generation and associated
|
||||
latency and usage; a correction is not guaranteed. See the
|
||||
[output contract](../formats.md#output-contract).
|
||||
|
||||
## Update Go Imports
|
||||
|
||||
Promptkit's module and public package import path is now
|
||||
@@ -37,8 +65,8 @@ The configuration and API adaptations below are also part of this migration.
|
||||
## Select And Compose The Configuration You Need
|
||||
|
||||
Supply each backend, profile, prompt, schema, and collection source that the
|
||||
application intends to use. Promptkit does not discover directories or install
|
||||
provider definitions implicitly. An application may select an explicit
|
||||
application intends to use. `NewEngine` selects the bundled OpenRouter underlay
|
||||
by default; callers explicitly select other sources. An application may select an explicit
|
||||
collection input with `CollectionInput`:
|
||||
|
||||
```go
|
||||
@@ -64,11 +92,10 @@ Promptkit does not rewrite references when a source is mounted. `extends`
|
||||
selects a sibling ID in the same list, not a cross-collection reference.
|
||||
Review collisions and all references when moving resources into collections.
|
||||
|
||||
The optional source-root `version` header now accepts `2` or `"2"`; headerless
|
||||
sources remain valid. A collection's `release_version` remains an optional
|
||||
label. Prompt-version labels have been removed: choose one definition per ID
|
||||
and store any application-specific release label outside Promptkit. In a
|
||||
standalone imported prompt item, root `version` is a source header, not a label.
|
||||
A collection's `release_version` remains an optional label. Prompt-version
|
||||
selection has been removed: choose one definition per ID. Application-specific
|
||||
version data may be retained at supported extension points, but is not used by
|
||||
Promptkit for resource selection.
|
||||
|
||||
For example, change a version-keyed definition:
|
||||
|
||||
@@ -84,7 +111,7 @@ prompts:
|
||||
validation_mode: none
|
||||
```
|
||||
|
||||
to one identified definition:
|
||||
to one identified wrapperless fragment, mounted with `SettingsInput`:
|
||||
|
||||
```yaml
|
||||
prompts:
|
||||
@@ -98,7 +125,8 @@ prompts:
|
||||
```
|
||||
|
||||
The same conversion applies to backends and profiles. Sibling inheritance uses
|
||||
identity references, and a later same-ID record replaces the whole lower record:
|
||||
identity references, and a later same-ID record replaces the whole lower record.
|
||||
This is another wrapperless fragment to mount with `SettingsInput`:
|
||||
|
||||
```yaml
|
||||
profiles:
|
||||
@@ -113,11 +141,11 @@ profiles:
|
||||
To customize a lower resource's fields, add a distinct inherited variant as
|
||||
above; when replacing the same ID, supply the complete intended record. Generic
|
||||
append, replace, imports, and inheritance are described by the
|
||||
[ConfigKit v1 configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md).
|
||||
[ConfigKit v1 configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md).
|
||||
Source-root controls are now `config:`, not `configkit:`. Imports are optional
|
||||
unless the selected import declares `required: true`.
|
||||
|
||||
Update ConfigKit imports to `go.maximumdirect.net/configkit` at v1.0.0: its Go
|
||||
Update ConfigKit imports to `go.maximumdirect.net/configkit` at v1.4.0: its Go
|
||||
types are incompatible with the old module path. Promptkit now requires Go
|
||||
1.27.0 or newer. Source builders take ordered slices with explicit IDs:
|
||||
|
||||
@@ -133,14 +161,18 @@ source := promptkit.ProfileSource("application", []promptkit.ProfileConfig{
|
||||
Replace former maps passed to `BackendSource` in the same way, setting each
|
||||
`BackendConfig.ID`. Duplicates are retained for loading to reject. Resource
|
||||
inspection paths now address list indexes, for example
|
||||
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
||||
and diffs expose composed values and provenance, while `InspectProfile` and
|
||||
other interpreted views expose Promptkit normalization and defaults; the views
|
||||
are not promised to be identical.
|
||||
`configkit.Keys("promptkit", "profiles").Index(0).Key("model")`. Configuration inspection
|
||||
and diffs expose resolved values, recorded normalization and provenance.
|
||||
`InspectProfile` reports the selected resolved profile without backend completion.
|
||||
Use `Prepare` or `PreparedExecution.Details` for completed request settings,
|
||||
including backend defaults and request overrides; see the
|
||||
[resource inspection guidance](pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||
|
||||
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
||||
`prompt_version` JSON members. Prompt hashes are opaque equality values and may
|
||||
change when the removed field no longer participates in hashing.
|
||||
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
||||
values and may also change because redundant captured fields were removed.
|
||||
Invalidate persisted caches or comparisons that span this change; hashes from
|
||||
inspection and preparation of the same captured definition still agree.
|
||||
|
||||
## Identity Mapping
|
||||
|
||||
@@ -161,7 +193,8 @@ These are conversion examples, not aliases or bundled resources. Mounting old
|
||||
defaults, or diagnostic defaults inside them. Edit those values to the exact
|
||||
qualified IDs you chose.
|
||||
|
||||
For example, an optional caller-owned standard profile may extend a selected
|
||||
For example, the following wrapperless fragments can be mounted with
|
||||
`SettingsInput`. An optional caller-owned standard profile may extend a selected
|
||||
Gemini profile in the same `openrouter` collection:
|
||||
|
||||
```yaml
|
||||
|
||||
@@ -18,7 +18,7 @@ import "go.maximumdirect.net/promptkit"
|
||||
For migration from the former module path, see
|
||||
[Update Go Imports](configuration-migration.md#update-go-imports).
|
||||
|
||||
Configuration APIs use `go.maximumdirect.net/configkit` v1.0.0. Its types are
|
||||
Configuration APIs use `go.maximumdirect.net/configkit` v1.4.0. Its types are
|
||||
not interchangeable with the old ConfigKit module path.
|
||||
|
||||
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
||||
@@ -52,30 +52,123 @@ Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
||||
explicitly and place its inputs in an ordered ConfigKit underlay. A group
|
||||
strictly composes its files; later groups can overlay earlier ones:
|
||||
|
||||
The bundled OpenRouter backend and profiles are installed in a lower underlay
|
||||
by default. Select a profile explicitly, for example:
|
||||
|
||||
```go
|
||||
input := promptkit.SettingsInput(configkit.Mapping("application", map[string]any{
|
||||
"prompts": []any{map[string]any{
|
||||
"id": "my-prompt",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||
}},
|
||||
}))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||
if err != nil { return err }
|
||||
result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "my-prompt", ProfileID: "gpt-luna-latest"})
|
||||
```
|
||||
|
||||
The backend looks up
|
||||
`OPENROUTER_API_KEY` when available; credentials are not required at engine
|
||||
construction. For the complete inventory and diagnostic defaults see the
|
||||
[format reference](../formats.md#bundled-openrouter-configuration). Use
|
||||
`promptkit.WithBundledOpenRouter(false)` for application-only inventories.
|
||||
Caller records replace matching bundled records in full; use `extends` with a
|
||||
distinct ID to inherit. Replacing `auto` also changes its cost-tier children's
|
||||
inherited target, while their plugins remain. External collections remain
|
||||
supported.
|
||||
|
||||
```go
|
||||
promptInputs, err := promptkit.DirectoryInputs("prompts")
|
||||
if err != nil { return err }
|
||||
profileInputs, err := promptkit.DirectoryInputs("profiles")
|
||||
if err != nil { return err }
|
||||
// These selected files are wrapperless Promptkit fragments.
|
||||
for i := range promptInputs { promptInputs[i].Destination = configkit.Keys("promptkit") }
|
||||
for i := range profileInputs { profileInputs[i].Destination = configkit.Keys("promptkit") }
|
||||
filesystem := configkit.NewFilesystem("application schemas", os.DirFS("."))
|
||||
schemaInput := configkit.Input{
|
||||
Source: promptkit.SchemaRootSource("application schemas", "schemas"),
|
||||
Destination: configkit.Keys("promptkit"),
|
||||
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."},
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||
{Inputs: promptInputs},
|
||||
{Inputs: profileInputs},
|
||||
{Inputs: []configkit.Input{schemaInput}},
|
||||
}},
|
||||
SchemaDir: "schemas",
|
||||
})
|
||||
```
|
||||
|
||||
For a single highest-priority document, set `Sources.Main` to a
|
||||
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
||||
can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
|
||||
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
||||
collaborator options remain available. Construction validates the final
|
||||
can use `configkit.File` directly. `PromptSource`, `ProfileSource`, `BackendSource`, and raw
|
||||
`configkit.Mapping` supply programmatic definitions. Runtime collaborator
|
||||
options remain available. Construction validates the final
|
||||
effective configuration and captures referenced prompt content; construct a
|
||||
new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go)
|
||||
for the exact public contract and the [framework format reference](../formats.md)
|
||||
for definition fields.
|
||||
|
||||
### Reuse An Application-Owned Load
|
||||
|
||||
An application that already loads ConfigKit configuration can embed
|
||||
`promptkit.Settings` at `yaml:"promptkit"`, compose its definition with
|
||||
`promptkit.Configure`, and pass the successful whole result to
|
||||
`promptkit.NewFromConfig`. The [preparation example](../../examples/go-library/prepare/main.go)
|
||||
shows a complete single-load workflow. Existing application resolve and
|
||||
validate callbacks run before Promptkit's, and its defaults, version rule,
|
||||
sequence registrations, and sensitivity predicate are preserved. Callers can
|
||||
wrap the returned callbacks for work that must run afterward.
|
||||
|
||||
`NewFromConfig` accepts a completed result or a wrapper exposing the three
|
||||
methods in [`Configuration`](../../engine.go). The selected `/promptkit`
|
||||
value can be `Settings`, `*Settings`, absent, or nil; an incompatible value,
|
||||
nil result, or unavailable inspection view returns `ErrInvalidConfig`. It
|
||||
does not load sources, rerun callbacks, or add a bundle. Select
|
||||
`BundledOpenRouterUnderlay()` before the application load if desired;
|
||||
`WithBundledOpenRouter` affects only `NewEngine`. `RuntimeConfig` supplies only
|
||||
client timeout, HTTP client, and transient-retry settings. Keep resolved settings
|
||||
consistent with the load's frozen inspection evidence at handoff; subsequent mutation does not change captured
|
||||
resources. Applications may register equivalent callbacks explicitly instead
|
||||
of using `Configure`. Construction errors retain their underlying causes;
|
||||
on the standalone path, a ConfigKit `LoadError` remains available through
|
||||
`errors.As`.
|
||||
|
||||
### Load Companion Prompts Relative To The Main Configuration
|
||||
|
||||
A non-file underlay can use the selected main configuration's directory for
|
||||
caller-declared imports. For example, this loads standalone prompt definitions
|
||||
from the adjacent `prompts/` directory into the `support` collection:
|
||||
|
||||
```go
|
||||
main := configkit.Input{Source: configkit.File(configPath)}
|
||||
companion, err := promptkit.CollectionInput("support", configkit.Bytes("companion prompts", []byte(`
|
||||
config:
|
||||
imports:
|
||||
- dir: prompts
|
||||
pattern: ["*.yaml", "*.yml"]
|
||||
destination: [prompts]
|
||||
placement: sequence-item
|
||||
required: true
|
||||
`)))
|
||||
if err != nil { return err }
|
||||
companion.MainRelative = configkit.MainRelative()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Main: &main,
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{companion}}},
|
||||
}})
|
||||
```
|
||||
|
||||
`CollectionInput` enables imports; for a raw `configkit.Input`, enable them with
|
||||
`Imports: true`. The binding itself does not enable imports, and an explicit
|
||||
`Input.Filesystem` takes precedence. Imported prompt `content_file` references
|
||||
keep their source-local context. See ConfigKit's
|
||||
[layered-loading contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md#layered-loading)
|
||||
for eligible inputs, fallback contexts, and behavior when the main has no usable
|
||||
filesystem context.
|
||||
|
||||
### Use Scoped Engine Views
|
||||
|
||||
Create a view when an application wants short request IDs to resolve within a
|
||||
@@ -115,6 +208,8 @@ defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS,
|
||||
if err != nil { return err }
|
||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||
if err != nil { return err }
|
||||
for i := range defaults { defaults[i].Destination = configkit.Keys("promptkit") }
|
||||
for i := range operator { operator[i].Destination = configkit.Keys("promptkit") }
|
||||
profile := promptkit.ProfileSource("application choices", []promptkit.ProfileConfig{
|
||||
{ID: "summary", Endpoint: &selectedEndpoint, Model: &selectedModel},
|
||||
})
|
||||
@@ -122,7 +217,7 @@ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.Configura
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: defaults},
|
||||
{Inputs: operator},
|
||||
{Inputs: []configkit.Input{{Source: profile}}},
|
||||
{Inputs: []configkit.Input{promptkit.SettingsInput(profile)}},
|
||||
},
|
||||
}})
|
||||
```
|
||||
@@ -147,12 +242,23 @@ document := configkit.Mapping("application configuration", map[string]any{
|
||||
},
|
||||
},
|
||||
})
|
||||
input := configkit.Input{Source: document}
|
||||
input := promptkit.SettingsInput(document)
|
||||
// Add input to a later underlay, or set Sources.Main = &input.
|
||||
```
|
||||
|
||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
||||
is not needed. Backend and profile definitions are supplied by the caller;
|
||||
`PromptSource`, `ProfileSource`, and `BackendSource` are typed conveniences when
|
||||
null is not needed. Prompt and backend helpers, like profiles, support sibling
|
||||
inheritance via `Extends`; partial records can be completed by other layers.
|
||||
For relative prompt `content_file` references, supply the ordinary ConfigKit
|
||||
`Input.Filesystem` context for that source, or use a
|
||||
[main-relative underlay](#load-companion-prompts-relative-to-the-main-configuration).
|
||||
The helpers do not read files or
|
||||
validate final definitions. Profile and backend provider maps use ConfigKit Mapping input forms: built-in
|
||||
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
|
||||
typed containers explicitly to an appropriate supported value, choosing numeric
|
||||
precision for your application. Mapping captures supported values immediately;
|
||||
unsupported or cyclic values are reported when loaded. Runtime override values
|
||||
follow their separate rules. Backend and profile definitions are supplied by the caller;
|
||||
there is no registration-only backend namespace.
|
||||
|
||||
When configuration or message files change, build and validate a new engine,
|
||||
@@ -211,7 +317,7 @@ sensitive application data.
|
||||
## Inspect The Captured Configuration
|
||||
|
||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||
operator-selected report of composed configuration, source descriptions,
|
||||
operator-selected report of resolved configuration, source descriptions,
|
||||
diagnostics, and path history. Configuration inspection uses list indexes in
|
||||
the collection's nested path rather than qualified runtime IDs. For a profile
|
||||
at index `i` in collection `team`, construct the path with `configkit.Keys`:
|
||||
@@ -219,7 +325,7 @@ at index `i` in collection `team`, construct the path with `configkit.Keys`:
|
||||
```go
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil { return err }
|
||||
path := configkit.Keys("collections", "team", "profiles").Index(i).Key("model")
|
||||
path := configkit.Keys("promptkit", "collections", "team", "profiles").Index(i).Key("model")
|
||||
explanation, found := view.Explain(path)
|
||||
if found && explanation.Effective.Redacted {
|
||||
// Apply the application's disclosure policy before sharing the report.
|
||||
@@ -230,11 +336,17 @@ if found && explanation.Effective.Redacted {
|
||||
the application decides whether those reports may be disclosed. Credential-
|
||||
shaped provider keys are redacted, but arbitrary prompt and provider content
|
||||
is not necessarily safe to share. Inspection uses the engine's captured result
|
||||
and does not reopen sources or resolve credentials. It retains composed values
|
||||
and provenance rather than Promptkit's normalized/defaulted interpretation.
|
||||
`InspectProfile` instead shows backend-completed execution settings for one profile, and `Prepare` or
|
||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||
and validation resources. None of those views is a substitute for the others.
|
||||
and does not reopen sources or resolve credentials. It retains effective values,
|
||||
including recorded field normalization and queue defaulting, alongside source
|
||||
provenance. The view focuses effective output and `HistoricalPaths` on
|
||||
`/promptkit`; it is not an isolation boundary. Metadata and source catalogs
|
||||
remain whole-load evidence, and explicit `Explain` queries may address paths
|
||||
outside the focus. The empty path explains the whole application. Diagnostics
|
||||
come from the original load and are returned as detached records.
|
||||
`InspectPrompt`, `InspectProfile`, and `InspectBackend` instead show individual
|
||||
resolved resources without backend-to-profile execution completion or provenance.
|
||||
`Prepare` or `PreparedExecution.Details` shows completed request-specific settings,
|
||||
rendered messages, inputs, and validation resources. None substitutes for the others.
|
||||
|
||||
To review changes between two independently captured configurations, compare
|
||||
their inspection views:
|
||||
@@ -263,11 +375,15 @@ for _, difference := range differences {
|
||||
```
|
||||
|
||||
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||
and ConfigKit's [v1.0.0 effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md#effective-configuration-diffs)
|
||||
and ConfigKit's [effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md#effective-configuration-diffs)
|
||||
for the exact comparison contract. The operation compares only frozen effective
|
||||
configuration values: it does not reread external files or compare their
|
||||
configuration values, including native ancestor differences; sibling-only
|
||||
records are excluded, but an ancestor record may contain sibling changes. It
|
||||
does not reread external files or compare their
|
||||
contents, resolved credential contents, later execution preparation, or
|
||||
request-specific overrides. An empty diff is not proof of equivalent execution.
|
||||
request-specific overrides. Inputs normalizing to the same effective value do
|
||||
not create a value diff; lists are compared atomically. An empty diff is not
|
||||
proof of equivalent execution.
|
||||
Records can contain ordinary application data, so use the application's
|
||||
disclosure policy when presenting them.
|
||||
|
||||
@@ -346,8 +462,10 @@ if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, input := range inspection.Inputs {
|
||||
// Compare the declared input with application configuration.
|
||||
if inspection.Inputs != nil {
|
||||
for _, input := range *inspection.Inputs {
|
||||
// Compare the declared input with application configuration.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -483,12 +601,17 @@ if err != nil {
|
||||
defer preparedExecution.Discard()
|
||||
|
||||
details := preparedExecution.Details()
|
||||
messageContentHash := details.MessageContentHash
|
||||
_ = messageContentHash // An opaque value available before model execution.
|
||||
// Inspect or persist an application-selected safe subset of details.
|
||||
|
||||
result, err := engine.RunPrepared(ctx, preparedExecution)
|
||||
```
|
||||
|
||||
Preparation does not call the model or reserve backend capacity.
|
||||
`MessageContentHash` compares the ordered message roles and text, whereas
|
||||
`RenderedPromptHash` also includes the session and cache-control hints. See
|
||||
the [`PreparedRun` GoDoc](../../types.go) for their contracts and compatibility.
|
||||
`RunPrepared` executes from the retained snapshot rather than reloading
|
||||
consumer sources. The handle is opaque in-process state, while `Details`
|
||||
contains rendered content and remains subject to the application's data
|
||||
@@ -505,6 +628,59 @@ retained callback without invoking it. The function's closure is retained by
|
||||
identity, so synchronize any shared state it uses. Prepared start events can
|
||||
include identities already resolved in the frozen snapshot.
|
||||
|
||||
## Bound Execution And Retain Prepared Details
|
||||
|
||||
Select the optional controls on [RunRequest](../../types.go). The operation
|
||||
budget is separate from the provider timeout and cancellation is cooperative.
|
||||
Notification is conditional on reaching the prepared point, synchronous, and
|
||||
runs while admission is held; its caller-owned details can be retained even if
|
||||
generation later fails. Do not rely on notification for preparation or admission
|
||||
failures. The application owns any subsequent diagnostic handling.
|
||||
|
||||
```go
|
||||
var details *promptkit.PreparedRun
|
||||
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||
PromptID: "answer", OperationTimeout: 30 * time.Second,
|
||||
OnPrepared: func(_ context.Context, prepared promptkit.PreparedRun) { details = &prepared },
|
||||
})
|
||||
if errors.Is(err, promptkit.ErrOperationTimeout) && errors.Is(err, context.DeadlineExceeded) {
|
||||
// details is available only if the notification was reached.
|
||||
}
|
||||
```
|
||||
|
||||
For coordinated turns, put the controls on each returned request. Assessment
|
||||
in the next-turn callback runs after the previous admission and child scope
|
||||
are released, so it can select its own budget:
|
||||
|
||||
```go
|
||||
next := func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result != nil {
|
||||
_, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "assessment", OperationTimeout: 10 * time.Second})
|
||||
return nil, err
|
||||
}
|
||||
return &promptkit.RunRequest{PromptID: "answer", OperationTimeout: 30 * time.Second,
|
||||
OnPrepared: func(_ context.Context, prepared promptkit.PreparedRun) { details = &prepared }}, nil
|
||||
}
|
||||
```
|
||||
|
||||
For split execution, [PrepareExecution](../../engine.go) retains the controls
|
||||
without starting their timer; idle time does not count. Use the same caller
|
||||
deadline for both calls if preparation and execution need one shared bound.
|
||||
|
||||
```go
|
||||
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{
|
||||
PromptID: "answer", OperationTimeout: 30 * time.Second,
|
||||
OnPrepared: func(_ context.Context, prepared promptkit.PreparedRun) { details = &prepared },
|
||||
})
|
||||
if err != nil { return err }
|
||||
defer handle.Discard()
|
||||
result, err := engine.RunPrepared(ctx, handle)
|
||||
```
|
||||
|
||||
See the [observation example](../../examples/go-library/observe/main.go) for an
|
||||
offline coordinated workflow and [ErrOperationTimeout](../../engine.go) for
|
||||
timeout attribution.
|
||||
|
||||
## Execute And Validate
|
||||
|
||||
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||
@@ -568,22 +744,26 @@ Set a small additional-call budget when a structurally invalid result can be
|
||||
corrected automatically:
|
||||
|
||||
```go
|
||||
request.Validation = &promptkit.OutputContract{
|
||||
request.Validation = &promptkit.OutputContractInput{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSONSchema,
|
||||
SchemaPath: "events.schema.json",
|
||||
RepairAttempts: 1,
|
||||
Schema: map[string]any{"type": "object"},
|
||||
RepairAttempts: new(1),
|
||||
}
|
||||
```
|
||||
|
||||
Each repair attempt is another model generation, so it can increase latency and
|
||||
Omission or null selects one repair for active validation and zero for `none`;
|
||||
an explicit zero selects single-pass validation. A replacement resolves its
|
||||
own budget and does not inherit the configured one. Each repair attempt is
|
||||
another model generation, so it can increase latency and
|
||||
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
|
||||
repair generations rather than transient HTTP attempts. Exhaustion still
|
||||
returns the final failed validation result. `basic` validation can also repair
|
||||
an empty candidate, but structural
|
||||
validity is not evidence of factual or domain correctness. See the
|
||||
validity is not evidence of factual or domain correctness and a correction is
|
||||
not guaranteed. See the
|
||||
[output-contract format reference](../formats.md#output-contract) and
|
||||
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
|
||||
[`OutputContractInput` GoDoc](../../types.go) for the exact budget and eligibility
|
||||
rules.
|
||||
|
||||
The built-in client also retries eligible temporary HTTP statuses. By default,
|
||||
@@ -619,6 +799,116 @@ application owns any semantic retry budget. When a
|
||||
pre-execution equality check is required, use `PrepareExecution` and compare
|
||||
its opaque rendered-prompt hash before invoking `RunPrepared`.
|
||||
|
||||
### Capture And Continue A Conversation
|
||||
|
||||
The [request GoDoc](../../types.go), [exchange GoDoc](../../exchange.go), and
|
||||
[prepared-execution GoDoc](../../prepared_execution.go) own the exact capture,
|
||||
continuation, inspection, and lifetime contracts.
|
||||
|
||||
Capture records the messages actually executed and the final assistant content.
|
||||
An exchange is an immutable in-process capability; `Details()` returns detached,
|
||||
encodable history that may contain sensitive content. Ordinary result encoding
|
||||
does not include the exchange. Continuations imply capture and may branch or use
|
||||
a different engine, profile, output contract, credential, or session:
|
||||
|
||||
```go
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "proposal", CaptureExchange: true})
|
||||
if err != nil { return err }
|
||||
left, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "refine", ContinueFrom: initial.Exchange})
|
||||
if err != nil { return err }
|
||||
right, err := anotherEngine.Run(ctx, promptkit.RunRequest{PromptID: "alternative", ContinueFrom: initial.Exchange})
|
||||
if err != nil { return err }
|
||||
_, _ = left, right // Independent branches; each result has its own exchange.
|
||||
```
|
||||
|
||||
For a frozen execution, set `CaptureExchange` in the request passed to
|
||||
`PrepareExecution`, then obtain the exchange from `RunPrepared`'s result.
|
||||
Preparation and validation alone do not produce an exchange. Captured history
|
||||
precedes newly rendered definition messages and appended messages. A completed
|
||||
structurally invalid result still has an exchange; an operational error returns
|
||||
no result. The caller chooses what constitutes acceptance.
|
||||
|
||||
```go
|
||||
prepared, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "proposal", CaptureExchange: true})
|
||||
if err != nil { return err }
|
||||
result, err := engine.RunPrepared(ctx, prepared)
|
||||
if err != nil { return err }
|
||||
details := result.Exchange.Details()
|
||||
_ = details // Explicitly encode or inspect when appropriate.
|
||||
```
|
||||
|
||||
### Coordinate Application-Directed Turns
|
||||
|
||||
The following sketches use illustrative application functions. The first records
|
||||
each newly delivered response before selecting another question:
|
||||
|
||||
```go
|
||||
outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||
Start: initial.Exchange, AdditionalTurns: 3,
|
||||
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result != nil {
|
||||
if !turn.Result.Validation.IsValid { return nil, fmt.Errorf("invalid response") }
|
||||
analysis.Record(turn.Result)
|
||||
}
|
||||
question, ok := analysis.NextQuestion()
|
||||
if !ok { return nil, nil }
|
||||
return &promptkit.RunRequest{PromptID: "research/investigate", Vars: map[string]string{"question": question}}, nil
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
An application can instead start with existing findings, assess each new
|
||||
candidate, and keep its decisions in a closure:
|
||||
|
||||
```go
|
||||
outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||
Start: proposal.Exchange, AdditionalTurns: 3, Observer: branchGroup.Observe,
|
||||
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result != nil {
|
||||
if !turn.Result.Validation.IsValid { return nil, fmt.Errorf("invalid revision") }
|
||||
candidate, withdrawn, err := decodeRevision(turn.Result.RawOutput)
|
||||
if err != nil { return nil, err }
|
||||
if withdrawn { branch.MarkWithdrawn(); return nil, nil }
|
||||
findings, err := assess(ctx, engine, candidate, validatorGroup.Observe)
|
||||
if err != nil { return nil, err }
|
||||
branch.Update(candidate, findings)
|
||||
if branch.Done() { return nil, nil }
|
||||
}
|
||||
return &promptkit.RunRequest{PromptID: branch.FollowUpPrompt(), Vars: branch.Variables()}, nil
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
`assess` is application code and explicitly selects its observer on any nested
|
||||
run. A completed turn releases its admission lease before the callback, so that
|
||||
assessment may use the same backend. `stopped` and `exhausted` describe control
|
||||
flow, not content acceptance. Cancellation before callback delivery can return one
|
||||
completed response in `Undelivered`; errors retain their ordinary identities.
|
||||
|
||||
### Collect Selected Execution Facts
|
||||
|
||||
Groups are independent of continuation and enrollment is explicit:
|
||||
|
||||
```go
|
||||
var group, validatorGroup promptkit.ExecutionGroup
|
||||
request.Observer = promptkit.ComposeObservers(diagnosticObserver, group.Observe)
|
||||
prepared, err := engine.PrepareExecution(ctx, request) // no event yet
|
||||
if err != nil { return err }
|
||||
result, err := engine.RunPrepared(ctx, prepared)
|
||||
if err != nil { return err }
|
||||
_ = result
|
||||
// Set CoordinateRequest.Observer to group.Observe for its continuation runs.
|
||||
// A nested validator selects validatorGroup.Observe on its own request.
|
||||
summary, validatorSummary := group.Snapshot(), validatorGroup.Snapshot()
|
||||
_, _ = summary, validatorSummary
|
||||
```
|
||||
|
||||
Only the first terminal event per run contributes counts and usage; coverage of
|
||||
currently observed members is separate from token-usage completeness. Empty
|
||||
groups report complete, known zero totals, while members without terminals do
|
||||
not invent totals. Overlapping groups are not generally additive. See the
|
||||
[offline example](../../examples/go-library/observe/main.go) for complete setup.
|
||||
|
||||
## Inputs, Profiles, And Overrides
|
||||
|
||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||
@@ -662,9 +952,9 @@ the engine to pick up changed definitions or message files.
|
||||
### Choose A Profile Ownership Model
|
||||
|
||||
Supply backend and profile definitions through the configuration sources your
|
||||
application selects. PromptKit does not install provider catalogs or built-in
|
||||
profile inventories; conventional backend ID constants are names only. Define
|
||||
the model, endpoint, credentials lookup, and settings that your application
|
||||
application selects, alongside the default-on bundled OpenRouter inventory.
|
||||
Conventional backend ID constants are names only. Define or replace the model,
|
||||
endpoint, credentials lookup, and settings that your application
|
||||
intends to support. A profile may inherit a sibling profile by its local key in
|
||||
the same root or collection list; its backend reference uses the exact
|
||||
configured identity. Caller-selected source layers determine the final values. The
|
||||
@@ -673,10 +963,9 @@ composition rules.
|
||||
|
||||
### Inspect A Profile Before Prompt Work
|
||||
|
||||
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
||||
profile without constructing a synthetic prompt or placeholder inputs. It
|
||||
resolves the profile's effective target but does not prepare or execute a
|
||||
prompt:
|
||||
Use [`Engine.InspectProfile`](../../engine.go) to inspect a resolved profile
|
||||
without constructing a synthetic prompt or placeholder inputs. It reports the
|
||||
profile's own fields, not a backend-completed target:
|
||||
|
||||
```go
|
||||
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||
@@ -684,23 +973,48 @@ if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
target := inspection.EffectiveModelParams
|
||||
if target.APIKeyEnv != "" {
|
||||
// This is a configured optional environment lookup source.
|
||||
} else if inspection.APIKeyRequired {
|
||||
// Arrange a direct credential before later execution.
|
||||
if inspection.APIKeyRequired != nil && *inspection.APIKeyRequired {
|
||||
// Arrange a usable direct or configured environment credential.
|
||||
}
|
||||
if inspection.APIKeyEnv != nil {
|
||||
// This is the configured environment lookup source, not its value.
|
||||
}
|
||||
```
|
||||
|
||||
Use this configuration-time boundary when only the profile and its target need
|
||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||
or rendering work; use prepared execution when that work must remain tied to a
|
||||
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
||||
`APIKeyRequired` is the explicit local requirement. The
|
||||
Use [`InspectBackend`](../../engine.go) to inspect the referenced backend
|
||||
separately. Its diagnostics are configured metadata, not live checks or derived
|
||||
destinations; `ErrBackendNotFound` identifies an absent backend. References in
|
||||
all three resource reports remain as authored. Use `Prepare` for completed
|
||||
request settings and rendering, or prepared execution when that work must remain
|
||||
tied to later execution. A reported `APIKeyEnv` identifies a configured source,
|
||||
not its value; `APIKeyRequired` can also be present. The
|
||||
[credential format reference](../formats.md#credentials) and the method's
|
||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||
contracts.
|
||||
|
||||
Resource inspection pointers represent optional fields: nil is absent, while a
|
||||
non-nil pointer preserves zero, false, or an empty string. Pointer-to-collection
|
||||
fields distinguish omission from an empty map or list. Results and nested mutable
|
||||
values belong to the caller. Ordinary generic encoders need no per-field adapter:
|
||||
|
||||
```go
|
||||
jsonBytes, err := json.Marshal(inspection)
|
||||
if err != nil { return err }
|
||||
yamlBytes, err := yaml.Marshal(inspection) // go.yaml.in/yaml/v4
|
||||
if err != nil { return err }
|
||||
_, _ = jsonBytes, yamlBytes
|
||||
```
|
||||
|
||||
For migration, replace `ProfileInspection.EffectiveModelParams` accesses with
|
||||
the optional resource fields, and use `Prepare` or `PreparedExecution.Details`
|
||||
where backend completion and request overrides are needed. Prompt inspection's
|
||||
`DefaultProfileID`, `Inputs`, nested input metadata, and output fields now use
|
||||
optional pointers where presence matters; check for nil before dereferencing.
|
||||
Consumers such as CommandKit can render the results directly instead of copying
|
||||
each field into a parallel report. See the result types' [GoDoc](../../resource_inspection.go)
|
||||
for keys and ownership; resource inspection does not replace `InspectConfiguration`
|
||||
provenance, `InspectRequirements`, or concrete-request `ValidateRequest`.
|
||||
|
||||
### Set A Per-Run Session And Reasoning
|
||||
|
||||
Supply a direct session ID when one prompt should be correlated with a
|
||||
@@ -742,7 +1056,7 @@ endpoint, model := "http://localhost:8000/v1", "example-model"
|
||||
profile := promptkit.ProfileSource("local profile", []promptkit.ProfileConfig{
|
||||
{ID: "local-summary", Endpoint: &endpoint, Model: &model},
|
||||
})
|
||||
// Add configkit.Input{Source: profile} to a caller underlay.
|
||||
// Add promptkit.SettingsInput(profile) to a caller underlay.
|
||||
```
|
||||
|
||||
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||
@@ -850,6 +1164,22 @@ secret-handling policies.
|
||||
|
||||
## Extension Interfaces
|
||||
|
||||
JSON Schema validation and native structured output use a self-contained root
|
||||
document. See [schema selection and resource context](../formats.md#schemas).
|
||||
Prepared executions retain their captured schema even if the source later changes.
|
||||
|
||||
### Migrating Schema Sources
|
||||
|
||||
`Config.SchemaDir`, `WithSchemaFS`, and `WithSchemaFile` have been removed.
|
||||
Supply a context-backed `SchemaRootSource` at the desired ConfigKit layer, as
|
||||
shown above, or declare `schema_root` in application or collection mappings.
|
||||
For a former single-file source, configure `schema_file` relative to its
|
||||
contributing source, or supply its directory as a root and use its basename in
|
||||
`schema_path`. To substitute an unrelated file per request, read and decode it
|
||||
in the caller and supply its value as `OutputContractInput.Schema`. There is no
|
||||
implicit working-directory root. Public structs gained fields; change any
|
||||
positional struct literals to keyed literals when migrating.
|
||||
|
||||
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||
context handling, ownership of copied values, nil responses, and preservation
|
||||
@@ -857,6 +1187,11 @@ of collaborator errors. Implementations must honor cancellation, safely manage
|
||||
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||
the application policy that motivated the injection.
|
||||
|
||||
A successful `ArtifactReader.Read` transfers the returned artifact and mutable
|
||||
body storage to Promptkit. Readers that previously reused a returned buffer
|
||||
must instead return independent storage and stop mutating it after return;
|
||||
private scratch buffers may still be reused when the returned bytes are copied.
|
||||
|
||||
## Handle Errors
|
||||
|
||||
Use `errors.Is` with the
|
||||
|
||||
@@ -58,6 +58,14 @@ exist.
|
||||
|
||||
## Maintainer Validation
|
||||
|
||||
Use the newest Go installation already available locally for ordinary
|
||||
development checks. Put its `bin` directory first on `PATH` and set
|
||||
`GOTOOLCHAIN=local` to keep that toolchain selected. For repository-isolated
|
||||
checks, set `GOWORK=off`; consumers may use their own workspaces. The selected
|
||||
validation patch does not by itself change the module's consumer minimum. See
|
||||
the [release toolchain policy](policy/release.md#consumer-minimum-and-release-toolchain)
|
||||
and [release procedure](release.md#prepare-the-candidate) for release selection.
|
||||
|
||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||
successful checks for unchanged content merely to create a commit. During
|
||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||
|
||||
272
docs/formats.md
272
docs/formats.md
@@ -6,71 +6,58 @@ This is the canonical reference for Promptkit's backend, profile, prompt, and
|
||||
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
||||
how to select sources and use an engine; the
|
||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
||||
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md)
|
||||
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md)
|
||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||
This reference describes what Promptkit does with the final effective values.
|
||||
|
||||
A configuration document may contain top-level `backends`, `profiles`, and
|
||||
`prompts` lists, plus a `collections` mapping of namespaced resource groups.
|
||||
Each resource has an explicit literal `id`, not a filename-derived identity.
|
||||
IDs and references must be nonblank and remain exact: Promptkit does not trim or
|
||||
re-key them. Unknown fields are errors. Promptkit installs no provider or
|
||||
profile catalog; callers select the definitions they need.
|
||||
A complete configuration document places Promptkit settings beneath `promptkit`.
|
||||
The `backends`, `profiles`, and `prompts` lists contain explicit literal
|
||||
`id` values; `collections` groups namespaced resources. IDs remain exact and
|
||||
do not include the `promptkit` path. Standalone resource fragments may omit
|
||||
the wrapper when explicitly mounted as described below.
|
||||
|
||||
```yaml
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: http://localhost:8000/v1
|
||||
concurrency_limit: 2
|
||||
profiles:
|
||||
- id: local-summary
|
||||
backend: local
|
||||
model: example-model
|
||||
prompts:
|
||||
- id: meeting.summary
|
||||
default_profile: local-summary
|
||||
messages:
|
||||
- role: user
|
||||
content: Summarize this note.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
collections:
|
||||
support:
|
||||
release_version: "1.4.0"
|
||||
backends:
|
||||
- id: offline
|
||||
endpoint: https://example.invalid/v1
|
||||
profiles:
|
||||
- id: base
|
||||
backend: support/offline
|
||||
model: offline-model
|
||||
- id: fast
|
||||
extends: {id: base}
|
||||
timeout_seconds: 30
|
||||
prompts:
|
||||
- id: answer
|
||||
default_profile: support/fast
|
||||
messages:
|
||||
- role: user
|
||||
content: Answer the selected caller input.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
promptkit:
|
||||
profiles:
|
||||
- id: local-summary
|
||||
endpoint: http://localhost:8000/v1
|
||||
model: example-model
|
||||
prompts:
|
||||
- id: meeting.summary
|
||||
default_profile: local-summary
|
||||
messages:
|
||||
- role: user
|
||||
content: Summarize this note.
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
collections:
|
||||
support:
|
||||
release_version: "1.4.0"
|
||||
backends:
|
||||
- id: offline
|
||||
endpoint: https://example.invalid/v1
|
||||
profiles:
|
||||
- id: fast
|
||||
backend: support/offline
|
||||
model: offline-model
|
||||
```
|
||||
|
||||
## Source Headers, Collections, And Resource IDs
|
||||
|
||||
An input may omit a source header. When present, the source-root `version` is
|
||||
reserved for ConfigKit's specification header and accepts only the exact string
|
||||
`"2"` or strict decimal integer `2`; null and other values are invalid. This
|
||||
header is distinct from a collection's `release_version` and ConfigKit's own
|
||||
software version. ConfigKit checks and consumes accepted headers on non-main
|
||||
inputs; a declaration on the main input remains visible in explicit
|
||||
configuration inspection. Headerless inputs still undergo ordinary final-value
|
||||
validation. ConfigKit owns generic import and source-history behavior.
|
||||
For a standalone prompt item, a root `version` is only a source header, not
|
||||
prompt metadata.
|
||||
Applications own ConfigKit's version policy and source controls. Promptkit adds
|
||||
no resource-version rule, including for collection, prompt, or profile records.
|
||||
An application's root `version` is distinct from a collection's descriptive
|
||||
`release_version`. Old root-level `prompts` or `profiles` keys are application
|
||||
data, not Promptkit resources, and are neither consumed nor specially rejected.
|
||||
|
||||
The `promptkit` section, each collection, backend, profile, and prompt retain
|
||||
unrecognized fields as application data. Such fields compose normally and are
|
||||
visible to application validation and configuration inspection, but do not
|
||||
become provider parameters or appear in resource reports. Grouping them under
|
||||
an application-specific key reduces collisions; a misspelled optional Promptkit
|
||||
field can also be accepted there. Nested Promptkit-owned objects, including
|
||||
message, input, output, and smoke-test objects, remain strict.
|
||||
|
||||
The root resource lists contain unqualified IDs. Each `collections` entry
|
||||
has a namespace key and may contain `release_version` plus the same three
|
||||
@@ -86,9 +73,10 @@ list. It is not a qualified or cross-collection reference.
|
||||
no label; an explicitly empty string is retained. It does not select a resource
|
||||
or change execution. Runtime lookups
|
||||
use the captured effective resource indexes. `InspectConfiguration` is an
|
||||
explicit operator view of composed settings and provenance. Normalization and
|
||||
defaults appear in Promptkit's interpreted execution views, not as changes to
|
||||
that composed tree. Source history is not a second runtime resolver, and the
|
||||
explicit operator view of resolved settings and provenance. Selected field
|
||||
normalizations and the derived queue default are recorded in that tree, with
|
||||
source history retained. Request-time completion remains separate. Source
|
||||
history is not a second runtime resolver, and the
|
||||
retained effective shape is not flattened or rewritten into those indexes.
|
||||
|
||||
For examples of explicitly selecting native collection inputs and inspecting
|
||||
@@ -97,8 +85,9 @@ inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
|
||||
breaking identity and prompt-selection changes, see the
|
||||
[configuration migration guide](consumers/configuration-migration.md).
|
||||
|
||||
Every effective backend, profile, and prompt definition is validated when
|
||||
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
||||
Every effective backend, profile, and prompt definition is validated during
|
||||
the registered configuration load and engine capture, even if never selected.
|
||||
A profile may
|
||||
be partial: an endpoint and model need only be complete after request overrides
|
||||
are applied. A prompt's default profile similarly need not be runnable without
|
||||
an overriding request. Missing request data, artifacts, schemas, credentials,
|
||||
@@ -109,13 +98,15 @@ and static template syntax at construction. It does not reread configuration or
|
||||
message files on ordinary runs; construct a new engine to observe edits.
|
||||
Schema and request artifact reads still happen during preparation. Use
|
||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
||||
for request-specific rendered details. For identity discovery before inspection,
|
||||
report, resource inspection for resolved prompt, profile, and backend fields,
|
||||
and preparation for completed request-specific details. For identity discovery before inspection,
|
||||
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||
|
||||
## Prompt Definitions
|
||||
|
||||
The `prompts` list contains definitions selected by their literal IDs:
|
||||
The `prompts` list contains definitions selected by their literal IDs. The
|
||||
following examples in this reference are wrapperless `promptkit` fragments
|
||||
unless explicitly shown as complete documents; mount them with `SettingsInput`.
|
||||
|
||||
```yaml
|
||||
prompts:
|
||||
@@ -146,8 +137,9 @@ prompts:
|
||||
`default_profile`, `description`, `session_id`, `inputs`, and cache-control
|
||||
fields may be absent or cleared with null where appropriate. A null whole
|
||||
definition, required role, output format, or validation mode is invalid.
|
||||
`required` and `repair_attempts` default to false and zero, respectively, and
|
||||
are not nullable. `InspectPrompt` selects the same captured definition without
|
||||
`required` defaults to false and is not nullable. An omitted or null
|
||||
`repair_attempts` resolves according to `validation_mode`; see the output
|
||||
contract below. `InspectPrompt` selects the same captured definition without
|
||||
rendering or resolving a profile.
|
||||
|
||||
### Inputs
|
||||
@@ -200,18 +192,30 @@ messages; the integration contract defines its wire representation.
|
||||
| --- | --- | --- |
|
||||
| `format` | yes | `text`, `markdown`, or `json` |
|
||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` |
|
||||
| `schema_path` | for `json_schema` | Path in the selected schema source |
|
||||
| `repair_attempts` | no | Integer 0–3; default 0. Positive values need `basic`, `json`, or `json_schema` |
|
||||
| `schema` | one selection for `json_schema` | Inline JSON Schema value |
|
||||
| `schema_file` | one selection for `json_schema` | File relative to the contributing field's source context |
|
||||
| `schema_path` | one selection for `json_schema` | Path within the applicable explicit schema root |
|
||||
| `repair_attempts` | no | Integer 0–3; omitted or null defaults to 0 for `none`, otherwise 1. Positive values need `basic`, `json`, or `json_schema` |
|
||||
|
||||
`none` skips content validation; `basic` requires nonempty content; `json`
|
||||
accepts any valid JSON value; `json_schema` checks the selected schema.
|
||||
In `json_schema` mode exactly one nonnull `schema`, `schema_file`, or
|
||||
`schema_path` must be selected. Null clears a selection; `false`, `true`, and
|
||||
`{}` are values. Blank references are invalid. Other modes ignore these fields
|
||||
for schema selection and loading. ConfigKit composes inline mappings normally;
|
||||
switching forms can require explicitly clearing an inherited alternative.
|
||||
Replacing an identified prompt record replaces its complete earlier record.
|
||||
`format` determines output artifact metadata. JSON Schema mode also supplies
|
||||
structured-output metadata to compatible clients. Plain `json` does not
|
||||
request a provider-native JSON-object constraint. A positive repair budget
|
||||
counts additional generations after failed validation; exhaustion returns the
|
||||
last candidate and its failed validation result. Operational errors remain
|
||||
errors. A request-level `OutputContract` replaces the whole configured one,
|
||||
not individual fields; an empty request format defaults to `text`.
|
||||
errors. A request-level `OutputContractInput` replaces the whole configured
|
||||
contract, not individual fields; an empty request format defaults to `text`.
|
||||
Its omitted budget resolves from its own validation mode, not the prompt's
|
||||
budget. Explicit zero disables repair. A failed validation can incur one extra
|
||||
generation by default, with corresponding latency and usage; correction is not
|
||||
guaranteed.
|
||||
|
||||
## Backend Definitions
|
||||
|
||||
@@ -228,7 +232,10 @@ negative value is invalid. For positive concurrency, omitted/null queue
|
||||
capacity defaults to 1024. An explicit queue capacity, including zero,
|
||||
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
||||
has no queue. Provider maps contain JSON-compatible values; null entries are
|
||||
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
||||
ordinary JSON null. A raw `api_key` is not a Promptkit credential field: an
|
||||
unrecognized backend field is application data and is not automatically redacted
|
||||
by Promptkit. Applications own sensitivity declarations and disclosure policy
|
||||
for their extension fields.
|
||||
|
||||
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
||||
It may contain `default_profile`, `reachability`, and `authentication`:
|
||||
@@ -301,7 +308,9 @@ Nonblank endpoints follow the backend URL rules; temperature is 0–2, top_p
|
||||
is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept
|
||||
JSON-compatible values with nonempty keys and cannot collide with the built-in
|
||||
client's [standard request fields](integrations/openai-compatible-chat.md#request-body).
|
||||
Credential values do not belong in profiles: raw `api_key` is invalid.
|
||||
A raw `api_key` is not a Promptkit credential setting; an unrecognized profile
|
||||
field is application data. Applications own its sensitivity declarations and
|
||||
disclosure policy, just as for backend extension fields.
|
||||
|
||||
### Defaults And Overrides
|
||||
|
||||
@@ -314,26 +323,92 @@ only a model or backend does not clear unrelated explicitly inherited fields.
|
||||
| Profile field | Absent | Explicit value | Explicit null |
|
||||
| --- | --- | --- | --- |
|
||||
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
|
||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Selected backend endpoint |
|
||||
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
||||
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
||||
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
|
||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Backend lookup name, if any |
|
||||
| `api_key_required` | False | Boolean, including false | False |
|
||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
|
||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Backend defaults |
|
||||
|
||||
An empty profile mapping does not clear backend map entries. After ConfigKit
|
||||
composition and inheritance, a whole-map null and omission both permit backend
|
||||
defaults. Nested maps merge recursively, while scalar and array values replace;
|
||||
nulls within provider maps or arrays remain literal data.
|
||||
When `api_key_required` is true and the profile source is absent or null,
|
||||
execution suppresses the backend credential source; an explicitly configured
|
||||
profile source remains in use.
|
||||
|
||||
For example, if the backend provides an endpoint, API-key environment name,
|
||||
and provider map, a profile containing `endpoint: null`, `api_key_env: null`,
|
||||
and `extra_params: null` now inherits those backend values after handoff (unless
|
||||
the required-key rule suppresses the credential source). Earlier versions used
|
||||
those nulls to suppress the corresponding fallback. Use `endpoint: ""` or
|
||||
`api_key_env: ""` to clear those individual strings. There is no profile-level
|
||||
whole-map suppression replacement; change the backend map through ordinary
|
||||
ConfigKit composition when different provider defaults are needed. ConfigKit's
|
||||
retained configuration and provenance continue to show the nulls.
|
||||
|
||||
An empty mapping does not clear inherited map entries; null clears the whole
|
||||
map. Nested maps merge recursively, while scalar and array values replace.
|
||||
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
||||
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
||||
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
||||
data, not a configuration inheritance instruction. An explicit zero timeout
|
||||
does not disable the caller context or HTTP transport timeout.
|
||||
|
||||
### Bundled OpenRouter Configuration
|
||||
|
||||
The default bundle installs root backend `openrouter`, endpoint
|
||||
`https://openrouter.ai/api/v1`, optional credential environment name
|
||||
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
|
||||
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
|
||||
applies. No prompt, implicit profile selection, or tuning defaults are installed.
|
||||
Disable the bundle for `NewEngine` with `WithBundledOpenRouter(false)`; an
|
||||
application-owned load selects `BundledOpenRouterUnderlay` explicitly.
|
||||
|
||||
The root profile IDs and their models are:
|
||||
|
||||
| Profile | Model |
|
||||
| --- | --- |
|
||||
| `gpt-astra-latest` | `~openai/gpt-astra-latest` |
|
||||
| `gpt-sol-latest` | `~openai/gpt-sol-latest` |
|
||||
| `gpt-terra-latest` | `~openai/gpt-terra-latest` |
|
||||
| `gpt-luna-latest` | `~openai/gpt-luna-latest` |
|
||||
| `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
||||
| `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
||||
| `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
||||
| `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
||||
| `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
||||
| `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
||||
| `auto` | `openrouter/auto` |
|
||||
| `free` | `openrouter/free` |
|
||||
|
||||
The profiles `auto-low`, `auto-medium`, `auto-high`, `auto-xhigh`, and
|
||||
`auto-max` extend `auto` and add an `auto-router` plugin with the corresponding
|
||||
`cost_tier`. Replacing `auto` changes their inherited backend and model while
|
||||
retaining their plugins. These are root IDs even when a model contains `/`.
|
||||
For example, a caller can replace `auto` as a complete record and extend another
|
||||
bundled profile under a new ID:
|
||||
|
||||
```yaml
|
||||
profiles:
|
||||
- id: auto
|
||||
backend: openrouter
|
||||
model: openrouter/free
|
||||
- id: my-luna
|
||||
extends: {id: gpt-luna-latest}
|
||||
timeout_seconds: 60
|
||||
```
|
||||
|
||||
See OpenRouter's [alias examples](https://openrouter.ai/~openai/gpt-luna-latest),
|
||||
[auto router and cost tiers](https://openrouter.ai/docs/guides/routing/routers/auto-router),
|
||||
and [free router](https://openrouter.ai/docs/guides/routing/routers/free-router)
|
||||
for provider behavior and availability.
|
||||
|
||||
### Source And Profile Precedence
|
||||
|
||||
`NewEngine` composes caller underlays in their given order, followed by an
|
||||
`NewEngine` composes the bundled OpenRouter underlay first by default, then
|
||||
caller underlays in their given order, followed by an
|
||||
optional main input. Later layers overlay earlier ones. Inputs within one
|
||||
underlay group are strictly composed: conflicting contributions are errors,
|
||||
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
||||
@@ -344,8 +419,12 @@ on an input. Single files can use ConfigKit inputs directly.
|
||||
|
||||
Root backends, profiles, and prompts, as well as collection declarations and
|
||||
their resource lists, follow these same source rules. `CollectionInput`
|
||||
mounts one explicitly selected source at a collection destination and enables
|
||||
mounts one explicitly selected source at `/promptkit/collections/<namespace>` and enables
|
||||
ConfigKit imports for that input; it does not discover or select sources.
|
||||
`SettingsInput` mounts wrapperless settings fragments at `/promptkit`; an
|
||||
id-first resource file uses ConfigKit sequence-item placement at the matching
|
||||
resource list. Directory discovery retains root destinations, so callers mount
|
||||
discovered wrapperless fragments explicitly. No document shape is inferred.
|
||||
A later same-ID record replaces the lower definition in full. Use a distinct
|
||||
inherited sibling when a variant should retain its parent's fields.
|
||||
A request `ProfileID` takes precedence over a prompt `default_profile`; if
|
||||
@@ -363,16 +442,39 @@ references are exact IDs and use their qualified form when they select a
|
||||
collection resource. There is no Promptkit-specific depth limit or second
|
||||
base-chain resolver. Missing references and cycles fail at construction. For
|
||||
generic inheritance and layer mechanics, consult
|
||||
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md).
|
||||
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md).
|
||||
|
||||
## Schemas
|
||||
|
||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
||||
referenced nested schemas resolve relative to the owning schema document.
|
||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
||||
invalid, or unresolvable schema causes an operational validation error.
|
||||
Schemas can be inline, source-relative files, or files under an explicit root.
|
||||
An optional `schema_root` may be set at the `promptkit` root and in each
|
||||
collection. A collection's nonnull root takes precedence over the application
|
||||
root for its prompts; an absent or null collection root falls back to the
|
||||
application root. A nonnull blank root is invalid. There is no implicit
|
||||
working-directory root. Roots and `schema_file` references resolve using the
|
||||
effective field's contributing ConfigKit filesystem and base directory;
|
||||
missing usable context is a configuration error. An explicit `.` is valid
|
||||
with usable context. A `schema_path` requires an applicable root. Root choice
|
||||
is made before reading: a missing file never triggers another-root search.
|
||||
Schema selection and context are captured during construction without reading
|
||||
schema files. Files are read and compiled during preparation; a missing,
|
||||
malformed, or unresolvable file is an operational validation error. Inline
|
||||
configuration values follow native ConfigKit composition and null handling.
|
||||
For request-level replacement, inline `Schema` needs no root and `SchemaPath`
|
||||
uses the selected prompt's applicable root. A replacement does not inherit the
|
||||
configured schema selection or `schema_file` context.
|
||||
Runtime inline values are encoded as JSON before preparation; file-backed JSON
|
||||
uses the existing number-preserving decoder. No additional numerical equivalence
|
||||
between input representations is implied.
|
||||
|
||||
Each schema must be self-contained: in-document references, IDs, and anchors are
|
||||
supported, but references requiring another file or network resource fail.
|
||||
References must also work without the source filename as a base; use fragment
|
||||
references or explicit schema IDs rather than relying on that filename.
|
||||
Valid JSON that fails the schema instead yields a failed validation result.
|
||||
Schema documents are detached without applying the provider extra-parameter
|
||||
copy limits; this does not change the schema decoder, compiler, or
|
||||
self-contained-resource rules.
|
||||
|
||||
## Credentials
|
||||
|
||||
|
||||
@@ -16,14 +16,13 @@ Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
||||
Before the client is called, the engine resolves framework, backend, profile,
|
||||
and request values into one execution target. Endpoint configuration is trimmed
|
||||
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||
information, a query, or a fragment. A non-empty endpoint from the target
|
||||
overrides the client's configured base URL. The final selected endpoint is
|
||||
validated again before transport.
|
||||
information, a query, or a fragment. The target's endpoint is validated again
|
||||
before transport.
|
||||
|
||||
The completion URL is composed through parsed URL path operations. Nested base
|
||||
paths are retained, repeated trailing slashes are normalized, and the result
|
||||
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||
sending when neither source supplies a valid endpoint.
|
||||
sending when the target does not supply a valid endpoint.
|
||||
|
||||
The target's backend ID is routing metadata for prepared values, results, and
|
||||
injected clients. The built-in client does not derive the URL from that ID and
|
||||
@@ -70,7 +69,9 @@ and numeric `usage`; unrelated fields and account limits are ignored. This is
|
||||
only a minimal identifying subset of OpenRouter's documented
|
||||
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||
not validation of the full account document. Other statuses and unrecognized
|
||||
200 responses do not establish acceptance.
|
||||
200 responses do not establish acceptance. A 200 response exceeding the bounded
|
||||
recognition allowance reports `authentication_response_limit_exceeded`, leaving
|
||||
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
|
||||
|
||||
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||
RoundTripper boundary and any response associated with it. It includes the
|
||||
@@ -87,12 +88,13 @@ transport for HTTP-only diagnostics.
|
||||
## Request Body
|
||||
|
||||
The request body always contains `model` and `messages`. The execution
|
||||
target's model takes precedence over the client's configured model, and one
|
||||
must be available.
|
||||
target's model must be available.
|
||||
|
||||
Each ordinary message contains its `role` and string `content`. A
|
||||
cache-controlled message instead uses a text content block containing `type`,
|
||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||
For a continuation, captured history precedes the new definition and appended
|
||||
messages. Ancestry is local metadata and is not an upstream request field.
|
||||
|
||||
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
||||
does so without provider-specific translation. Tool and deprecated function
|
||||
@@ -115,9 +117,10 @@ The client conditionally includes:
|
||||
strict flag, and schema document. Plain JSON validation does not add an
|
||||
object-only response constraint.
|
||||
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
||||
Configured profile maps recursively overlay backend defaults, while a nonempty
|
||||
runtime override replaces the complete configured map. The [format reference](../formats.md#defaults-and-overrides)
|
||||
owns these resolution rules. The resulting effective map is merged into the
|
||||
top-level body. Empty keys, values that cannot be encoded, and
|
||||
collisions with these reserved fields are rejected before any provider call:
|
||||
|
||||
- `model`
|
||||
|
||||
@@ -68,9 +68,8 @@ Preparation and validation never hold an active permit.
|
||||
|
||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||
returns an idempotent release function. The coordinator uses it around
|
||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
||||
operation for focused internal callers and tests, but engine assembly no
|
||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
||||
individual attempts. There is one queue and one active limit per limited backend
|
||||
pool.
|
||||
|
||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||
@@ -121,7 +120,7 @@ validation, bounded admission, idempotent release, context handling, and
|
||||
unlimited admission. The
|
||||
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||
unlimited calls, passthrough behavior, and panic release.
|
||||
unlimited calls, and panic release using test-only one-shot calls.
|
||||
|
||||
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||
admission, lease lifetime, failure release, and shared initial/repair
|
||||
|
||||
@@ -26,10 +26,9 @@ client uses its endpoint, credential metadata, generation fields, and extra
|
||||
parameters. `BackendID` remains routing metadata
|
||||
for the generation boundary and is not mapped into the provider payload.
|
||||
|
||||
Construction trims and validates a nonempty configured base URL and clones any
|
||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
||||
mutating the caller's client. An empty configured base remains valid because a
|
||||
resolved request target may supply the endpoint. Generation preparation then:
|
||||
Construction clones any supplied `http.Client` so Promptkit can apply its
|
||||
timeout default without mutating the caller's client. Generation preparation
|
||||
then:
|
||||
|
||||
1. validates shared execution-setting invariants and the final selected base
|
||||
endpoint;
|
||||
@@ -42,6 +41,10 @@ resolved request target may supply the endpoint. Generation preparation then:
|
||||
the complete-request failure remains a request failure;
|
||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||
|
||||
Provider values are detached with trusted copying before encoding. Custom JSON
|
||||
encoding methods and per-value encoding-failure diagnostics operate on those
|
||||
private copies rather than retained prepared or diagnostic metadata.
|
||||
|
||||
Each execution of that prepared request creates a fresh HTTP request, body
|
||||
reader, and attempt timeout context. It closes the response body and returns
|
||||
the HTTP status, decoded response or error, reported token values and presence,
|
||||
@@ -77,7 +80,8 @@ not retain headers, credentials, raw request or response bodies, or a redirect
|
||||
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||
Reachability and named authentication probes use the same configured built-in
|
||||
client but discard their response bodies after local recognition. The
|
||||
client. The OpenRouter adapter owns endpoint derivation, bounded account-response
|
||||
recognition, and body closure; the smoke coordinator interprets its evidence. The
|
||||
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||
owns their observable HTTP semantics.
|
||||
|
||||
@@ -105,14 +109,13 @@ public ownership and redaction semantics belong to the
|
||||
|
||||
## Failure Categories
|
||||
|
||||
The package preserves distinct error identities for invalid client
|
||||
configuration, invalid generation requests, request execution failures,
|
||||
non-success provider statuses, and malformed successful responses. Provider
|
||||
response bodies are never exposed in raw form through non-success errors.
|
||||
The package preserves distinct error identities for invalid generation
|
||||
requests, request execution failures, non-success provider statuses, and
|
||||
malformed successful responses. Provider response bodies are never exposed in
|
||||
raw form through non-success errors.
|
||||
|
||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
||||
invalid final selected endpoint is an invalid generation request and is
|
||||
rejected before transport.
|
||||
A missing or invalid final selected endpoint is an invalid generation request
|
||||
and is rejected before transport.
|
||||
|
||||
Authentication resolves a trimmed direct key before a trimmed configured
|
||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||
@@ -156,6 +159,13 @@ caller deadlines, generation deadlines, and whole-request client timeouts
|
||||
retain `context.DeadlineExceeded`. The runner adds its generation category
|
||||
without discarding those identities or depending on HTTP status mapping.
|
||||
|
||||
When a transport failure carries a custom context cancellation cause, the
|
||||
client also retains the standard context identity. It correlates that cause
|
||||
with the returned failure rather than labeling unrelated transport errors from
|
||||
context state alone. Cancellation while reading a successful response body
|
||||
retains the original read failure alongside the context identity within the
|
||||
redacting wrapper.
|
||||
|
||||
## Test Ownership
|
||||
|
||||
The
|
||||
|
||||
@@ -10,6 +10,13 @@ lifecycle instrumentation spans the facade and runner boundaries.
|
||||
Prepared execution starts only after its private payload is atomically claimed;
|
||||
the facade receives that invocation's recorder privately and emits the terminal
|
||||
event after public error mapping. Failed claims and preparation emit nothing.
|
||||
The public projection retains private operation-owner metadata separately from
|
||||
inspectable child causes. Terminal classification uses that owner, preserving
|
||||
context precedence and generation-specific transport/provider details rather
|
||||
than mistaking a collaborator's nested lookup error for the parent's operation.
|
||||
Timeout attribution is finalized and the child scope released before terminal
|
||||
delivery, so observer latency cannot change the already selected outcome.
|
||||
Prepared-content notification is a separate opt-in callback, not an event.
|
||||
|
||||
The facade masks inherited recorder and prepared-claim state when entering
|
||||
preparation or execution. Delegated calls from an injected collaborator retain
|
||||
@@ -23,6 +30,8 @@ callback. The caller supplies normalized failure metadata; the recorder never
|
||||
parses an error string or receives provider bodies, credentials, headers, or
|
||||
other content. Events are values with no mutable containers. Callback panics
|
||||
propagate to the caller.
|
||||
The immediate parent run ID is set before start delivery and is retained as
|
||||
selection metadata changes. It describes ancestry, not implicit group membership.
|
||||
Resolved prompt and profile IDs are the exact runtime identities selected for
|
||||
the invocation, including namespace qualification when supplied by an engine
|
||||
view.
|
||||
@@ -44,6 +53,13 @@ attempts above the first in a generation, and repair generations with at least
|
||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||
attempt events.
|
||||
|
||||
An execution group is a zero-ready synchronized collector of explicitly selected
|
||||
events. It enrolls by run ID and aggregates each run's first terminal counters and
|
||||
presence-aware usage exactly once. Snapshots copy members and totals under one
|
||||
lock; terminal coverage and token completeness are distinct. Observer fan-out
|
||||
copies the callback slice and invokes callbacks synchronously without holding a
|
||||
group lock. No group history, registry, or automatic nested membership exists.
|
||||
|
||||
Smoke diagnostics mask inherited execution-recorder state while invoking
|
||||
generation or HTTP collaborators. Their private attempt evidence is assembled
|
||||
into a `SmokeTestReport`; it does not create ordinary run events or enter an
|
||||
|
||||
@@ -11,27 +11,28 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, resource and requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||
| `examples/go-library/observe` | Demonstrates offline retry, observation, prepared notification, and separately budgeted coordinated proposal and assessment runs using two profiles on one finite backend. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/configuration` | Loads root and namespaced collection resources through ConfigKit; records effective field presence, validates final values including exact references and optional diagnostic metadata, captures indexed prompt definitions and source-relative message content, compiled templates, input metadata snapshots, sorted backend/profile/prompt identities, resource namespace sets, effective collection declarations, backend and presence-aware profile snapshots, and the unchanged read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/configuration` | Supplies component rules and captures final shared settings and native inspection evidence into owned indexed resources, compiled templates, requirement metadata, identities, and execution profiles. | [Configuration capture](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||
| `internal/settings` | Declares the shared public configuration data model exposed through root aliases under ADR 0002; contains no loading or execution logic. | [Settings declarations](../../internal/settings/settings.go), [ADR 0002](../adr/0002-shared-configuration-data-model.md) |
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The internal engine assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
@@ -30,6 +30,29 @@ internal callers and tests.
|
||||
Each invocation carries its state in request, prepared-run, and result values.
|
||||
The runner has no durable run or session store.
|
||||
|
||||
Request output-contract replacements carry an optional repair budget. The
|
||||
runner resolves that budget from the replacement's validation mode before
|
||||
building the concrete validation plan; prepared execution retains the resolved
|
||||
integer. Configured budgets are already resolved by the configuration adapter.
|
||||
|
||||
Captured exchanges are immutable parent-linked domain values. Each holds only
|
||||
its new rendered suffix, final assistant message, and run identities; flattening
|
||||
iteratively fills a single detached message buffer for preparation or projects
|
||||
directly into public inspection values. Preparation
|
||||
prepends this history to newly rendered definition messages and appended caller
|
||||
messages before computing the rendered hash. Follow-up selection and credentials
|
||||
remain those of the new request. Prepared execution retains capture and parent
|
||||
references in its single-claim payload, not its durable details; discard clears
|
||||
the payload. Completed runs capture the exact final output, including exhausted
|
||||
structural repair, while operational errors produce no exchange.
|
||||
|
||||
The sequential coordinator uses ordinary facade runs and retains the current
|
||||
immutable exchange independently of callback-owned values. Each run releases its
|
||||
logical admission lease and finishes observation before application callback
|
||||
delivery. Context checks preserve a completed but undelivered result only when
|
||||
cancellation is observed before invoking the callback. Coordinator budgets count
|
||||
started runs, not transport attempts or structural repairs.
|
||||
|
||||
## Shared Prompt Selection
|
||||
|
||||
The internal engine applies each namespace view to slashless caller selections
|
||||
@@ -42,14 +65,13 @@ Ordinary preparation selects the full prompt definition and construction-time
|
||||
hash. It retains its early request-ID check before direct-session
|
||||
normalization. Referenced message content and static template syntax were
|
||||
captured when the engine was built.
|
||||
Rendering uses those captured parses, cloning them per call to bind request
|
||||
input helpers; it does not parse templates during execution.
|
||||
|
||||
`InspectPrompt` selects captured metadata through an optional repository
|
||||
capability, falling back to the full definition and existing hash resolution
|
||||
for internal repositories without it. It returns owned metadata without
|
||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
||||
admission, or model collaborators. The root
|
||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
Resource inspection bypasses the runner: the internal engine selects a captured
|
||||
configuration resource, and the facade detaches its values for the caller.
|
||||
Prompt resource inspection includes captured message text without rendering or
|
||||
touching profile, artifact, schema, admission, or model collaborators.
|
||||
|
||||
`InspectRequirements` uses the same metadata selection to resolve the
|
||||
request-profile/default identity, then obtains the captured requirement
|
||||
@@ -57,24 +79,18 @@ snapshot by the selected prompt ID. The analyzer receives only
|
||||
those identity fields rather than an execution definition. It analyzes the
|
||||
snapshot with the supplied caller variables and normalized direct-session
|
||||
bypass; it does not resolve target readiness or enter the preparation
|
||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
||||
source fallback. The root
|
||||
pipeline. An unavailable captured snapshot is a prompt-load failure. The root
|
||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||
operation's exact contract.
|
||||
|
||||
## Shared Profile Selection
|
||||
|
||||
The runner uses one profile-selection and target-resolution boundary for
|
||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
||||
request profile or a prompt default; inspection begins with its required
|
||||
explicit profile ID. Both use the captured, presence-aware target constructed
|
||||
from the final effective profile and backend configuration. Request overrides
|
||||
apply afterward; preparation checks endpoint and model completeness only after
|
||||
those overrides. Inspection can report a partial profile without requiring a
|
||||
runnable target. Neither path checks credential availability or performs prompt,
|
||||
artifact, schema, rendering, admission, or model-client work. The root
|
||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
||||
exact contract.
|
||||
The runner selects a request profile or a prompt default and uses its captured,
|
||||
presence-aware target constructed from final profile and backend configuration.
|
||||
Request overrides apply afterward; preparation checks endpoint and model
|
||||
completeness only after those overrides. Resource profile inspection bypasses
|
||||
this path and reports the profile before backend completion. Credential
|
||||
availability is checked when execution preparation requires it.
|
||||
|
||||
## Smoke-Test Diagnostics
|
||||
|
||||
@@ -115,12 +131,12 @@ performs only the work needed to validate routing and admission:
|
||||
|
||||
1. validate the required prompt selection and normalize any direct session ID;
|
||||
2. select the captured prompt definition and hash;
|
||||
3. select the request profile or the prompt's default profile;
|
||||
4. retain the profile's selected backend identity, when present;
|
||||
5. apply explicit request overrides to its captured backend-completed target;
|
||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
7. resolve and validate the effective output contract without loading its
|
||||
schema; and
|
||||
3. resolve and validate the effective output contract and select its applicable
|
||||
bound file without loading its schema;
|
||||
4. select the request profile or the prompt's default profile;
|
||||
5. retain the profile's selected backend identity, when present;
|
||||
6. apply explicit request overrides to its captured backend-completed target;
|
||||
7. validate endpoint, model, numeric overrides, and credential requirements; and
|
||||
8. retain the definition, source identities, effective settings, output
|
||||
contract, and preparation start time in invocation-local state.
|
||||
|
||||
@@ -131,9 +147,9 @@ profile, or backend:
|
||||
schema metadata from it when required;
|
||||
2. load and hash input artifacts;
|
||||
3. render messages and the prompt-defined session;
|
||||
4. apply any direct session ID, then append already-normalized request messages
|
||||
after the rendered definition messages;
|
||||
5. hash the effective rendered prompt; and
|
||||
4. apply any direct session ID, then compose captured history, rendered
|
||||
definition messages, and already-normalized request messages in that order;
|
||||
5. hash the effective rendered prompt and its ordered message content; and
|
||||
6. construct the prepared value and preparation timing.
|
||||
|
||||
`Prepare` runs both phases consecutively and never performs capacity admission.
|
||||
@@ -142,11 +158,10 @@ one execution-precedence and error-ordering implementation while allowing a
|
||||
full backend pool to reject work before expensive schema, artifact, and
|
||||
rendering operations. `Prepare` discards the plan after returning its public
|
||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||
and discards it when the operation ends. Prepared execution transfers the
|
||||
completed preparation's owned message, input-hash, and copied-schema values
|
||||
into its private payload, while deep-copying mutable provider extras and
|
||||
creating an independent details snapshot. Claim completion or discard clears
|
||||
the execution payload without removing the details snapshot.
|
||||
and discards it when the operation ends. Prepared execution retains one
|
||||
immutable, redacted metadata snapshot shared by execution and internal details.
|
||||
The facade detaches mutable public details. Claim completion or discard clears
|
||||
execution-only payload references without modifying the retained metadata.
|
||||
|
||||
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||
path as `Prepare` and discards the resulting internal prepared value without
|
||||
@@ -175,8 +190,14 @@ sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
|
||||
encoding of that session, every role and content value, and cache-control
|
||||
presence and values; its hexadecimal value is opaque.
|
||||
|
||||
The separate message-content hash is computed once after that same composition.
|
||||
It streams a versioned prefix, message count, and length-framed role and text
|
||||
bytes into SHA-256, excluding session and cache metadata. Fixed reference
|
||||
vectors protect the selected v1 scheme across refactors; the public
|
||||
[`PreparedRun` GoDoc](../../types.go) owns its compatibility contract.
|
||||
|
||||
The configuration snapshot is read-only after engine construction. Concurrent
|
||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
||||
`Prepare` and `Run` calls use detached captured targets and keep invocation
|
||||
state local.
|
||||
|
||||
## Run Flow
|
||||
@@ -199,6 +220,16 @@ mapping and after the runner's deferred admission release. Preparation and
|
||||
prepared execution follows the same terminal mapping after a successful claim;
|
||||
invalid or discarded handles do not acquire an observer.
|
||||
|
||||
The facade establishes an optional operation scope for ordinary runs before
|
||||
observation and preparation. A claimed prepared execution establishes its own
|
||||
scope from the retained duration; preparation and idle time do not consume it.
|
||||
The shared scope finalizer marks only a context failure caused by that scope's
|
||||
own deadline, preserving its underlying operation and cause. The scope is
|
||||
released before terminal delivery. At the shared prepared execution boundary,
|
||||
the runner conditionally invokes the adapter for the actual prepared snapshot
|
||||
while admission is held, with context checks before and after notification.
|
||||
The facade adapter projects a detached public value only when selected.
|
||||
|
||||
Validation preparation and execution honor cancellation at every
|
||||
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||
Schema reads are bounded and context-checked between chunks; JSON decoding,
|
||||
|
||||
@@ -10,39 +10,24 @@ source semantics. Internal packages are not consumer APIs.
|
||||
|
||||
## Typed Configuration Loading
|
||||
|
||||
`internal/configuration` is the single adapter from ConfigKit's final
|
||||
effective configuration to execution-owned definitions. It performs one
|
||||
layered load, captures effective presence (including explicit null), builds
|
||||
detached normalized interpretations during validation, and validates all effective
|
||||
root and collection backend, profile, and prompt values. Replaced lower-layer
|
||||
domain values are not separately validated. It indexes root resources under
|
||||
their unqualified IDs and collection resources under their full
|
||||
`namespace/local-key` IDs while retaining ConfigKit's effective tree in its
|
||||
authored shape. It builds an immutable backend index and presence-aware profile
|
||||
targets, applying final backend defaults only after profile inheritance. Runtime
|
||||
lookups use those final indexes and exact stored references; they do not rewrite
|
||||
or resolve resources from ConfigKit source history. The runtime/domain packages
|
||||
do not depend on ConfigKit.
|
||||
The root facade composes Promptkit rules into a ConfigKit definition. An
|
||||
application-owned load or the standalone constructor produces final settings
|
||||
at `/promptkit`; `internal/configuration` captures execution-owned resources
|
||||
from those settings and a native inspection view without another load.
|
||||
`internal/settings` supplies the shared data declarations exposed through
|
||||
root aliases. The adapter records normalization and validates effective values,
|
||||
then uses one metadata snapshot for presence and source origins. Runtime/domain
|
||||
packages do not depend on ConfigKit.
|
||||
|
||||
ConfigKit owns source headers, imports, composition, and source evidence.
|
||||
Promptkit does not preparse contributions: it validates the final effective
|
||||
values once, captures the indexed identities and their original resource
|
||||
contexts, and retains ConfigKit's composed result for explicit operator inspection.
|
||||
Accepted non-main source headers are consumed during composition; the main
|
||||
source declaration remains available in that inspection view. Collection
|
||||
release labels are descriptive metadata, not selection inputs. The
|
||||
[format reference](../formats.md#source-headers-collections-and-resource-ids)
|
||||
owns the exact source-header and collection contracts.
|
||||
|
||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||
once to ConfigKit v1.0.0 and returns its public difference records. Comparison
|
||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||
both results; reports may still contain ordinary application data. See the
|
||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
and ConfigKit's [v1.0.0 comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md#effective-configuration-diffs).
|
||||
Capture detaches known settings, builds indexed definitions and profile targets,
|
||||
and binds source-relative resources from the effective field's source context.
|
||||
It retains a frozen native focused view and copied diagnostics, not the mutable
|
||||
application result. Configuration inspection delegates reporting, redaction,
|
||||
history and comparison to ConfigKit; explicit queries and metadata retain
|
||||
whole-application paths and evidence. Runtime never derives values from history.
|
||||
The [format reference](../formats.md#source-headers-collections-and-resource-ids)
|
||||
owns the configuration contract and the [consumer guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||
describes reporting scope.
|
||||
|
||||
After resource materialization and effective-profile snapshot construction, the
|
||||
adapter captures lexically ordered identity inventories for backends, profiles,
|
||||
@@ -53,13 +38,15 @@ captured values without consulting ConfigKit or copying resource definitions.
|
||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||
files under an explicit root, provide resource context, and leave imports
|
||||
disabled. Each returned slice belongs to one strict-composition group; file
|
||||
order does not confer priority. Caller underlays are composed in their given
|
||||
order does not confer priority. The standalone facade prepends a dedicated
|
||||
embedded OpenRouter underlay unless disabled, without modifying caller-owned slices.
|
||||
Caller underlays are composed in their given
|
||||
order, followed by the optional main input. The
|
||||
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
||||
real loader and final-value handoff.
|
||||
|
||||
The public `CollectionInput` helper mounts one caller-selected source at
|
||||
`collections/<namespace>` and enables ConfigKit imports for that input. The
|
||||
`/promptkit/collections/<namespace>` and enables ConfigKit imports for that input. The
|
||||
helper does not read the source; ConfigKit retains responsibility for loading,
|
||||
composition, imports, and source-local resource context. Imported or inherited
|
||||
`content_file` values continue resolving relative to the source that contributed
|
||||
@@ -82,23 +69,27 @@ declaration; absent and null fields remain distinguishable from explicit
|
||||
strings, including empty strings. It closes content-file handles before
|
||||
returning and retains no source capability for public inspection.
|
||||
|
||||
Prompt materialization shares a lazy metadata snapshot across root and collection
|
||||
Prompt materialization shares one metadata snapshot across root and collection
|
||||
resources during construction, avoiding a full metadata copy per collection.
|
||||
Input metadata, message content and templates, and output contracts are captured
|
||||
separately before the definition, inspection, and hash are assembled. Schema
|
||||
root and source-relative file bindings capture the effective field context
|
||||
without opening schema files; capabilities are excluded from the hash. Inline
|
||||
schema values and file-reference strings participate in the definition hash.
|
||||
|
||||
Normalization and defaults are applied to execution-owned interpretations,
|
||||
including message roles and cache controls. ConfigKit retains the composed
|
||||
values and origins, rather than synthetic Promptkit normalization events. List
|
||||
indexes identify fields for presence, validation, sensitivity, and resource
|
||||
context; full IDs select captured runtime indexes. Recording a sequence-wide
|
||||
normalization would replace origins of unchanged content-file fields.
|
||||
Normalization and defaults are recorded at individual resolved field paths,
|
||||
including message roles and cache controls. List indexes identify fields for
|
||||
presence, validation, sensitivity, and resource context; full IDs select
|
||||
captured runtime indexes. Unchanged content-file fields retain their origins.
|
||||
Backend-to-profile and request-time completion remain separate runtime work.
|
||||
|
||||
The runner selects full definitions from this index for preparation and uses
|
||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||
internal requirement snapshot lookup uses the selected prompt ID alone.
|
||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
||||
returns declared metadata without rendering or resolving profile, input, or
|
||||
schema resources, and its public shape and prompt hashes do not include the
|
||||
captured presence state. Runtime rendering still checks request variables and
|
||||
The runner selects full definitions from this index for preparation and retains
|
||||
lightweight prompt metadata for `InspectRequirements`; the requirement snapshot
|
||||
lookup uses the selected prompt ID alone. Separate configuration-owned resource
|
||||
inspection projects the resolved record and captured messages without rendering
|
||||
or resolving profile, input, or schema resources. These paths do not rescan
|
||||
configuration or reopen message files. Presence metadata does not change the
|
||||
opaque definition hash. Runtime rendering still checks request variables and
|
||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||
cover resource origins, context containment, capture, and selection.
|
||||
|
||||
@@ -146,11 +137,15 @@ change rendering behavior.
|
||||
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
||||
The configuration adapter validates final effective resource references and
|
||||
field values, then builds target
|
||||
snapshots with backend defaults and presence/suppression state. A profile can
|
||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||
that captured target without prompt, artifact, schema, credential lookup, or
|
||||
provider work. Preparation applies runtime overrides and then checks target
|
||||
field values, then captures backend definitions and profile target snapshots
|
||||
with backend defaults and outbound-control presence. Profiles without provider
|
||||
overrides share capture-owned immutable backend parameter maps; target lookups
|
||||
still detach mutable values before runtime use. Null profile endpoint,
|
||||
credential source, and whole provider map permit backend fallback after
|
||||
composition, subject to the required-credential rule. A profile can remain
|
||||
partial until runtime overrides complete it. Resource inspection selects indexed
|
||||
resolved backend and profile records without execution completion or prompt,
|
||||
artifact, schema, credential lookup, or provider work. Preparation applies runtime overrides and then checks target
|
||||
completeness. Identity-only profile selection can identify a partial captured
|
||||
profile without copying its target or requiring readiness. The
|
||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||
@@ -208,20 +203,24 @@ buffer-to-string copy. The
|
||||
|
||||
## Schemas And Output Validation
|
||||
|
||||
`internal/validate` provides validators backed by an operating-system
|
||||
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||
`internal/validate` prepares a source-neutral input containing an output contract
|
||||
and an optional bound `fs.FS` file. Invalid generated content is returned as a validation
|
||||
result; inability to load, register, or compile a schema is an operational
|
||||
error. Basic validation checks body emptiness on the byte slice directly
|
||||
instead of first making a response-sized string copy.
|
||||
|
||||
Every preparation operation creates one operation-local validation plan. None,
|
||||
basic, and JSON modes retain the effective output contract without source
|
||||
access. JSON Schema mode loads the root document once, resolves and compiles
|
||||
each transitive reference, and retains the compiled validator. Schema compiler
|
||||
resources use canonical escaped file or private-scheme URLs; loaders decode
|
||||
their paths once and enforce the configured source boundary. The
|
||||
access. JSON Schema mode uses the captured inline document or loads the bound
|
||||
root document once, compiles its declared
|
||||
schema locations, and retains the compiled root validator. External resources
|
||||
are rejected without file or network loading; in-document references remain
|
||||
supported. Bound files use canonical escaped private-scheme URLs; inline values
|
||||
use a stable private retrieval URL. The
|
||||
provider-facing structured-output metadata uses the root document captured by
|
||||
the same plan.
|
||||
the same plan. Its detached schema document is not revalidated against the
|
||||
runtime provider-value copier's traversal limits. Smoke inference uses a
|
||||
separate JSON-only prepared plan before checking the prepared schema plan.
|
||||
|
||||
Schema preparation and execution remain synchronous. Promptkit checks
|
||||
cancellation before and after source resolution, JSON decoding, compilation,
|
||||
|
||||
@@ -26,67 +26,60 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||
|
||||
## ConfigKit
|
||||
|
||||
Selected module: `go.maximumdirect.net/configkit` at `v1.0.0`.
|
||||
Selected module: `go.maximumdirect.net/configkit` at `v1.4.0`.
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53/COPYRIGHT) |
|
||||
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53/LICENSE).
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3/COPYRIGHT) |
|
||||
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3/LICENSE).
|
||||
|
||||
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53).
|
||||
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3).
|
||||
|
||||
## ReleaseKit
|
||||
|
||||
Selected module: `go.maximumdirect.net/releasekit` at `v0.10.0` (maintainer tool).
|
||||
|
||||
Copyright (C) 2026 Eric Rakestraw
|
||||
|
||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243/COPYRIGHT) |
|
||||
[LICENSE](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243/LICENSE).
|
||||
|
||||
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243).
|
||||
|
||||
## jsonschema
|
||||
|
||||
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.2`.
|
||||
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.3`.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/29cbed948d24a04700eb94436416b18a07953b71/LICENSE).
|
||||
[LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/b0fc661f4939578bc429f408c18202573dbafcc4/LICENSE).
|
||||
|
||||
[Source](https://github.com/santhosh-tekuri/jsonschema/tree/29cbed948d24a04700eb94436416b18a07953b71).
|
||||
[Source](https://github.com/santhosh-tekuri/jsonschema/tree/b0fc661f4939578bc429f408c18202573dbafcc4).
|
||||
|
||||
## Go Text
|
||||
|
||||
Selected module: `golang.org/x/text` at `v0.14.0` (transitive dependency).
|
||||
Selected module: `golang.org/x/text` at `v0.44.0` (transitive dependency).
|
||||
|
||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
||||
Copyright 2009 The Go Authors.
|
||||
|
||||
SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
||||
[LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) |
|
||||
[PATENTS](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS).
|
||||
[LICENSE](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/LICENSE) |
|
||||
[PATENTS](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/PATENTS).
|
||||
|
||||
[Source](https://github.com/golang/text/tree/6c97a165dd661335ff7bce6104a008558123c353).
|
||||
|
||||
## Go YAML v3
|
||||
|
||||
Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency).
|
||||
|
||||
Copyright (c) 2006-2010 Kirill Simonov
|
||||
|
||||
Copyright (c) 2006-2011 Kirill Simonov
|
||||
|
||||
Copyright (c) 2011-2019 Canonical Ltd
|
||||
|
||||
Copyright 2011-2016 Canonical Ltd.
|
||||
|
||||
SPDX-License-Identifier: MIT AND Apache-2.0
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[LICENSE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
|
||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
||||
|
||||
[Source](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
||||
[Source](https://github.com/golang/text/tree/6070ac32abcdd1670599eeb3ee4d513c8184a513).
|
||||
|
||||
## Go YAML v4
|
||||
|
||||
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0-rc.6` (transitive dependency).
|
||||
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0`.
|
||||
|
||||
Copyright 2006-2010 Kirill Simonov
|
||||
|
||||
@@ -98,7 +91,34 @@ SPDX-License-Identifier: MIT AND Apache-2.0
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||
[LICENSE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/LICENSE) |
|
||||
[NOTICE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/NOTICE).
|
||||
[LICENSE](https://github.com/yaml/go-yaml/blob/f2f9563503f11b7a60da0c532175dcc693ce8d5a/LICENSE) |
|
||||
[NOTICE](https://github.com/yaml/go-yaml/blob/f2f9563503f11b7a60da0c532175dcc693ce8d5a/NOTICE).
|
||||
|
||||
[Source](https://github.com/yaml/go-yaml/tree/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125).
|
||||
[Source](https://github.com/yaml/go-yaml/tree/f2f9563503f11b7a60da0c532175dcc693ce8d5a).
|
||||
|
||||
## Goldmark
|
||||
|
||||
Selected module: `github.com/yuin/goldmark/v2` at `v2.1.16` (maintainer-tool dependency).
|
||||
|
||||
Copyright (c) 2019 Yusuke Inuzuka
|
||||
|
||||
SPDX-License-Identifier: MIT
|
||||
|
||||
License and notices: [MIT](licenses/MIT.txt) |
|
||||
[LICENSE](https://github.com/yuin/goldmark/blob/c3f939122ba4dd6f80e05043ad195b82e77fcb41/LICENSE).
|
||||
|
||||
[Source](https://github.com/yuin/goldmark/tree/c3f939122ba4dd6f80e05043ad195b82e77fcb41).
|
||||
|
||||
## Go Modules
|
||||
|
||||
Selected module: `golang.org/x/mod` at `v0.43.0` (maintainer-tool dependency).
|
||||
|
||||
Copyright 2009 The Go Authors.
|
||||
|
||||
SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
||||
[LICENSE](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde/LICENSE) |
|
||||
[PATENTS](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde/PATENTS).
|
||||
|
||||
[Source](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde).
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
**Base Policy Version:** 1.3
|
||||
|
||||
**Last Updated**: 2026-09-27
|
||||
**Last Updated**: 2026-11-21
|
||||
|
||||
**Upstream Source:** [architecture.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
||||
|
||||
@@ -51,9 +51,10 @@ The [internal component overview](../internal/overview.md) owns the inventory
|
||||
of implemented packages and their concrete responsibilities. This policy owns
|
||||
the boundaries and dependency rules those components must preserve.
|
||||
|
||||
The internal engine composes caller-selected ConfigKit inputs once, then
|
||||
assembles the captured definitions, capacity manager, renderer, validator,
|
||||
outbound client, and use-case runner. The configuration adapter interprets and
|
||||
The facade either registers Promptkit rules in the application's ConfigKit load
|
||||
or performs one standalone load, then passes captured configuration to the
|
||||
internal engine. The internal engine assembles the captured definitions,
|
||||
capacity manager, renderer, validator, outbound client, and use-case runner. The configuration adapter interprets and
|
||||
validates only the final effective values, including presence, null, and
|
||||
resource context. It retains source history for explicit operator inspection,
|
||||
not for validation or a second resolution pass. The root facade translates
|
||||
@@ -128,7 +129,8 @@ consumer contract requires a public package.
|
||||
Each package must have one cohesive responsibility and a clear dependency
|
||||
direction. Internal packages must not expose their types merely to simplify
|
||||
wiring, and the public facade must not leak internal representations through
|
||||
exported signatures. Production packages under `internal/` must not import the
|
||||
exported signatures, except for the deliberately public configuration data model
|
||||
specified below. Production packages under `internal/` must not import the
|
||||
module root; the root facade may depend on internal implementation packages.
|
||||
New public packages require a durable consumer need that cannot be served
|
||||
cleanly by the root facade.
|
||||
@@ -136,12 +138,34 @@ cleanly by the root facade.
|
||||
The [internal component overview](../internal/overview.md) must be updated as
|
||||
packages are implemented or their responsibilities change.
|
||||
|
||||
### Shared Configuration Data Exception
|
||||
|
||||
[ADR 0002](../adr/0002-shared-configuration-data-model.md) permits one canonical
|
||||
set of public configuration settings and their nested data types in a small,
|
||||
data-only internal package, exposed through aliases in the root `promptkit`
|
||||
package. The facade and internal configuration adapter may use those same
|
||||
declarations directly. Their fields, tags, and nested types are intentional
|
||||
public contracts and receive the same API review as root-defined types.
|
||||
|
||||
The shared package owns data declarations, not loading, normalization,
|
||||
validation, or execution behavior. This exception does not authorize aliases
|
||||
for arbitrary internal implementation types or merging configuration settings
|
||||
with source-construction helpers, execution requests, runtime objects, or
|
||||
inspection reports that have different meanings. Internal packages still must
|
||||
not import the root facade. Runtime capture still owns the values it retains;
|
||||
shared declarations do not replace necessary ownership copies.
|
||||
|
||||
All other facade, dependency, and ownership rules remain in force. Concrete
|
||||
package placement and implementation status belong in the active plan and,
|
||||
once implemented, the internal component inventory.
|
||||
|
||||
## Exported API Discipline
|
||||
|
||||
Export the smallest contract required by real consumers. Exported declarations
|
||||
must have accurate GoDoc, stable semantics, and tests proportionate to their
|
||||
compatibility risk. Avoid speculative extension points, aliases for internal
|
||||
types, and public constructors that expose assembly details.
|
||||
types outside the shared configuration data exception, and public constructors
|
||||
that expose assembly details.
|
||||
|
||||
Once an exported API exists, its Go declaration and GoDoc own its exact public
|
||||
contract. Architecture documentation owns boundary rules, not a duplicate API
|
||||
|
||||
@@ -74,8 +74,8 @@ Successful publication needs no separate remote read-back or CI wait.
|
||||
|
||||
For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`.
|
||||
It is not a required step before publication. The upstream
|
||||
[candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/check.md)
|
||||
and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/cli.md)
|
||||
[candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/check.md)
|
||||
and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/cli.md)
|
||||
own shared checks and exit-status definitions.
|
||||
|
||||
## Deliver The Hosted Release
|
||||
@@ -97,14 +97,26 @@ For a later retry, use a clean checkout of the original tag with its recorded
|
||||
module files and supply the token there. In a separate checkout, for example:
|
||||
|
||||
```sh
|
||||
git switch --detach vMAJOR.MINOR.PATCH
|
||||
./scripts/releasekit.sh deliver vMAJOR.MINOR.PATCH
|
||||
(
|
||||
set -e
|
||||
release_version=vMAJOR.MINOR.PATCH
|
||||
release_tag_ref="refs/tags/$release_version"
|
||||
git fetch origin "$release_tag_ref:$release_tag_ref"
|
||||
release_tag_commit=$(git rev-parse --verify "$release_tag_ref^{commit}")
|
||||
git switch --detach "$release_tag_commit"
|
||||
release_worktree_status=$(git status --porcelain --untracked-files=all)
|
||||
if [ -n "$release_worktree_status" ]; then
|
||||
printf '%s\n' 'Delivery retry requires a clean tagged checkout.' >&2
|
||||
exit 1
|
||||
fi
|
||||
./scripts/releasekit.sh deliver "$release_version"
|
||||
)
|
||||
```
|
||||
|
||||
Resolve uncertain hosted writes before retrying the same immutable version.
|
||||
Matching content is accepted; conflicts are not overwritten. Do not rerun
|
||||
`publish` or move tags to repair delivery. Follow the upstream
|
||||
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/delivery.md)
|
||||
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/delivery.md)
|
||||
for result interpretation and recovery. These commands apply to tags containing
|
||||
this delivery configuration; older releases retain their original workflows.
|
||||
|
||||
@@ -131,13 +143,13 @@ and explicitly select the matching attempt:
|
||||
Discovery is local; reconciliation observes the original destination without
|
||||
rerunning validation, changing tags, or delivering hosted content. Neither is
|
||||
a routine gate after successful publication. Follow the upstream
|
||||
[recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/release.md#resolve-publication-failures)
|
||||
[recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/release.md#resolve-publication-failures)
|
||||
for outcomes, incomplete evidence, and older attempts without records.
|
||||
|
||||
Records are stored under Git's common directory by default and can contain
|
||||
credential-bearing endpoints. Keep them private, out of source control and
|
||||
shared logs; retain them while needed for recovery and remove them when that
|
||||
value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
|
||||
value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
|
||||
for storage and retention details.
|
||||
|
||||
Current absence does not prove a version was never published. Remove a retained
|
||||
|
||||
65
docs/releases/v0.18.0.md
Normal file
65
docs/releases/v0.18.0.md
Normal file
@@ -0,0 +1,65 @@
|
||||
# Promptkit v0.18.0
|
||||
|
||||
Configuration inspection now exposes Promptkit's recorded field normalization
|
||||
and derived queue-capacity defaults through ConfigKit v1.1.0. Engines also
|
||||
include a default-on OpenRouter configuration. The minimum supported Go
|
||||
version remains 1.27.0.
|
||||
|
||||
## Bundled OpenRouter Backend And Profiles
|
||||
|
||||
New engines include an embedded OpenRouter backend and seventeen root profiles:
|
||||
ten latest-family aliases, the auto and free routers, and five auto-router
|
||||
cost-tier variants. Select a profile explicitly in a prompt or request; the
|
||||
bundle does not supply an implicit execution profile or contact providers
|
||||
during construction. The backend uses `OPENROUTER_API_KEY` when available under
|
||||
the existing optional-credential rules.
|
||||
|
||||
The bundle is the lowest-priority underlay. Applications can replace its records
|
||||
or derive named variants through ordinary configuration, and external collections
|
||||
remain supported. Cost-tier variants inherit the effective `auto` profile, so
|
||||
replacing that parent changes their inherited target while retaining plugins.
|
||||
The [bundle reference](../formats.md#bundled-openrouter-configuration) owns the
|
||||
inventory and composition details.
|
||||
|
||||
Explicit smoke diagnostics now have OpenRouter authentication metadata and an
|
||||
inference fallback profile. They remain opt-in and may incur provider costs.
|
||||
Latest aliases and routers can change upstream; configured slugs do not pin
|
||||
the concrete model, pricing, or availability. See the
|
||||
[consumer guide](../consumers/pkg-promptkit.md#construct-an-engine) for selection
|
||||
and opt-out examples.
|
||||
|
||||
## Recorded Configuration Normalization
|
||||
|
||||
Inspection shows normalized backend and profile settings, prompt metadata, and
|
||||
configured message roles and cache controls, with field-level resolution
|
||||
history. Existing source evidence remains available, and unchanged content-file
|
||||
origins continue to select their resource context.
|
||||
|
||||
Configuration diffs compare normalized effective values, so spelling-only
|
||||
changes that normalize identically may no longer appear. Lists remain atomic in
|
||||
diffs. Inspection does not include backend-to-profile completion or request
|
||||
overrides; see the [configuration inspection guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration).
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
New engines install a bundled OpenRouter backend and root profiles by default.
|
||||
Inventory listings and inspection sources therefore change, and references to
|
||||
bundled IDs that previously failed may now resolve. Pass
|
||||
`promptkit.WithBundledOpenRouter(false)` to retain application-only behavior.
|
||||
Caller records can replace bundled definitions through ordinary composition;
|
||||
same-ID replacements replace complete records rather than merging selected fields.
|
||||
No configuration-format migration is required from v0.17.0.
|
||||
Consumers that display or compare configuration inspection results should account
|
||||
for normalized values and derived defaults replacing their earlier authored
|
||||
representations. Request-time overrides and caller-appended messages retain
|
||||
their existing processing.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.18.0
|
||||
```
|
||||
|
||||
See the [framework format reference](../formats.md) for current configuration
|
||||
semantics. Consumers upgrading from versions before v0.17.0 should also follow
|
||||
the [configuration and API migration guide](../consumers/configuration-migration.md).
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/docs/oss/attribution.md)
|
||||
80
docs/releases/v0.19.0.md
Normal file
80
docs/releases/v0.19.0.md
Normal file
@@ -0,0 +1,80 @@
|
||||
# Promptkit v0.19.0
|
||||
|
||||
This release completes two rounds of audit remediation, correcting error and
|
||||
diagnostic reporting and simplifying configuration, validation, and value
|
||||
ownership. It includes intentional compatibility changes for custom artifact
|
||||
readers, programmatic configuration sources, and schemas. The minimum supported
|
||||
Go version remains 1.27.0; ConfigKit remains at v1.1.0.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
Consumers upgrading from v0.18.0 should review these changes:
|
||||
|
||||
- A successful `ArtifactReader.Read` transfers ownership of the returned
|
||||
artifact and its mutable body storage to Promptkit. Custom readers must stop
|
||||
mutating returned values through retained aliases. Readers that reuse scratch
|
||||
buffers must return independent storage. Promptkit retains its defensive
|
||||
body copy, but that copy does not make concurrent buffer reuse safe. See the
|
||||
[extension guidance](../consumers/pkg-promptkit.md#extension-interfaces).
|
||||
- Provider maps passed through `ProfileSource` and `BackendSource` now use
|
||||
ConfigKit's native Mapping input forms directly: built-in scalars, `[]any`,
|
||||
and `map[string]any` recursively. Convert `json.Number` and typed containers
|
||||
explicitly to supported values, choosing numeric precision appropriate to
|
||||
the application. The former recursive compatibility conversion is removed.
|
||||
Runtime execution overrides retain their separate accepted-value behavior.
|
||||
- Schemas used for local validation and native structured output must be
|
||||
self-contained. Move externally referenced definitions into the root document
|
||||
and use internal references or explicit schema IDs. References requiring
|
||||
another file or network resource, or relying on the source filename as a base,
|
||||
fail during preparation. Promptkit does not bundle external schemas. See the
|
||||
[schema reference](../formats.md#schemas).
|
||||
|
||||
Prepared schemas no longer inherit the depth and node limits used when copying
|
||||
runtime provider values. Schemas previously rejected solely by those incidental
|
||||
limits may now be accepted; decoder/compiler checks and the self-contained
|
||||
schema requirement still apply.
|
||||
|
||||
No configuration-format migration is required from v0.18.0. After addressing
|
||||
any affected integrations:
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.19.0
|
||||
```
|
||||
|
||||
## Errors And Diagnostics
|
||||
|
||||
Public errors and terminal observations now preserve the category of the
|
||||
operation that failed even when an injected collaborator returns an error from
|
||||
another Promptkit operation. Underlying causes remain available through
|
||||
`errors.Is` and `errors.As`, without repeatedly duplicating nested cause trees.
|
||||
|
||||
Built-in generation preserves response-body read failures separately from
|
||||
malformed response content, and failures during request preparation are
|
||||
distinguished from dispatched attempts.
|
||||
|
||||
OpenRouter authentication diagnostics bound response recognition to 1 MiB.
|
||||
An oversized HTTP 200 response reports
|
||||
`SmokeTestReasonAuthenticationResponseLimitExceeded`, leaving credential
|
||||
acceptance unknown. Cancellation during the read remains an interruption.
|
||||
Consumers that enumerate diagnostic reasons should handle the new reason;
|
||||
see the [diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||
|
||||
## Internal Simplification And Ownership
|
||||
|
||||
Prompt lookup and rendering now consistently use captured configuration, and
|
||||
smoke syntax and schema checks share the prepared output-validation lifecycle.
|
||||
Unused routing defaults, alternate private workflows, and duplicate wire
|
||||
representations have been removed.
|
||||
|
||||
Provider-setting resolution, inspection, prepared details, and smoke reports
|
||||
avoid redundant validation and copying of already accepted values. Prepared
|
||||
execution shares one private redacted metadata snapshot while preserving
|
||||
independent caller-visible values. Provider values are detached before custom
|
||||
JSON encoders run, protecting retained metadata from encoder-side mutation.
|
||||
|
||||
Requirement analysis, namespace-filtered listings, and repair-feedback
|
||||
construction also avoid unnecessary traversal and temporary allocation. These
|
||||
changes preserve existing request isolation and execution behavior without
|
||||
introducing new performance guarantees.
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/docs/oss/attribution.md)
|
||||
55
docs/releases/v0.20.0.md
Normal file
55
docs/releases/v0.20.0.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# Promptkit v0.20.0
|
||||
|
||||
This minor release adds generically renderable resource inspection and typed
|
||||
prompt source construction. The Go consumer minimum remains 1.27.0 and
|
||||
ConfigKit remains at v1.1.0.
|
||||
|
||||
## Compatibility And Migration
|
||||
|
||||
`InspectProfile` now returns the resolved profile resource instead of the
|
||||
backend-completed `EffectiveModelParams`. Replace accesses to that field with
|
||||
the optional profile fields; use `Prepare` or `PreparedExecution.Details` for
|
||||
completed request settings. `InspectPrompt` now includes captured messages and
|
||||
uses optional pointers for presence-sensitive fields, including default profile,
|
||||
inputs, input metadata, and output settings. Check pointers before dereferencing.
|
||||
All three resource reports support ordinary JSON and YAML rendering without
|
||||
per-field adapters. See the [consumer guidance](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||
|
||||
After configuration composition and inheritance, profile null endpoint,
|
||||
API-key source, and whole provider map now permit backend defaults during
|
||||
execution as well as appearing absent in resource inspection. Use explicit
|
||||
empty strings to clear endpoint or API-key source. There is no replacement for
|
||||
profile-level suppression of an entire backend provider map; edit backend
|
||||
defaults through composition instead. The required-key rule still applies.
|
||||
See [defaults and overrides](../formats.md#defaults-and-overrides) for the
|
||||
migration example and retained ConfigKit null provenance.
|
||||
|
||||
## Resource Inspection And Sources
|
||||
|
||||
`InspectBackend` reports the resolved backend and configured diagnostics without
|
||||
contacting a provider or deriving diagnostic destinations. Prompt and profile
|
||||
inspection likewise report selected resources independently of request readiness;
|
||||
configuration provenance, requirement analysis, and preparation remain separate.
|
||||
`PromptSource` adds typed programmatic prompt definitions, and `BackendConfig`
|
||||
supports sibling inheritance. ConfigKit continues to own source composition,
|
||||
snapshotting, and relative message-file contexts.
|
||||
|
||||
The source helpers' [GoDoc](../../prompts.go) and
|
||||
[backend configuration API](../../backends.go) describe programmatic construction.
|
||||
|
||||
## Refinements
|
||||
|
||||
Package GoDoc and migration guidance now consistently describe the resource
|
||||
inspection contract, and the full package overview is visible in generated
|
||||
documentation. Prompt inspection avoids repeating the same metadata lookup for
|
||||
each declared input.
|
||||
|
||||
## Upgrade
|
||||
|
||||
After adapting affected inspection consumers and reviewing profile null usage:
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.20.0
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/docs/oss/attribution.md)
|
||||
34
docs/releases/v0.21.0.md
Normal file
34
docs/releases/v0.21.0.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# Promptkit v0.21.0
|
||||
|
||||
This minor release updates ConfigKit from v1.1.0 to v1.2.0, making main-relative
|
||||
companion imports available through Promptkit's existing configuration-source
|
||||
API. The Go consumer minimum remains 1.27.0.
|
||||
|
||||
## Main-Relative Companion Imports
|
||||
|
||||
Applications can bind a non-file underlay's declared imports to the selected
|
||||
main configuration's filesystem context. For example, an application can load
|
||||
prompt definitions from an adjacent `prompts/` directory without calculating
|
||||
that directory before loading the main configuration.
|
||||
|
||||
Imports remain explicitly selected by the caller; the upgrade does not enable
|
||||
automatic discovery. Imported prompt message files retain their source-local
|
||||
context. The [consumer example](../consumers/pkg-promptkit.md#load-companion-prompts-relative-to-the-main-configuration)
|
||||
shows collection mounting and YAML filename filtering. ConfigKit's
|
||||
[layered-loading contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#layered-loading)
|
||||
defines context selection and fallback behavior.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
Promptkit's exported methods and request shapes are unchanged. No configuration
|
||||
format migration is required from v0.20.0. Applications constructing ConfigKit
|
||||
`Input` or `LayeredOptions` with positional struct literals must switch to
|
||||
field-named literals because those upstream types gained fields. See the
|
||||
[ConfigKit release notes](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/releases/v1.2.0.md)
|
||||
for upstream changes.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.21.0
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/docs/oss/attribution.md)
|
||||
67
docs/releases/v0.22.0.md
Normal file
67
docs/releases/v0.22.0.md
Normal file
@@ -0,0 +1,67 @@
|
||||
# Promptkit v0.22.0
|
||||
|
||||
This minor release adds captured conversation exchanges, optional sequential
|
||||
coordination, and explicit execution-group summaries. Applications can continue
|
||||
or branch a conversation without reconstructing rendered history, while keeping
|
||||
response interpretation and workflow decisions in application code.
|
||||
|
||||
## Capture And Continue Conversations
|
||||
|
||||
Set `RunRequest.CaptureExchange` to capture a completed run's effective messages
|
||||
and final assistant content. Supply its `RunResult.Exchange` as `ContinueFrom`
|
||||
on a new request to continue it. Exchanges are reusable, support independent
|
||||
branches and different engines, and expose detached history through `Details()`.
|
||||
Prepared execution also supports capture of the exact snapshot it executes.
|
||||
|
||||
Internal structural-repair instructions and intermediate candidates are not
|
||||
conversation history. A completed result with failed structural validation can
|
||||
still be captured; an operational error returning no result creates no exchange.
|
||||
Follow-ups select their own profiles, output contracts, credentials, and sessions
|
||||
under ordinary request rules. See [capture and continuation guidance](../consumers/pkg-promptkit.md#capture-and-continue-a-conversation)
|
||||
and the [public exchange contract](../../exchange.go).
|
||||
|
||||
## Coordinate Application-Directed Turns
|
||||
|
||||
`Engine.Coordinate` advances one conversation branch using an application callback
|
||||
and an additional-turn budget. The application selects the next request or stops;
|
||||
Promptkit handles turn progression and cancellation. Each completed run releases
|
||||
backend admission before the callback, allowing assessment calls to use the same
|
||||
backend. Stop and budget exhaustion are control-flow outcomes, not judgments of
|
||||
response acceptance. Transport retries and structural repairs remain separate
|
||||
from the turn budget.
|
||||
|
||||
See the [coordination walkthrough](../consumers/pkg-promptkit.md#coordinate-application-directed-turns)
|
||||
and [coordinator GoDoc](../../coordinate.go) for callback and cancellation behavior.
|
||||
|
||||
## Collect Selected Execution Facts
|
||||
|
||||
`ExecutionGroup` collects explicitly selected execution observations, while
|
||||
`ComposeObservers` combines groups with existing diagnostic observers. Detached
|
||||
group snapshots report member identities, terminal coverage, attempt/retry/repair
|
||||
counts, and available token totals without counting a run more than once.
|
||||
Terminal coverage remains distinct from token-usage completeness; membership is
|
||||
not inferred from ancestry or nested calls.
|
||||
|
||||
See [execution-group guidance](../consumers/pkg-promptkit.md#collect-selected-execution-facts),
|
||||
the [group GoDoc](../../execution_group.go), and the updated
|
||||
[offline observation example](../../examples/go-library/observe/main.go).
|
||||
|
||||
## Efficiency And Compatibility
|
||||
|
||||
History assembly and inspection avoid intermediate full-history copies. Observer
|
||||
composition retains storage only for selected callbacks and avoids fan-out
|
||||
closures for zero or one selected callback.
|
||||
|
||||
Capture and coordination are opt-in; existing execution workflows and
|
||||
configuration formats need no migration from v0.21.0. The Go consumer minimum
|
||||
remains 1.27.0, and dependency versions are unchanged. Exported request, prepared,
|
||||
result, and observation structs gained fields; consumers using positional struct
|
||||
literals must switch to keyed literals. Continuation metadata adds `parent_run_id`
|
||||
to prepared/result JSON when applicable; the opaque exchange itself is excluded.
|
||||
Exchanges are in-process capabilities, not serialized restartable sessions.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.22.0
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/docs/oss/attribution.md)
|
||||
62
docs/releases/v0.23.0.md
Normal file
62
docs/releases/v0.23.0.md
Normal file
@@ -0,0 +1,62 @@
|
||||
# Promptkit v0.23.0
|
||||
|
||||
This minor release adds optional whole-operation timeouts and prepared-request
|
||||
notifications to ordinary and coordinated execution. Applications can bound
|
||||
each continuation independently and retain the request selected for execution
|
||||
without replacing Promptkit's turn loop or preparing the same request twice.
|
||||
|
||||
## Bound Individual Operations
|
||||
|
||||
`RunRequest.OperationTimeout` selects a fresh cooperative budget covering
|
||||
preparation and execution, including capacity waiting, retries, validation,
|
||||
structural repair, and prepared-request notification. It is independent of the
|
||||
provider generation timeout. Zero or negative values select no additional
|
||||
timeout; existing parent cancellation and earlier deadlines remain effective.
|
||||
|
||||
For split execution, `PrepareExecution` retains the duration without starting
|
||||
a timer. `RunPrepared` starts that budget after claiming the handle, so prior
|
||||
preparation and idle time do not consume it. Callers can still supply one shared
|
||||
deadline context to bound both calls together.
|
||||
|
||||
Local budget failures match both `ErrOperationTimeout` and
|
||||
`context.DeadlineExceeded`. Each coordinated turn uses its selected budget;
|
||||
expiry stops that coordination through the existing operational-error path
|
||||
without canceling its parent or sibling work. Completed outcomes are fixed
|
||||
before terminal observation and are not changed by a slow terminal observer.
|
||||
Cancellation does not forcibly interrupt callbacks or collaborators that ignore
|
||||
their context.
|
||||
|
||||
See the [execution-controls guide](../consumers/pkg-promptkit.md#bound-execution-and-retain-prepared-details)
|
||||
and the [public request contract](../../types.go) and
|
||||
[timeout error contract](../../engine.go) for exact behavior.
|
||||
|
||||
## Retain Prepared Details Before Generation
|
||||
|
||||
`RunRequest.OnPrepared` optionally receives a detached, credential-redacted
|
||||
`PreparedRun` and the operation context before first generation. It uses the
|
||||
actual prepared snapshot, including continuation history, without rereading
|
||||
inputs or rerendering. Applications may retain these details even if generation
|
||||
later fails without returning a result.
|
||||
|
||||
Notification is synchronous, at most once per logical execution, and runs while
|
||||
logical admission is held. It is not repeated for retries or structural repair.
|
||||
Preparation or admission failure can prevent delivery. The application owns
|
||||
diagnostic storage and disclosure decisions; ordinary observers remain
|
||||
metadata-only. See the [OnPrepared GoDoc](../../types.go) and updated
|
||||
[offline observation example](../../examples/go-library/observe/main.go).
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
Both controls are opt-in. Existing keyed requests and configuration files need
|
||||
no migration from v0.22.0. `Prepare` and `ValidateRequest` continue using only
|
||||
their supplied contexts and do not invoke `OnPrepared`. `RunRequest` gained
|
||||
fields; consumers using positional struct literals must switch to keyed
|
||||
literals. Prepared/result JSON representations are unchanged.
|
||||
|
||||
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.23.0
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/docs/oss/attribution.md)
|
||||
33
docs/releases/v0.23.1.md
Normal file
33
docs/releases/v0.23.1.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Promptkit v0.23.1
|
||||
|
||||
This patch release fixes construction-error wrapping so downstream consumers
|
||||
can classify failures from their underlying causes without treating the
|
||||
`ErrInvalidConfig` category as an independent failure.
|
||||
|
||||
## Fixed
|
||||
|
||||
`NewEngine` now uses single-cause wrapping for both configuration-loading and
|
||||
option-application failures. `errors.Is` still recognizes `ErrInvalidConfig`,
|
||||
and `errors.As` still exposes ConfigKit loading errors with their diagnostics
|
||||
and source context. Genuine joined causes, including cancellation, remain
|
||||
inspectable. Consumers retain responsibility for mapping those failures to
|
||||
application outcomes or exit statuses.
|
||||
|
||||
See the [constructor GoDoc](../../engine.go) and
|
||||
[error-handling guide](../consumers/pkg-promptkit.md#handle-errors) for the
|
||||
public error contracts and usage.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
No configuration or request changes are required when upgrading from v0.23.0.
|
||||
Consumers that inspect constructor error trees directly should account for
|
||||
the category no longer appearing as a separate sibling branch; use
|
||||
`errors.Is` and `errors.As` for category and typed-cause inspection.
|
||||
|
||||
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.23.1
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/docs/oss/attribution.md)
|
||||
35
docs/releases/v0.23.2.md
Normal file
35
docs/releases/v0.23.2.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# Promptkit v0.23.2
|
||||
|
||||
This patch release refreshes configuration, validation, and maintainer-tool
|
||||
dependencies. No Promptkit configuration or request API migration is required.
|
||||
|
||||
## Dependency Updates
|
||||
|
||||
- ConfigKit v1.2.0 → v1.2.1 and go-yaml v4.0.0-rc.6 → v4.0.0 move
|
||||
configuration loading to stable go-yaml, with ConfigKit's corresponding
|
||||
loader-plugin adaptation.
|
||||
- jsonschema v6.0.2 → v6.0.3 includes fixes to validation-location reporting,
|
||||
numeric edge cases, and email validation.
|
||||
- Go Text (`golang.org/x/text`) v0.14.0 → v0.44.0 refreshes the transitive
|
||||
dependency used by jsonschema.
|
||||
- ReleaseKit v0.9.0 → v0.10.0 updates maintainer tooling, selecting Goldmark
|
||||
v2.1.16 and Go Modules (`golang.org/x/mod`) v0.43.0. ReleaseKit's Go 1.27.0
|
||||
minimum is already satisfied by Promptkit's existing minimum.
|
||||
|
||||
Dependency notices and revision links are maintained in the
|
||||
[attribution document](../oss/attribution.md). Maintainers should use the
|
||||
updated [release runbook](../release.md); publication and hosted delivery
|
||||
remain separate operations.
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
The Go consumer minimum remains 1.27.0. Existing configuration, requests,
|
||||
and release commands require no changes from v0.23.1. Upstream validator
|
||||
fixes may affect validation results or diagnostic locations for edge cases.
|
||||
See the [consumer guide](../consumers/pkg-promptkit.md) for current usage.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.23.2
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/oss/attribution.md)
|
||||
36
docs/releases/v0.23.3.md
Normal file
36
docs/releases/v0.23.3.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Promptkit v0.23.3
|
||||
|
||||
This patch release corrects timeout attribution and preserves error identities
|
||||
in the built-in OpenAI-compatible HTTP client.
|
||||
|
||||
## Fixed
|
||||
|
||||
- Operation budgets expiring during an in-flight HTTP generation request now
|
||||
match both `ErrOperationTimeout` and `context.DeadlineExceeded`. Terminal
|
||||
observations report a deadline failure instead of a transport failure.
|
||||
- HTTP failures carrying custom caller cancellation or deadline causes retain
|
||||
both the custom cause and the standard context identity. Unrelated transport
|
||||
failures are not relabeled solely because a context has been canceled.
|
||||
- Cancellation while reading a successful response body preserves the original
|
||||
reader error alongside the context identity, without exposing its diagnostic
|
||||
text through the redacted error message.
|
||||
|
||||
Regression coverage exercises `Run` and `RunPrepared`, public error identities,
|
||||
and attempt and terminal classifications. See the [request and timeout
|
||||
contracts](../../types.go), [public error GoDoc](../../engine.go), and
|
||||
[error-handling guide](../consumers/pkg-promptkit.md#handle-errors).
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
No configuration or request API migration is required from v0.23.2. Consumers
|
||||
using `errors.Is` for cancellation or deadline handling now receive the
|
||||
correct identities on the affected paths; applications continue to own their
|
||||
status mapping and diagnostic presentation.
|
||||
|
||||
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.23.3
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/docs/oss/attribution.md)
|
||||
83
docs/releases/v0.24.0.md
Normal file
83
docs/releases/v0.24.0.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# Promptkit v0.24.0
|
||||
|
||||
This minor release adds application-owned configuration loading, prompt-owned
|
||||
schemas, and a prepared message-content hash. It also enables one repair attempt
|
||||
by default for enabled output validation. This is a breaking pre-v1 release;
|
||||
consumers upgrading from v0.23.x should review the migration steps below.
|
||||
|
||||
## Added And Changed
|
||||
|
||||
- Applications can embed `promptkit.Settings`, compose Promptkit's rules with
|
||||
`Configure`, load configuration once through ConfigKit, and pass the entire
|
||||
result to `NewFromConfig`. `NewEngine` remains the standalone convenience
|
||||
constructor. See the [shared-loading guide](../consumers/pkg-promptkit.md#reuse-an-application-owned-load)
|
||||
and [offline preparation example](../../examples/go-library/prepare/main.go).
|
||||
- Prompts can select inline, source-relative, or explicit-root JSON Schemas.
|
||||
Collections can supply their own schema root. Schema resources are loaded
|
||||
during preparation and retained by prepared execution. See the
|
||||
[schema reference](../formats.md#schemas).
|
||||
- Prepared results include `MessageContentHash` (`message_content_hash` in
|
||||
JSON), an opaque equality value for ordered message roles and exact text,
|
||||
independent of session identity, cache hints, and other metadata. The existing
|
||||
rendered-prompt hash algorithm is unchanged. See the
|
||||
[prepared-result GoDoc](../../types.go) and
|
||||
[preparation guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
||||
- Configuration inspection uses a native ConfigKit view focused on `/promptkit`
|
||||
and adds `HistoricalPaths`. It retains whole-load evidence and supports
|
||||
explicit queries outside that focus; comparison can include ancestor changes.
|
||||
See the [inspection guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration).
|
||||
|
||||
## Fixes And Efficiency
|
||||
|
||||
- Prompt-input inspection and requirements now retain effective metadata supplied
|
||||
by application defaults, including inherited input requirements.
|
||||
- Configuration capture avoids expanding inheritance provenance just to determine
|
||||
input-field presence, and shares immutable backend defaults across profiles
|
||||
without overrides while preserving defensive copies at public lookups.
|
||||
- Schema handling avoids copying inactive runtime schemas and unused slice
|
||||
capacity. Inline schemas are separate from provider extra-parameter copy limits.
|
||||
See the [schema reference](../formats.md#schemas) and
|
||||
[request GoDoc](../../types.go).
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
The [configuration migration guide](../consumers/configuration-migration.md#move-settings-under-promptkit)
|
||||
owns the full migration walkthrough. In particular:
|
||||
|
||||
- Move complete-document resources and `schema_root` under `promptkit`.
|
||||
Previous root-level resources are now application data, not Promptkit
|
||||
resources; IDs and references keep their existing names. `SettingsInput`
|
||||
mounts wrapperless fragments at `/promptkit`; `CollectionInput` mounts them
|
||||
at `/promptkit/collections/<namespace>`. Discovery helpers still produce
|
||||
root-mounted inputs, so adjust fragment destinations explicitly.
|
||||
- For an application-owned load, select `BundledOpenRouterUnderlay` before
|
||||
loading if you want the bundled catalog. Bundling remains default-on for
|
||||
`NewEngine`; it is not implicitly added by `NewFromConfig`.
|
||||
- Replace `RunRequest.Validation` values of type `*OutputContract` with
|
||||
`*OutputContractInput`; its `RepairAttempts` field is a pointer. Omitted or
|
||||
null budgets now resolve to one for `basic`, `json`, and `json_schema`, or
|
||||
zero for `none`. Set an explicit zero to retain single-pass behavior.
|
||||
Runtime replacements still replace the whole contract and resolve their
|
||||
defaults independently. A failed check may now incur another generation call,
|
||||
latency, and usage, without guaranteeing a valid result. See the
|
||||
[output contract](../formats.md#output-contract).
|
||||
- `PromptOutputInspection.RepairAttempts` is now a resolved integer rather than
|
||||
an optional pointer, and its JSON includes zero. `OutputContract` remains the
|
||||
resolved contract type used by prepared details. See the
|
||||
[inspection GoDoc](../../resource_inspection.go) and [contract GoDoc](../../types.go).
|
||||
- Schema paths require an explicitly configured applicable root; there is no
|
||||
implicit working-directory root. Review file references and make schemas
|
||||
self-contained as described in the [schema reference](../formats.md#schemas).
|
||||
- Promptkit no longer validates resource-version fields. Applications own version
|
||||
policy and sensitivity rules for their extension fields; see the
|
||||
[format reference](../formats.md).
|
||||
|
||||
Older prepared-result JSON without `message_content_hash` decodes with an empty
|
||||
value. Callers using positional `PreparedRun` literals may need to adjust them.
|
||||
ConfigKit is updated to v1.4.0; the Go consumer minimum remains 1.27.0.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.24.0
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/docs/oss/attribution.md)
|
||||
@@ -38,30 +38,17 @@ consumers.
|
||||
|
||||
## Ideas
|
||||
|
||||
### Field-Level Normalization With Preserved List Provenance
|
||||
### Schema Bundling
|
||||
|
||||
Reevaluate normalizing resource fields directly in ConfigKit's effective
|
||||
configuration after upstream ConfigKit supports recording changes within a
|
||||
list without replacing the source attribution of unchanged fields.
|
||||
Allow consumers to author reusable schemas across multiple files and package
|
||||
them into a single self-contained schema for local validation and provider
|
||||
submission. This could restore multi-file authoring convenience while retaining
|
||||
the [self-contained schema requirement](../formats.md#schemas).
|
||||
|
||||
This could let configuration inspection show PromptKit-normalized values and
|
||||
field-level change history while preserving the origins needed to resolve
|
||||
relative resources such as `content_file`. Assess whether that inspection
|
||||
benefit justifies changing the execution-owned normalization approach.
|
||||
|
||||
Include normalization of roles and cache-control values in configured messages
|
||||
in this reevaluation. This could make inspection reflect the values used for
|
||||
execution and reduce separate normalization paths. Upstream support would need
|
||||
to preserve untouched field origins within nested message lists as well as outer
|
||||
resource lists. PromptKit would continue owning the normalization rules;
|
||||
ConfigKit would record their effects and provenance. Caller-appended messages
|
||||
would still need request-time normalization because they do not pass through
|
||||
ConfigKit.
|
||||
Consider moving these transformations only if it improves inspection consistency
|
||||
and simplifies implementation without changing execution behavior.
|
||||
|
||||
Upstream support is the reconsideration trigger, not a dependency of the
|
||||
current ConfigKit migration or a commitment to adopt the enhancement.
|
||||
Bundling would need to preserve reference semantics, identifiers, anchors, and
|
||||
recursion. A complete bundle would not guarantee that a provider supports every
|
||||
schema feature. This is a possible future enhancement; implementation is not
|
||||
currently approved.
|
||||
|
||||
## Entry Format
|
||||
|
||||
|
||||
273
engine.go
273
engine.go
@@ -4,11 +4,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit/internal/backend"
|
||||
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
internalengine "go.maximumdirect.net/promptkit/internal/engine"
|
||||
"go.maximumdirect.net/promptkit/internal/observation"
|
||||
@@ -30,6 +32,8 @@ var (
|
||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||
// every configured profile source. It does not also match ErrProfileLoad.
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
// ErrBackendNotFound identifies a backend absent from resource inspection.
|
||||
ErrBackendNotFound = errors.New("backend not found")
|
||||
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||
// nor the selected prompt's default profile is present. Such an error also
|
||||
// matches ErrInvalidRequest.
|
||||
@@ -57,6 +61,13 @@ var (
|
||||
// A [CapacityError] reports the selected backend ID. It is not an invalid
|
||||
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||
// ErrOperationTimeout identifies expiration of the timeout selected by
|
||||
// RunRequest.OperationTimeout. Such errors also match context.DeadlineExceeded
|
||||
// and retain their underlying operation and collaborator identities.
|
||||
// Earlier parent cancellation or deadlines and provider-only deadlines do
|
||||
// not acquire this identity. A completed result remains completed even if
|
||||
// a terminal observer outlasts the selected budget.
|
||||
ErrOperationTimeout = errors.New("operation timeout exceeded")
|
||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||
// response. A built-in OpenAI-compatible non-2xx response is available as a
|
||||
// [GenerationError]. Errors returned by an injected LLMClient remain
|
||||
@@ -81,12 +92,12 @@ var (
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
||||
// Engine inspects resources, enumerates prompt requirements, and
|
||||
// prepares and runs Promptkit prompt requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
||||
// [Engine.InspectConfiguration],
|
||||
// [Engine.InspectPrompt],
|
||||
// [Engine.InspectPrompt], [Engine.InspectBackend],
|
||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
||||
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
||||
@@ -101,8 +112,9 @@ type Engine struct {
|
||||
}
|
||||
|
||||
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
||||
// highest-priority main input. Each underlay's inputs strictly compose within
|
||||
// their group; later underlays overlay earlier ones. A nil Main uses
|
||||
// highest-priority main input. By default a bundled OpenRouter underlay precedes
|
||||
// the caller underlays; [WithBundledOpenRouter] can disable it. Inputs within
|
||||
// each underlay strictly compose; later underlays overlay earlier ones. A nil Main uses
|
||||
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
||||
// may fail construction if invalid.
|
||||
type ConfigurationSources struct {
|
||||
@@ -115,9 +127,6 @@ type ConfigurationSources struct {
|
||||
type Config struct {
|
||||
// Sources combines maintained definitions with caller-selected inputs.
|
||||
Sources ConfigurationSources
|
||||
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
||||
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
||||
SchemaDir string
|
||||
// Timeout is the transport-wide safety cap for the built-in LLM client
|
||||
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
|
||||
// value selects the 10-minute default.
|
||||
@@ -137,7 +146,7 @@ type Config struct {
|
||||
// Option customizes engine construction.
|
||||
//
|
||||
// NewEngine applies options in argument order and ignores nil options. Within
|
||||
// each schema-source, model-client, and artifact-reader category, the last
|
||||
// each model-client and artifact-reader category, the last
|
||||
// non-nil valid option replaces earlier options. An invalid option fails
|
||||
// construction even if a later option would replace it.
|
||||
type Option interface {
|
||||
@@ -151,7 +160,18 @@ func (f optionFunc) apply(options *engineOptions) error {
|
||||
}
|
||||
|
||||
type engineOptions struct {
|
||||
internal internalengine.Options
|
||||
internal internalengine.Options
|
||||
disableBundledOpenRouter bool
|
||||
}
|
||||
|
||||
// WithBundledOpenRouter enables or disables the built-in OpenRouter underlay.
|
||||
// It is enabled by default for NewEngine; when repeated, the last option wins.
|
||||
// It has no effect on an already resolved NewFromConfig handoff.
|
||||
func WithBundledOpenRouter(enabled bool) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
options.disableBundledOpenRouter = !enabled
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||
@@ -187,64 +207,145 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
||||
})
|
||||
}
|
||||
|
||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
||||
//
|
||||
// Prompt schema_path values resolve within this source when schema validation
|
||||
// or structured output is requested. fsys must be non-nil and root must be
|
||||
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
return options.internal.SetSchemaFS(fsys, root)
|
||||
})
|
||||
}
|
||||
|
||||
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
||||
//
|
||||
// Prompt schema_path values refer to the file's base name. path must name an
|
||||
// existing non-directory file when NewEngine applies the option. This option
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
return options.internal.SetSchemaFile(path)
|
||||
})
|
||||
}
|
||||
|
||||
// NewEngine constructs an Engine from configuration and options.
|
||||
//
|
||||
// ConfigKit composes caller-selected underlays in their given order, followed
|
||||
// by the optional main input. Construction validates the final effective
|
||||
// by the optional main input, after the default-on bundled OpenRouter underlay.
|
||||
// The bundle adds a root backend and profiles to inventories, but no prompts
|
||||
// or implicit profile selection. Construction validates the final effective
|
||||
// backend, profile, and prompt definitions and captures referenced prompt
|
||||
// content. A missing prompt collection is allowed. Schema documents and
|
||||
// request artifacts remain runtime resources.
|
||||
//
|
||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||
// configuration or options, preserving a ConfigKit LoadError when loading
|
||||
// fails. Each constructed
|
||||
// Engine has independent backend-capacity pools. Construction does not perform
|
||||
// model requests or require credentials. A negative Config.TransientRetries
|
||||
// is invalid.
|
||||
// NewEngine returns an error matching ErrInvalidConfig through errors.Is for
|
||||
// invalid configuration or options. The error unwraps to the original
|
||||
// construction cause, preserving a ConfigKit LoadError when loading fails.
|
||||
// Each constructed Engine has independent backend-capacity pools. Construction
|
||||
// does not perform model requests or require credentials. A negative
|
||||
// Config.TransientRetries is invalid.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
options, err := applyEngineOptions(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runtime := RuntimeConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: cfg.TransientRetries}
|
||||
if _, err := domain.ResolveTransientRetries(2, runtime.TransientRetries); err != nil {
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||
}
|
||||
underlays := append([]configkit.Underlay(nil), cfg.Sources.Underlays...)
|
||||
if !options.disableBundledOpenRouter {
|
||||
underlays = append([]configkit.Underlay{BundledOpenRouterUnderlay()}, underlays...)
|
||||
}
|
||||
main := configkit.Input{Source: configkit.NoSource()}
|
||||
if cfg.Sources.Main != nil {
|
||||
main = *cfg.Sources.Main
|
||||
}
|
||||
definition := Configure(configkit.Definition[standaloneConfiguration]{}, func(value *standaloneConfiguration) *Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[standaloneConfiguration]{Underlays: underlays, Definition: definition})
|
||||
if err != nil {
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||
}
|
||||
return newFromConfig(result, runtime, options)
|
||||
}
|
||||
|
||||
type standaloneConfiguration struct {
|
||||
PromptKit Settings `yaml:"promptkit"`
|
||||
Application map[string]any `yaml:",inline"`
|
||||
}
|
||||
|
||||
// Configuration is the completed application-owned configuration evidence
|
||||
// needed to capture PromptKit resources without loading configuration again.
|
||||
type Configuration interface {
|
||||
Value(configkit.Path) (any, bool)
|
||||
InspectSubtree(configkit.Path) (configkit.Inspection, bool)
|
||||
Diagnostics() []configkit.Diagnostic
|
||||
}
|
||||
|
||||
// RuntimeConfig selects runtime-only client and retry settings for NewFromConfig.
|
||||
// Its fields have the same semantics as the corresponding Config fields.
|
||||
type RuntimeConfig struct {
|
||||
// Timeout is the built-in client's transport timeout fallback.
|
||||
Timeout time.Duration
|
||||
// HTTPClient is cloned for the built-in client and ignored by injected clients.
|
||||
HTTPClient *http.Client
|
||||
// TransientRetries selects additional built-in attempts; nil defaults to two.
|
||||
TransientRetries *int
|
||||
}
|
||||
|
||||
// NewFromConfig captures resources from an application-owned completed load.
|
||||
// The application must register PromptKit rules during that load, using Configure
|
||||
// or equivalent explicit composition. Missing or nil selected settings contribute
|
||||
// no resources; Settings and *Settings are accepted, while other nonnil types,
|
||||
// nil results, and unavailable inspection views fail with ErrInvalidConfig.
|
||||
// Construction consumes resolved settings while inspection retains the load's
|
||||
// frozen evidence. Callers should keep those consistent at handoff. After
|
||||
// construction, later settings mutations do not alter the engine. This method
|
||||
// does not load sources, rerun callbacks, or select bundled underlays.
|
||||
func NewFromConfig(result Configuration, cfg RuntimeConfig, opts ...Option) (*Engine, error) {
|
||||
options, err := applyEngineOptions(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newFromConfig(result, cfg, options)
|
||||
}
|
||||
|
||||
func applyEngineOptions(opts []Option) (engineOptions, error) {
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
if opt == nil {
|
||||
continue
|
||||
}
|
||||
if err := opt.apply(&options); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
||||
return engineOptions{}, newPublicOperationError(ErrInvalidConfig, err)
|
||||
}
|
||||
}
|
||||
return options, nil
|
||||
}
|
||||
|
||||
assembled, err := internalengine.New(
|
||||
internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
|
||||
options.internal,
|
||||
)
|
||||
func newFromConfig(result Configuration, cfg RuntimeConfig, options engineOptions) (*Engine, error) {
|
||||
if result == nil || isNilConfiguration(result) {
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, errors.New("configuration result is nil"))
|
||||
}
|
||||
if _, err := domain.ResolveTransientRetries(2, cfg.TransientRetries); err != nil {
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||
}
|
||||
value, available := result.Value(configkit.Keys("promptkit"))
|
||||
var selected *Settings
|
||||
if available && value != nil {
|
||||
switch section := value.(type) {
|
||||
case Settings:
|
||||
selected = §ion
|
||||
case *Settings:
|
||||
selected = section
|
||||
default:
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, fmt.Errorf("promptkit section has incompatible type %T", value))
|
||||
}
|
||||
}
|
||||
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
|
||||
if !ok {
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, errors.New("configuration inspection is unavailable"))
|
||||
}
|
||||
snapshot, err := configuration.Capture(selected, view, result.Diagnostics())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||
}
|
||||
assembled, err := internalengine.New(snapshot, internalengine.Config{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)}, options.internal)
|
||||
if err != nil {
|
||||
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||
}
|
||||
return &Engine{internal: assembled}, nil
|
||||
}
|
||||
|
||||
func isNilConfiguration(result Configuration) bool {
|
||||
value := reflect.ValueOf(result)
|
||||
switch value.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
|
||||
return value.IsNil()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// InspectPrompt resolves one prompt ID without selecting a profile or starting
|
||||
// execution work.
|
||||
//
|
||||
@@ -254,16 +355,17 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
// qualified IDs pass through.
|
||||
//
|
||||
// A successful result proves that the selected definition and any referenced
|
||||
// message content files were structurally loaded. Inputs are returned in
|
||||
// definition order. DefaultProfileID is declared metadata only and is not
|
||||
// message content files were structurally loaded. Inputs and captured,
|
||||
// unrendered messages are returned in definition order. DefaultProfileID is
|
||||
// declared metadata only and is not
|
||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||
// Schema path when declared but without loading or compiling that schema.
|
||||
// Schema selection when declared but without loading or compiling schema files.
|
||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||
// the selected captured definition; its spelling, length,
|
||||
// encoding, algorithm, and security properties are not contracts.
|
||||
//
|
||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
||||
// rendered messages, or execution settings. It does not resolve a profile or
|
||||
// This method does not return parsed templates, filesystem capabilities,
|
||||
// schema file contents, rendered messages, or execution settings. It does not resolve a profile or
|
||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||
// contact a provider, or generate model output. The returned PromptInspection
|
||||
// and its input slice are caller-owned. Inspection does not reload configuration
|
||||
@@ -286,21 +388,19 @@ func (e *Engine) InspectPrompt(
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainPromptInspection(inspection), nil
|
||||
return fromPromptResource(inspection), nil
|
||||
}
|
||||
|
||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||
// starting execution work.
|
||||
//
|
||||
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
||||
// in the captured effective configuration. It applies the framework timeout
|
||||
// baseline, selected
|
||||
// backend, and then selected profile to EffectiveModelParams without a request
|
||||
// override. BackendID is empty for an endpoint-only profile.
|
||||
// in the captured effective configuration. It reports the resolved profile
|
||||
// resource without backend completion, execution defaults, or request overrides.
|
||||
//
|
||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
||||
// value. APIKeyRequired instead reports a direct credential requirement and is
|
||||
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
||||
// APIKeyEnv in the returned resource is an environment-variable name, never its
|
||||
// value. APIKeyRequired, when present and true, reports that a credential is
|
||||
// required; an explicit profile environment source can also be present. InspectProfile neither derives
|
||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||
// absent or blank named environment variable is not an error.
|
||||
//
|
||||
@@ -324,7 +424,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromDomainProfileInspection(inspection), nil
|
||||
return fromProfileResource(inspection), nil
|
||||
}
|
||||
|
||||
// InspectBackend selects a resolved backend by exact ID from the captured
|
||||
// configuration, qualifying a slashless ID with the backend namespace. It
|
||||
// reports configured resource fields and diagnostics, not credential values,
|
||||
// derived diagnostic destinations, or runtime capacity occupancy. References
|
||||
// are not resolved and no network request is made. The result and its mutable
|
||||
// values are caller-owned. Source changes require a new engine.
|
||||
//
|
||||
// An invalid engine matches ErrInvalidConfig, an invalid ID matches
|
||||
// ErrInvalidRequest, and an absent backend matches ErrBackendNotFound (not
|
||||
// ErrInvalidRequest). Cancellation remains detectable via errors.Is. No
|
||||
// partial result is returned on error.
|
||||
func (e *Engine) InspectBackend(ctx context.Context, backendID string) (*BackendInspection, error) {
|
||||
if e == nil || e.internal == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
}
|
||||
inspection, err := e.internal.InspectBackend(ctx, backendID)
|
||||
if err != nil {
|
||||
if errors.Is(err, backend.ErrBackendNotFound) {
|
||||
return nil, fmt.Errorf("%w: %w", ErrBackendNotFound, err)
|
||||
}
|
||||
return nil, mapPublicError(err)
|
||||
}
|
||||
return fromBackendResource(inspection), nil
|
||||
}
|
||||
|
||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||
@@ -346,6 +471,7 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
||||
// reported in the applicable operation category; no general errors.Is
|
||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
||||
// RunRequest.OperationTimeout and RunRequest.OnPrepared have no effect here.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
prepared, err := e.prepareRequest(ctx, req)
|
||||
if err != nil {
|
||||
@@ -367,6 +493,7 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
|
||||
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
|
||||
// when execution must use the snapshot produced by preparation.
|
||||
// Execution-only controls do not start a timer or invoke a notification here.
|
||||
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
|
||||
_, err := e.prepareRequest(ctx, req)
|
||||
return err
|
||||
@@ -409,6 +536,8 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr
|
||||
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
||||
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
||||
// privately for RunPrepared; preparation itself emits no events.
|
||||
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
|
||||
// starting a timer or invoking the notification during preparation.
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
@@ -439,6 +568,9 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
// returns the final failed validation result with cumulative usage and actual
|
||||
// repair attempts. An inability to generate or validate returns an error and
|
||||
// no partial result.
|
||||
// RunRequest.OperationTimeout selects a cooperative whole-operation deadline;
|
||||
// RunRequest.OnPrepared selects conditional notification before generation.
|
||||
// The outcome and timeout attribution are fixed before terminal observation.
|
||||
//
|
||||
// The built-in client retries eligible provider statuses using the effective
|
||||
// Config or request budget. With R additional transient attempts and P repair
|
||||
@@ -470,18 +602,30 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
// result on error.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
|
||||
defer scope.Close()
|
||||
ctx = scope.Context
|
||||
if req.Observer == nil {
|
||||
return e.run(ctx, req)
|
||||
result, err := e.run(ctx, req)
|
||||
err = mapPublicError(scope.Finalize(err))
|
||||
scope.Close()
|
||||
return result, err
|
||||
}
|
||||
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
||||
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
|
||||
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
|
||||
}
|
||||
recorder.Start()
|
||||
if idErr != nil {
|
||||
err := fmt.Errorf("failed to create run id: %w", idErr)
|
||||
err := mapPublicError(scope.Finalize(fmt.Errorf("failed to create run id: %w", idErr)))
|
||||
scope.Close()
|
||||
terminal := observedFinish(nil, err)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure})
|
||||
return nil, err
|
||||
}
|
||||
result, err := e.run(observation.WithRecorder(ctx, recorder), req)
|
||||
err = mapPublicError(scope.Finalize(err))
|
||||
scope.Close()
|
||||
terminal := observedFinish(result, err)
|
||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure,
|
||||
StatusCode: terminal.StatusCode, RetriesExhausted: terminal.RetriesExhausted})
|
||||
@@ -500,7 +644,7 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
|
||||
result, err := e.internal.Run(ctx, domainReq)
|
||||
if err != nil {
|
||||
return nil, mapPublicError(err)
|
||||
return nil, err
|
||||
}
|
||||
return fromDomainRunResult(result), nil
|
||||
}
|
||||
@@ -544,6 +688,9 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
|
||||
// Callbacks run outside the handle and capacity locks; panics propagate and
|
||||
// do not promise a terminal event.
|
||||
// The retained RunRequest.OperationTimeout starts after claim; the retained
|
||||
// RunRequest.OnPrepared is notified only if execution reaches its prepared
|
||||
// point. Final timeout attribution precedes terminal observation.
|
||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
if e == nil || e.internal == nil {
|
||||
|
||||
40
errors.go
40
errors.go
@@ -13,10 +13,48 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
// publicOperationError keeps classification separate from inspectable causes.
|
||||
// Cache wording so formatting nested operations does not repeatedly rebuild it.
|
||||
type publicOperationError struct {
|
||||
owner error
|
||||
cause error
|
||||
message string
|
||||
}
|
||||
|
||||
func newPublicOperationError(owner, cause error) *publicOperationError {
|
||||
return &publicOperationError{owner: owner, cause: cause, message: fmt.Sprintf("%v: %v", owner, cause)}
|
||||
}
|
||||
|
||||
func (e *publicOperationError) Error() string { return e.message }
|
||||
func (e *publicOperationError) Unwrap() error { return e.cause }
|
||||
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
|
||||
|
||||
func mapPublicError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var operation *usecase.OperationError
|
||||
if errors.As(err, &operation) && operation != nil {
|
||||
owner := publicErrorFor(operation.Category)
|
||||
if owner != nil {
|
||||
// Project the underlying specialized error separately, retaining both
|
||||
// the outer operation and the original cause for Is/As callers.
|
||||
mapped := mapPublicError(operation.Cause)
|
||||
if errors.Is(operation.Cause, llm.ErrTransport) {
|
||||
mapped = errors.Join(ErrTransport, mapped)
|
||||
}
|
||||
if errors.Is(operation.Cause, llm.ErrMalformedResponse) {
|
||||
mapped = errors.Join(ErrMalformedResponse, mapped)
|
||||
}
|
||||
// The projection already retains ordinary causes; adding the original
|
||||
// again would duplicate its entire subtree at each enclosing operation.
|
||||
return newPublicOperationError(owner, mapped)
|
||||
}
|
||||
}
|
||||
return mapUnownedPublicError(err)
|
||||
}
|
||||
|
||||
func mapUnownedPublicError(err error) error {
|
||||
var internalCapacityError *usecase.CapacityError
|
||||
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||
@@ -62,6 +100,8 @@ func mapPublicError(err error) error {
|
||||
|
||||
func publicErrorFor(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, usecase.ErrOperationTimeout):
|
||||
return ErrOperationTimeout
|
||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||
return ErrPromptNotFound
|
||||
case errors.Is(err, profile.ErrProfileNotFound):
|
||||
|
||||
@@ -10,6 +10,25 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||
)
|
||||
|
||||
func TestConstructionErrorPreservesJoinedCause(t *testing.T) {
|
||||
operational := errors.New("independent operational failure")
|
||||
cause := errors.Join(operational, context.Canceled)
|
||||
engine, err := NewEngine(Config{}, optionFunc(func(*engineOptions) error {
|
||||
return cause
|
||||
}))
|
||||
if engine != nil || err == nil {
|
||||
t.Fatalf("construction = (%#v, %v), want failure", engine, err)
|
||||
}
|
||||
if unwrapped := errors.Unwrap(err); unwrapped != cause {
|
||||
t.Fatalf("construction cause = %v, want the unchanged joined cause", unwrapped)
|
||||
}
|
||||
for _, target := range []error{ErrInvalidConfig, operational, context.Canceled} {
|
||||
if !errors.Is(err, target) {
|
||||
t.Errorf("construction error = %v, want identity %v", err, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
||||
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)
|
||||
|
||||
@@ -80,3 +99,12 @@ func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
||||
t.Fatalf("request encoding category = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapPublicErrorRetainsOwnerWithSpecializedCapacity(t *testing.T) {
|
||||
internal := &usecase.CapacityError{BackendID: "limited"}
|
||||
err := mapPublicError(&usecase.OperationError{Category: usecase.ErrLLMGenerate, Cause: internal})
|
||||
var capacity *CapacityError
|
||||
if !errors.Is(err, ErrLLMGenerate) || !errors.As(err, &capacity) || capacity.BackendID != "limited" || !errors.Is(err, ErrCapacityExceeded) {
|
||||
t.Fatalf("mapped owned capacity = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
@@ -69,12 +70,18 @@ func (l *eventLog) terminal() (promptkit.ExecutionEvent, error) {
|
||||
}
|
||||
|
||||
type summary struct {
|
||||
ProposalAttempts uint64 `json:"proposal_attempts"`
|
||||
ProposalTransientRetries uint64 `json:"proposal_transient_retries"`
|
||||
ProposalUsageComplete bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
||||
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
||||
AssessmentOutcome string `json:"assessment_outcome"`
|
||||
ProposalAttempts uint64 `json:"proposal_attempts"`
|
||||
ProposalTransientRetries uint64 `json:"proposal_transient_retries"`
|
||||
ProposalUsageComplete bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
||||
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
||||
AssessmentOutcome string `json:"assessment_outcome"`
|
||||
CoordinationOutcome promptkit.CoordinationOutcome `json:"coordination_outcome"`
|
||||
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts uint64 `json:"group_attempts"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
}
|
||||
|
||||
func run() (summary, error) {
|
||||
@@ -83,29 +90,35 @@ func run() (summary, error) {
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
for i := range inputs {
|
||||
inputs[i].Destination = configkit.Keys("promptkit")
|
||||
}
|
||||
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
||||
filesystem := configkit.NewFilesystem("example schemas", assets)
|
||||
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||
backend.ID = backendID
|
||||
inputs = append(inputs,
|
||||
configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
|
||||
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||
backend,
|
||||
})},
|
||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
|
||||
})),
|
||||
promptkit.SettingsInput(promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
|
||||
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
|
||||
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
|
||||
})},
|
||||
})),
|
||||
)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
||||
HTTPClient: &http.Client{Transport: transport},
|
||||
}, promptkit.WithSchemaFS(assets, "schemas"))
|
||||
}, promptkit.WithBundledOpenRouter(false)) // Keep the example's configuration self-contained.
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
|
||||
var proposalLog eventLog
|
||||
var executions promptkit.ExecutionGroup
|
||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, Observer: proposalLog.record,
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
@@ -118,35 +131,56 @@ func run() (summary, error) {
|
||||
return summary{}, err
|
||||
}
|
||||
var assessmentLog eventLog
|
||||
assessment, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(proposal.RawOutput)}, Observer: assessmentLog.record,
|
||||
var preparedDetails promptkit.PreparedRun
|
||||
decision := "rejected"
|
||||
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
|
||||
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
|
||||
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result == nil {
|
||||
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
|
||||
}
|
||||
if !turn.Result.Validation.IsValid {
|
||||
return nil, fmt.Errorf("follow-up failed structural validation")
|
||||
}
|
||||
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !assessment.Validation.IsValid {
|
||||
return nil, fmt.Errorf("assessment failed structural validation")
|
||||
}
|
||||
var proposalValue struct {
|
||||
Risk string `json:"risk"`
|
||||
}
|
||||
var assessmentValue struct {
|
||||
Recommendation string `json:"recommendation"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(turn.Result.RawOutput), &proposalValue); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
||||
decision = "accepted"
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if !assessment.Validation.IsValid {
|
||||
return summary{}, fmt.Errorf("assessment failed structural validation")
|
||||
exhausted, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: proposal.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
return &promptkit.RunRequest{PromptID: "example.proposal"}, nil
|
||||
}})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
assessmentTerminal, err := assessmentLog.terminal()
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
var proposalValue struct {
|
||||
Risk string `json:"risk"`
|
||||
}
|
||||
var assessmentValue struct {
|
||||
Recommendation string `json:"recommendation"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(proposal.RawOutput), &proposalValue); err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
decision := "rejected"
|
||||
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
||||
decision = "accepted"
|
||||
}
|
||||
group := executions.Snapshot()
|
||||
return summary{
|
||||
ProposalAttempts: proposalTerminal.Attempts,
|
||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||
@@ -154,6 +188,12 @@ func run() (summary, error) {
|
||||
AssessmentAttempts: assessmentTerminal.Attempts,
|
||||
AssessmentTransientRetries: assessmentTerminal.TransientRetries,
|
||||
AssessmentOutcome: decision,
|
||||
CoordinationOutcome: coordination.Outcome,
|
||||
ExhaustedOutcome: exhausted.Outcome,
|
||||
GroupMembers: len(group.Members),
|
||||
GroupAttempts: group.Attempts,
|
||||
PreparedMessages: len(preparedDetails.Messages),
|
||||
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -12,19 +12,25 @@ func TestObserveExample(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
ProposalAttempts *int `json:"proposal_attempts"`
|
||||
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||
Outcome string `json:"assessment_outcome"`
|
||||
ProposalAttempts *int `json:"proposal_attempts"`
|
||||
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||
Outcome string `json:"assessment_outcome"`
|
||||
CoordinationOutcome string `json:"coordination_outcome"`
|
||||
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts int `json:"group_attempts"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
}
|
||||
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.ProposalAttempts == nil || *got.ProposalAttempts != 2 || got.ProposalRetries == nil || *got.ProposalRetries != 1 ||
|
||||
got.UsageComplete == nil || *got.UsageComplete || got.AssessmentAttempts == nil || *got.AssessmentAttempts != 1 ||
|
||||
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" {
|
||||
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
|
||||
t.Fatalf("unexpected observation summary: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,4 +14,12 @@ prompts:
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: assessment.schema.json
|
||||
repair_attempts: 0
|
||||
schema:
|
||||
$schema: https://json-schema.org/draft/2020-12/schema
|
||||
type: object
|
||||
required: [recommendation, confidence]
|
||||
properties:
|
||||
recommendation: {type: string}
|
||||
confidence: {type: number}
|
||||
additionalProperties: false
|
||||
|
||||
@@ -14,4 +14,5 @@ prompts:
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
repair_attempts: 0
|
||||
schema_path: proposal.schema.json
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["recommendation", "confidence"],
|
||||
"properties": {
|
||||
"recommendation": {"type": "string"},
|
||||
"confidence": {"type": "number"}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
version: "2"
|
||||
config:
|
||||
imports:
|
||||
- path: fragments/resources.yaml
|
||||
|
||||
@@ -13,15 +13,16 @@ import (
|
||||
)
|
||||
|
||||
type summary struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
SelectedProfile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
ProfileTimeout int `json:"profile_timeout_seconds"`
|
||||
Namespaces []string `json:"namespaces"`
|
||||
Collections []collectionSummary `json:"collections"`
|
||||
ScopedPrompts []string `json:"scoped_prompts"`
|
||||
ReviewPrompts []string `json:"review_prompts"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
SelectedProfile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
MessageContentHash string `json:"message_content_hash"`
|
||||
ProfileTimeout int `json:"profile_timeout_seconds"`
|
||||
Namespaces []string `json:"namespaces"`
|
||||
Collections []collectionSummary `json:"collections"`
|
||||
ScopedPrompts []string `json:"scoped_prompts"`
|
||||
ReviewPrompts []string `json:"review_prompts"`
|
||||
}
|
||||
|
||||
type collectionSummary struct {
|
||||
@@ -29,6 +30,10 @@ type collectionSummary struct {
|
||||
ReleaseVersion *string `json:"release_version"`
|
||||
}
|
||||
|
||||
type applicationSettings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(context.Background(), "examples/go-library/prepare", os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
@@ -58,12 +63,18 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
definition := promptkit.Configure(configkit.Definition[applicationSettings]{}, func(value *applicationSettings) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[applicationSettings]{
|
||||
Definition: definition,
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{support, review}},
|
||||
{Inputs: []configkit.Input{override}},
|
||||
},
|
||||
}})
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -77,7 +88,7 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
|
||||
if _, found := view.Explain(configkit.Keys("promptkit", "collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
|
||||
return fmt.Errorf("collection profile override was not found")
|
||||
}
|
||||
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||
@@ -110,10 +121,10 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if profile.EffectiveModelParams.TimeoutSeconds != 30 {
|
||||
if profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 30 {
|
||||
return fmt.Errorf(
|
||||
"later profile override was not applied: timeout is %d seconds",
|
||||
profile.EffectiveModelParams.TimeoutSeconds,
|
||||
"later profile override was not applied: timeout is %v",
|
||||
profile.TimeoutSeconds,
|
||||
)
|
||||
}
|
||||
namespaces, err := scoped.ListNamespaces(ctx)
|
||||
@@ -152,14 +163,15 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||
encoder := json.NewEncoder(out)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(summary{
|
||||
PromptID: prepared.PromptID,
|
||||
SelectedProfile: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
ProfileTimeout: profile.EffectiveModelParams.TimeoutSeconds,
|
||||
Namespaces: namespaces,
|
||||
Collections: collectionSummaries,
|
||||
ScopedPrompts: scopedPromptIDs,
|
||||
ReviewPrompts: reviewPromptIDs,
|
||||
PromptID: prepared.PromptID,
|
||||
SelectedProfile: prepared.SelectedProfileID,
|
||||
Model: prepared.EffectiveModelParams.Model,
|
||||
MessageCount: len(prepared.Messages),
|
||||
MessageContentHash: prepared.MessageContentHash,
|
||||
ProfileTimeout: *profile.TimeoutSeconds,
|
||||
Namespaces: namespaces,
|
||||
Collections: collectionSummaries,
|
||||
ScopedPrompts: scopedPromptIDs,
|
||||
ReviewPrompts: reviewPromptIDs,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,21 +13,22 @@ func TestPrepareExample(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
Profile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
Timeout int `json:"profile_timeout_seconds"`
|
||||
Namespaces []string `json:"namespaces"`
|
||||
Collections []map[string]json.RawMessage `json:"collections"`
|
||||
ScopedPrompts []string `json:"scoped_prompts"`
|
||||
ReviewPrompts []string `json:"review_prompts"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
Profile string `json:"selected_profile"`
|
||||
Model string `json:"model"`
|
||||
MessageCount int `json:"message_count"`
|
||||
MessageContentHash string `json:"message_content_hash"`
|
||||
Timeout int `json:"profile_timeout_seconds"`
|
||||
Namespaces []string `json:"namespaces"`
|
||||
Collections []map[string]json.RawMessage `json:"collections"`
|
||||
ScopedPrompts []string `json:"scoped_prompts"`
|
||||
ReviewPrompts []string `json:"review_prompts"`
|
||||
}
|
||||
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.PromptID != "support/prepare" || got.Profile != "support/fast" || got.Model != "offline-model" ||
|
||||
got.MessageCount <= 0 || got.Timeout != 30 || !reflect.DeepEqual(got.Namespaces, []string{"review", "support"}) ||
|
||||
got.MessageCount <= 0 || got.MessageContentHash == "" || got.Timeout != 30 || !reflect.DeepEqual(got.Namespaces, []string{"review", "support"}) ||
|
||||
!reflect.DeepEqual(got.ScopedPrompts, []string{"support/prepare"}) || !reflect.DeepEqual(got.ReviewPrompts, []string{"review/check"}) || len(got.Collections) != 2 {
|
||||
t.Fatalf("unexpected preparation summary: %+v", got)
|
||||
}
|
||||
|
||||
@@ -50,11 +50,11 @@ func main() {
|
||||
func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
|
||||
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml"))},
|
||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
||||
}))}}},
|
||||
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml")), Destination: configkit.Keys("promptkit")},
|
||||
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(deterministicClient{})) // Use only this example's offline configuration.
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
47
exchange.go
Normal file
47
exchange.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package promptkit
|
||||
|
||||
import "go.maximumdirect.net/promptkit/internal/domain"
|
||||
|
||||
const exchangeString = "promptkit.Exchange{opaque}"
|
||||
|
||||
// Exchange is an immutable, reusable in-process snapshot of a completed
|
||||
// conversation. It is not a persistence or restart format. Different engines
|
||||
// can continue it, and independent branches do not alter one another.
|
||||
//
|
||||
// History preserves the effective request's message content, roles, and cache
|
||||
// metadata, followed by the exact final [RunResult.RawOutput] as an assistant
|
||||
// message. Intermediate structural-repair candidates and corrective instructions
|
||||
// are excluded. A continuation extends that logical history according to
|
||||
// [RunRequest.ContinueFrom], without changing its source exchange.
|
||||
type Exchange struct{ internal *domain.Exchange }
|
||||
|
||||
// ExchangeDetails contains a detached copy of an exchange's logical history.
|
||||
type ExchangeDetails struct {
|
||||
// RunID identifies the execution that produced this exchange.
|
||||
RunID string `json:"run_id"`
|
||||
// ParentRunID identifies the immediate source execution, or is empty.
|
||||
ParentRunID string `json:"parent_run_id,omitempty"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// Details returns detached history, or zero details for a nil or empty handle.
|
||||
// Messages include caller and model content and require appropriate handling.
|
||||
// Inspection uses only retained state, without rendering, filesystem reads, or
|
||||
// provider calls. Mutating returned messages or cache metadata cannot affect
|
||||
// this exchange, another branch, or subsequent Details calls.
|
||||
func (e *Exchange) Details() ExchangeDetails {
|
||||
if e == nil || e.internal == nil {
|
||||
return ExchangeDetails{}
|
||||
}
|
||||
messages := make([]RenderedMessage, e.internal.MessageCount())
|
||||
e.internal.VisitMessages(func(index int, message domain.RenderedMessage) {
|
||||
messages[index] = RenderedMessage{Role: message.Role, Content: message.Content, CacheControl: fromDomainCacheControl(message.CacheControl)}
|
||||
})
|
||||
return ExchangeDetails{RunID: e.internal.RunID(), ParentRunID: e.internal.ParentRunID(), Messages: messages}
|
||||
}
|
||||
|
||||
// String returns an opaque representation without conversation content.
|
||||
func (e Exchange) String() string { return exchangeString }
|
||||
|
||||
// GoString returns an opaque representation without conversation content.
|
||||
func (e Exchange) GoString() string { return exchangeString }
|
||||
113
exchange_capture_contract_test.go
Normal file
113
exchange_capture_contract_test.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestExchangeCaptureAndPreparedContinuation(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||
ctx := context.Background()
|
||||
plain, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil || plain.Exchange != nil {
|
||||
t.Fatalf("plain run = %#v, %v", plain, err)
|
||||
}
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil || initial.Exchange == nil {
|
||||
t.Fatalf("captured run = %#v, %v", initial, err)
|
||||
}
|
||||
want := []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "question"}, {Role: promptkit.RoleAssistant, Content: "answer"}}
|
||||
if details := initial.Exchange.Details(); details.RunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) {
|
||||
t.Fatalf("details = %#v", details)
|
||||
}
|
||||
encoded, err := json.Marshal(initial)
|
||||
if err != nil || strings.Contains(string(encoded), "exchange") || strings.Contains(string(encoded), "question") {
|
||||
t.Fatalf("result JSON = %s, %v", encoded, err)
|
||||
}
|
||||
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if handle.Details().ParentRunID != initial.RunID {
|
||||
t.Fatal("prepared ancestry missing")
|
||||
}
|
||||
continued, err := engine.RunPrepared(ctx, handle)
|
||||
if err != nil || continued.Exchange == nil || continued.ParentRunID != initial.RunID {
|
||||
t.Fatalf("continued = %#v, %v", continued, err)
|
||||
}
|
||||
want = append(want, promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "question"}, promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: "answer"})
|
||||
if details := continued.Exchange.Details(); details.ParentRunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) {
|
||||
t.Fatalf("continued details = %#v", details)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExchangeCapturesInvalidFinalOutput(t *testing.T) {
|
||||
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}}
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true,
|
||||
Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}})
|
||||
if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil {
|
||||
t.Fatalf("invalid result = %#v, %v", result, err)
|
||||
}
|
||||
if messages := result.Exchange.Details().Messages; len(messages) != 2 || messages[1].Content != " " {
|
||||
t.Fatalf("history = %#v", messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExchangeFailuresLeaveSourceReusable(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||
ctx := context.Background()
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := initial.Exchange.Details()
|
||||
request := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange}
|
||||
handle, err := engine.PrepareExecution(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handle.Discard()
|
||||
if result, err := engine.RunPrepared(ctx, handle); result != nil || !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("discarded = %#v, %v", result, err)
|
||||
}
|
||||
client.err = errors.New("provider failure")
|
||||
if result, err := engine.Run(ctx, request); result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||
t.Fatalf("failure = %#v, %v", result, err)
|
||||
}
|
||||
client.err = nil
|
||||
result, err := engine.Run(ctx, request)
|
||||
if err != nil || result.Exchange == nil || !reflect.DeepEqual(initial.Exchange.Details(), original) {
|
||||
t.Fatalf("reused source = %#v, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyExchangeRejectedBeforePreparation(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||
request := promptkit.RunRequest{PromptID: "missing", ContinueFrom: &promptkit.Exchange{}}
|
||||
ctx := context.Background()
|
||||
if _, err := engine.Prepare(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("prepare error = %v", err)
|
||||
}
|
||||
if err := engine.ValidateRequest(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("validate error = %v", err)
|
||||
}
|
||||
if _, err := engine.PrepareExecution(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("prepare execution error = %v", err)
|
||||
}
|
||||
if _, err := engine.Run(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("run error = %v", err)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
t.Fatal("invalid exchange reached the model")
|
||||
}
|
||||
}
|
||||
38
exchange_test.go
Normal file
38
exchange_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestExchangeDetailsAndRequestConversion(t *testing.T) {
|
||||
var nilExchange *Exchange
|
||||
if got := nilExchange.Details(); got.RunID != "" || got.Messages != nil {
|
||||
t.Fatalf("nil details = %#v", got)
|
||||
}
|
||||
if _, err := toDomainRunRequest(RunRequest{ContinueFrom: &Exchange{}}); err == nil {
|
||||
t.Fatal("empty exchange accepted")
|
||||
}
|
||||
history := domain.CaptureExchange(nil, "source", []domain.RenderedMessage{{Role: domain.RoleUser, Content: "private", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}}}, "answer")
|
||||
handle := &Exchange{internal: history}
|
||||
request, err := toDomainRunRequest(RunRequest{ContinueFrom: handle})
|
||||
if err != nil || request.ContinueFrom != history || !request.CaptureExchange {
|
||||
t.Fatalf("request = %#v, error = %v", request, err)
|
||||
}
|
||||
details := handle.Details()
|
||||
details.Messages[0].CacheControl.TTL = "changed"
|
||||
if handle.Details().Messages[0].CacheControl.TTL != "1h" {
|
||||
t.Fatal("details changed exchange")
|
||||
}
|
||||
if strings.Contains(fmt.Sprintf("%v %#v", handle, handle), "private") {
|
||||
t.Fatal("format exposed history")
|
||||
}
|
||||
encoded, err := json.Marshal(PreparedRun{ParentRunID: "source"})
|
||||
if err != nil || !strings.Contains(string(encoded), `"parent_run_id":"source"`) {
|
||||
t.Fatalf("prepared JSON = %s, %v", encoded, err)
|
||||
}
|
||||
}
|
||||
192
execution_budget_contract_test.go
Normal file
192
execution_budget_contract_test.go
Normal file
@@ -0,0 +1,192 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestOperationBudgetCoversPreparationAndReleasesAdmission(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
reader := &capacityArtifactReader{entered: make(chan struct{}, 2), release: release}
|
||||
client := newCapacityGateClient(closedCapacityChannel(), 2)
|
||||
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader)
|
||||
parent := context.Background()
|
||||
req := capacityInputRequest("http://example.invalid/v1")
|
||||
req.OperationTimeout = time.Second
|
||||
_, err := engine.Run(parent, req)
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || !errors.Is(err, promptkit.ErrArtifactLoad) || parent.Err() != nil {
|
||||
t.Fatalf("preparation error=%v parent=%v", err, parent.Err())
|
||||
}
|
||||
close(release)
|
||||
if result, err := engine.Run(parent, req); err != nil || result == nil {
|
||||
t.Fatalf("capacity not released: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOperationBudgetCoversGenerationCapacityWait(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
client := newCapacityGateClient(release, 2)
|
||||
engine := newBackendCapacityEngine(t, client, 1, capacityInt(1), nil)
|
||||
results := make(chan capacityRunResult, 1)
|
||||
go runCapacityRequest(engine, context.Background(), promptkit.RunRequest{PromptID: "prompt"}, results)
|
||||
<-client.started
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second})
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("capacity wait error=%v", err)
|
||||
}
|
||||
close(release)
|
||||
if result := <-results; result.err != nil {
|
||||
t.Fatal(result.err)
|
||||
}
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOperationBudgetInterruptsRetryAndRepair(t *testing.T) {
|
||||
for _, repair := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "retry", true: "repair"}[repair], func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
calls := 0
|
||||
config := simpleConfig("message")
|
||||
one := 1
|
||||
config.TransientRetries = &one
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if repair && calls == 1 {
|
||||
return retryHTTPResponse(200, "not json"), nil
|
||||
}
|
||||
if repair {
|
||||
<-r.Context().Done()
|
||||
return nil, r.Context().Err()
|
||||
}
|
||||
response := retryHTTPResponse(503, "")
|
||||
response.Header = make(http.Header)
|
||||
response.Header.Set("Retry-After", "5")
|
||||
return response, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := simpleGenerationRequest()
|
||||
req.OperationTimeout = time.Second
|
||||
if repair {
|
||||
req.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||
}
|
||||
_, err = engine.Run(context.Background(), req)
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("error=%v calls=%d", err, calls)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperationOutcomeFrozenBeforeTerminalObserver(t *testing.T) {
|
||||
for _, split := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "run", true: "prepared"}[split], func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
req := promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
time.Sleep(2 * time.Second)
|
||||
terminal = event
|
||||
}
|
||||
}}
|
||||
var result *promptkit.RunResult
|
||||
var err error
|
||||
if split {
|
||||
var handle *promptkit.PreparedExecution
|
||||
handle, err = engine.PrepareExecution(context.Background(), req)
|
||||
if err == nil {
|
||||
result, err = engine.RunPrepared(context.Background(), handle)
|
||||
}
|
||||
} else {
|
||||
result, err = engine.Run(context.Background(), req)
|
||||
}
|
||||
if err != nil || result == nil || terminal.Outcome != promptkit.ExecutionOutcomeSuccess {
|
||||
t.Fatalf("result=%v error=%v terminal=%+v", result, err, terminal)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotificationConsumesOperationBudget(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
t.Fatal("generation ran")
|
||||
return nil, nil
|
||||
}))
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) { <-ctx.Done() }})
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOperationTimeoutDoesNotRelabelOtherFailures(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
failure := errors.New("independent failure")
|
||||
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
<-ctx.Done()
|
||||
return nil, failure
|
||||
}))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
terminal = event
|
||||
}
|
||||
}})
|
||||
if !errors.Is(err, failure) || errors.Is(err, promptkit.ErrOperationTimeout) || terminal.Failure != promptkit.ExecutionFailureGeneration {
|
||||
t.Fatalf("error=%v terminal=%+v", err, terminal)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOperationTimeoutTerminalObserverSeesFinalFailure(t *testing.T) {
|
||||
for _, split := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "run", true: "prepared"}[split], func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
req := promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
time.Sleep(time.Second)
|
||||
terminal = event
|
||||
}
|
||||
}}
|
||||
var err error
|
||||
if split {
|
||||
var handle *promptkit.PreparedExecution
|
||||
handle, err = engine.PrepareExecution(context.Background(), req)
|
||||
if err == nil {
|
||||
_, err = engine.RunPrepared(context.Background(), handle)
|
||||
}
|
||||
} else {
|
||||
_, err = engine.Run(context.Background(), req)
|
||||
}
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || terminal.Failure != promptkit.ExecutionFailureDeadline {
|
||||
t.Fatalf("error=%v terminal=%+v", err, terminal)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
38
execution_controls_conversion_test.go
Normal file
38
execution_controls_conversion_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestExecutionControlsConversion(t *testing.T) {
|
||||
ctx := context.WithValue(context.Background(), struct{}{}, "selected")
|
||||
called := false
|
||||
req := RunRequest{OperationTimeout: 3 * time.Second, OnPrepared: func(got context.Context, details PreparedRun) {
|
||||
called = true
|
||||
if got != ctx || details.PromptID != "prompt" {
|
||||
t.Fatalf("unexpected notification: %v, %+v", got, details)
|
||||
}
|
||||
details.InputHashes["input"] = "changed"
|
||||
}}
|
||||
converted, err := toDomainRunRequest(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if converted.OperationTimeout != req.OperationTimeout || converted.OnPrepared == nil || called {
|
||||
t.Fatal("execution controls were not retained without invocation")
|
||||
}
|
||||
req.OnPrepared = func(context.Context, PreparedRun) { t.Fatal("callback selection changed") }
|
||||
prepared := &domain.PreparedRun{PromptID: "prompt", InputHashes: map[string]string{"input": "original"}}
|
||||
converted.OnPrepared(ctx, prepared)
|
||||
if !called || prepared.InputHashes["input"] != "original" {
|
||||
t.Fatal("notification did not deliver detached details")
|
||||
}
|
||||
withoutCallback, err := toDomainRunRequest(RunRequest{})
|
||||
if err != nil || withoutCallback.OnPrepared != nil {
|
||||
t.Fatalf("nil callback was adapted: %v", err)
|
||||
}
|
||||
}
|
||||
56
execution_group.go
Normal file
56
execution_group.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/observation"
|
||||
)
|
||||
|
||||
// ExecutionGroup collects facts from explicitly selected execution events.
|
||||
// Its zero value is ready for concurrent use. Do not copy it after first use.
|
||||
type ExecutionGroup struct{ group observation.Group }
|
||||
|
||||
// ExecutionGroupMember identifies an observed run and its terminal coverage.
|
||||
type ExecutionGroupMember struct {
|
||||
RunID string `json:"run_id"`
|
||||
TerminalObserved bool `json:"terminal_observed"`
|
||||
}
|
||||
|
||||
// ExecutionGroupSummary is a detached snapshot of terminal-only totals.
|
||||
// Usage sums known terminal values, with Known as their logical OR and
|
||||
// Complete as their logical AND. Coverage describes whether every observed
|
||||
// member has a terminal, independently of usage completeness. An empty group
|
||||
// has complete coverage and known complete zero usage; a group with members
|
||||
// but no terminals has unknown, incomplete usage. Overlapping groups are not
|
||||
// generally additive.
|
||||
type ExecutionGroupSummary struct {
|
||||
Members []ExecutionGroupMember `json:"members"`
|
||||
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||
Attempts uint64 `json:"attempts"`
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
RepairCalls uint64 `json:"repair_calls"`
|
||||
Usage ObservedTokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// Observe enrolls an event's run, counting its first terminal event only.
|
||||
func (g *ExecutionGroup) Observe(event ExecutionEvent) {
|
||||
count := func(c ObservedTokenCount) domain.ObservedTokenCount {
|
||||
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
|
||||
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
|
||||
}
|
||||
|
||||
// Snapshot returns an internally consistent, caller-owned summary. Coverage
|
||||
// describes observed members only, not future members or application progress.
|
||||
func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
|
||||
value := g.group.Snapshot()
|
||||
count := func(c domain.ObservedTokenCount) ObservedTokenCount {
|
||||
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
members := make([]ExecutionGroupMember, len(value.Members))
|
||||
for i, member := range value.Members {
|
||||
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
|
||||
}
|
||||
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
|
||||
}
|
||||
151
execution_group_contract_test.go
Normal file
151
execution_group_contract_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestExecutionGroupExplicitMembershipAndTerminalTotals(t *testing.T) {
|
||||
responses := []*http.Response{
|
||||
observedHTTPResponse(200, `{"choices":[{"message":{"content":"initial"}}],"usage":{"prompt_tokens":2}}`),
|
||||
observedHTTPResponse(503, `{}`),
|
||||
observedHTTPResponse(200, `{"choices":[{"message":{"content":"invalid"}}],"usage":{"prompt_tokens":3}}`),
|
||||
observedHTTPResponse(200, `{"choices":[{"message":{"content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":4}}`),
|
||||
observedHTTPResponse(200, `{"choices":[{"message":{"content":"invalid"}}],"usage":{"prompt_tokens":5}}`),
|
||||
observedHTTPResponse(422, `{"error":{}}`),
|
||||
observedHTTPResponse(200, `{"choices":[{"message":{"content":"validator"}}],"usage":{"prompt_tokens":6}}`),
|
||||
}
|
||||
calls := 0
|
||||
one := 1
|
||||
config := simpleConfig("message")
|
||||
config.TransientRetries = &one
|
||||
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
response := responses[calls]
|
||||
calls++
|
||||
return response, nil
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
var all, selected promptkit.ExecutionGroup
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, Observer: all.Observe})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validation := &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||
callbackCount := 0
|
||||
var branchID, validatorID string
|
||||
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: promptkit.ComposeObservers(all.Observe, selected.Observe), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
callbackCount++
|
||||
if turn.Result == nil {
|
||||
return &promptkit.RunRequest{PromptID: "prompt", Validation: validation}, nil
|
||||
}
|
||||
branchID = turn.Result.RunID
|
||||
return nil, nil
|
||||
}})
|
||||
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbackCount != 2 {
|
||||
t.Fatalf("coordinate=%#v, %v", result, err)
|
||||
}
|
||||
if failed, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Validation: validation, Observer: all.Observe}); err == nil || failed != nil {
|
||||
t.Fatalf("failed branch=%#v, %v", failed, err)
|
||||
}
|
||||
validator, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: selected.Observe})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validatorID = validator.RunID
|
||||
a, b := all.Snapshot(), selected.Snapshot()
|
||||
if calls != 7 || len(a.Members) != 3 || a.Attempts != 6 || a.TransientRetries != 1 || a.RepairCalls != 2 || a.Usage.PromptTokens.Value != 14 || !a.Usage.PromptTokens.Known || !a.TerminalCoverageComplete {
|
||||
t.Fatalf("all summary=%#v calls=%d", a, calls)
|
||||
}
|
||||
if len(b.Members) != 2 || b.Attempts != 4 || b.Usage.PromptTokens.Value != 13 || !b.TerminalCoverageComplete {
|
||||
t.Fatalf("selected summary=%#v", b)
|
||||
}
|
||||
ids := map[string]bool{}
|
||||
for _, member := range b.Members {
|
||||
ids[member.RunID] = true
|
||||
}
|
||||
if !ids[branchID] || !ids[validatorID] || ids[initial.RunID] {
|
||||
t.Fatalf("selected members=%#v", b.Members)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionGroupPreparedSilenceAndPanicCoverage(t *testing.T) {
|
||||
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}))
|
||||
ctx := context.Background()
|
||||
var group promptkit.ExecutionGroup
|
||||
prepared, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared.Discard()
|
||||
_, _ = engine.RunPrepared(ctx, prepared)
|
||||
if len(group.Snapshot().Members) != 0 {
|
||||
t.Fatal("preparation or failed claim enrolled a run")
|
||||
}
|
||||
func() {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Error("expected panic")
|
||||
}
|
||||
}()
|
||||
_, _ = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) {
|
||||
_ = group.Snapshot()
|
||||
if event.Kind == promptkit.ExecutionStarted {
|
||||
panic("stop")
|
||||
}
|
||||
})})
|
||||
}()
|
||||
result := group.Snapshot()
|
||||
if len(result.Members) != 1 || result.TerminalCoverageComplete || result.Members[0].TerminalObserved || result.Attempts != 0 {
|
||||
t.Fatalf("panic summary=%#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionGroupPreparedAndCallbackMembership(t *testing.T) {
|
||||
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}))
|
||||
ctx := context.Background()
|
||||
var continuations, nested promptkit.ExecutionGroup
|
||||
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: nested.Observe})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
preparedResult, err := engine.RunPrepared(ctx, prepared)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var callbackID string
|
||||
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: continuations.Observe, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result == nil {
|
||||
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||
}
|
||||
callbackResult, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: nested.Observe})
|
||||
if err == nil {
|
||||
callbackID = callbackResult.RunID
|
||||
}
|
||||
return nil, err
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, b := continuations.Snapshot(), nested.Snapshot()
|
||||
if len(a.Members) != 1 || len(b.Members) != 2 || !a.TerminalCoverageComplete || !b.TerminalCoverageComplete {
|
||||
t.Fatalf("summaries = %#v %#v", a, b)
|
||||
}
|
||||
ids := map[string]bool{}
|
||||
for _, member := range b.Members {
|
||||
ids[member.RunID] = true
|
||||
}
|
||||
if !ids[preparedResult.RunID] || !ids[callbackID] || ids[initial.RunID] {
|
||||
t.Fatalf("members=%#v", b.Members)
|
||||
}
|
||||
}
|
||||
167
execution_timeout_contract_test.go
Normal file
167
execution_timeout_contract_test.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestBuiltInGenerationOperationTimeout(t *testing.T) {
|
||||
for _, split := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "run", true: "prepared"}[split], func(t *testing.T) {
|
||||
started := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
select {
|
||||
case started <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-release
|
||||
}))
|
||||
defer provider.Close()
|
||||
defer close(release)
|
||||
|
||||
config := simpleConfig("message")
|
||||
zero := 0
|
||||
config.TransientRetries = &zero
|
||||
config.HTTPClient = provider.Client()
|
||||
engine, err := promptkit.NewEngine(config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var terminal promptkit.ExecutionEvent
|
||||
req := promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second,
|
||||
Execution: &promptkit.ExecutionTargetOverride{Endpoint: provider.URL + "/v1", TimeoutSeconds: &zero},
|
||||
Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
terminal = event
|
||||
}
|
||||
},
|
||||
}
|
||||
parent := context.Background()
|
||||
var result *promptkit.RunResult
|
||||
if split {
|
||||
handle, prepareErr := engine.PrepareExecution(parent, req)
|
||||
if prepareErr != nil {
|
||||
t.Fatal(prepareErr)
|
||||
}
|
||||
result, err = engine.RunPrepared(parent, handle)
|
||||
} else {
|
||||
result, err = engine.Run(parent, req)
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
default:
|
||||
t.Fatal("operation expired before provider received the request")
|
||||
}
|
||||
for _, target := range []error{promptkit.ErrOperationTimeout, context.DeadlineExceeded, promptkit.ErrLLMGenerate, promptkit.ErrTransport} {
|
||||
if !errors.Is(err, target) {
|
||||
t.Errorf("error=%v, want identity %v", err, target)
|
||||
}
|
||||
}
|
||||
var transportErr *url.Error
|
||||
if !errors.As(err, &transportErr) {
|
||||
t.Errorf("error=%v, want retained HTTP transport cause", err)
|
||||
}
|
||||
if result != nil || parent.Err() != nil || terminal.Failure != promptkit.ExecutionFailureDeadline {
|
||||
t.Fatalf("result=%v parent=%v terminal=%+v", result, parent.Err(), terminal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOperationTimeoutAttribution(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second,
|
||||
Observer: func(event promptkit.ExecutionEvent) {
|
||||
if event.Kind == promptkit.ExecutionFinished {
|
||||
terminal = event
|
||||
}
|
||||
}})
|
||||
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || !errors.Is(err, promptkit.ErrLLMGenerate) || terminal.Failure != promptkit.ExecutionFailureDeadline {
|
||||
t.Fatalf("error=%v terminal=%+v", err, terminal)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunProviderDeadlineIsNotOperationTimeout(t *testing.T) {
|
||||
engine := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return nil, context.DeadlineExceeded
|
||||
}))
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Minute})
|
||||
if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, promptkit.ErrOperationTimeout) {
|
||||
t.Fatalf("provider deadline attribution: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedExecutionRetainsTimeoutWithoutIdleTimer(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}))
|
||||
req := promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second}
|
||||
handle, err := engine.PrepareExecution(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.OperationTimeout = 0
|
||||
time.Sleep(2 * time.Second)
|
||||
start := time.Now()
|
||||
_, err = engine.RunPrepared(context.Background(), handle)
|
||||
if time.Since(start) != time.Second || !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||
t.Fatalf("elapsed=%v error=%v", time.Since(start), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPreparedExecutionRespectsCallerDeadline(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}))
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", OperationTimeout: 5 * time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
_, err = engine.RunPrepared(ctx, handle)
|
||||
if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, promptkit.ErrOperationTimeout) {
|
||||
t.Fatalf("caller deadline attribution: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPreparedExecutionDoesNotTimePreparation(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
reader := &capacityArtifactReader{entered: make(chan struct{}, 1), release: release}
|
||||
engine := newBackendCapacityEngine(t, newCapacityGateClient(closedCapacityChannel(), 1), 1, capacityInt(0), reader)
|
||||
req := capacityInputRequest("http://example.invalid/v1")
|
||||
req.OperationTimeout = time.Second
|
||||
go func() { time.Sleep(2 * time.Second); close(release) }()
|
||||
start := time.Now()
|
||||
handle, err := engine.PrepareExecution(context.Background(), req)
|
||||
if err != nil || time.Since(start) != 2*time.Second {
|
||||
t.Fatalf("prepare elapsed=%v error=%v", time.Since(start), err)
|
||||
}
|
||||
if _, err = engine.RunPrepared(context.Background(), handle); err != nil {
|
||||
t.Fatalf("execution after preparation: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
204
focused_inspection_integration_test.go
Normal file
204
focused_inspection_integration_test.go
Normal file
@@ -0,0 +1,204 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type focusedInspectionRoot struct {
|
||||
Sibling string `yaml:"sibling"`
|
||||
PromptKit *promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
func TestFocusedInspectionReturnsDetachedDiagnostics(t *testing.T) {
|
||||
var resolves, validates atomic.Int64
|
||||
definition := integrationDefinition(&resolves, &validates)
|
||||
result, err := configkit.Load(configkit.Bytes("diagnostics", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := view.Diagnostics()
|
||||
if len(first) != 1 || first[0].Primary == nil {
|
||||
t.Fatalf("diagnostics = %+v", first)
|
||||
}
|
||||
original := first[0].Primary.SourceID
|
||||
first[0].Primary.SourceID = "changed"
|
||||
again := view.Diagnostics()
|
||||
if again[0].Primary.SourceID != original {
|
||||
t.Fatalf("diagnostics alias caller mutation: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
func focusedInspection(t *testing.T, source string, sensitive func(configkit.Path) bool) (promptkit.ConfigurationInspection, *configkit.Result[focusedInspectionRoot]) {
|
||||
t.Helper()
|
||||
definition := promptkit.Configure(configkit.Definition[focusedInspectionRoot]{Sensitive: sensitive}, func(value *focusedInspectionRoot) *promptkit.Settings { return value.PromptKit })
|
||||
result, err := configkit.Load(configkit.Bytes("focused", []byte(source)), configkit.Options[focusedInspectionRoot]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return view, result
|
||||
}
|
||||
|
||||
func TestFocusedInspectionRetainsNativeEvidence(t *testing.T) {
|
||||
view, result := focusedInspection(t, "sibling: private\npromptkit:\n app_secret: hidden\n app_value: original\n", func(path configkit.Path) bool {
|
||||
return path.String() == "/sibling" || path.String() == "/promptkit/app_secret"
|
||||
})
|
||||
result.Config.PromptKit.Application["app_value"] = "mutated"
|
||||
var rendered bytes.Buffer
|
||||
if err := view.WriteEffective(&rendered); err != nil || !strings.Contains(rendered.String(), "original") || strings.Contains(rendered.String(), "hidden") || strings.Contains(rendered.String(), "mutated") || strings.Contains(rendered.String(), "sibling") {
|
||||
t.Fatalf("focused effective = %q, %v", rendered.String(), err)
|
||||
}
|
||||
if _, ok := view.Metadata().Lookup(configkit.Keys("sibling")); !ok {
|
||||
t.Fatal("unfiltered metadata omitted sibling")
|
||||
}
|
||||
sibling, ok := view.Explain(configkit.Keys("sibling"))
|
||||
if !ok || !sibling.Effective.Redacted {
|
||||
t.Fatalf("outside-focus explanation = %+v, %t", sibling, ok)
|
||||
}
|
||||
if _, ok := view.Explain(configkit.Keys("promptkit")); !ok {
|
||||
t.Fatal("focus explanation missing")
|
||||
}
|
||||
if _, ok := view.Explain(configkit.Path{}); !ok {
|
||||
t.Fatal("whole-application explanation missing")
|
||||
}
|
||||
if len(view.Sources()) == 0 {
|
||||
t.Fatal("native source catalog missing")
|
||||
}
|
||||
for _, path := range view.HistoricalPaths() {
|
||||
if path.String() != "/promptkit" && !strings.HasPrefix(path.String(), "/promptkit/") {
|
||||
t.Fatalf("out of scope history: %v", path)
|
||||
}
|
||||
}
|
||||
if len(view.HistoricalPaths()) == 0 {
|
||||
t.Fatal("focused history empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFocusedInspectionHonorsAncestorSensitivityAndHistory(t *testing.T) {
|
||||
redacted, _ := focusedInspection(t, "promptkit: {app_value: secret-value}\n", func(path configkit.Path) bool { return path.String() == "/promptkit" })
|
||||
var rendered bytes.Buffer
|
||||
if err := redacted.WriteEffective(&rendered); err != nil || strings.Contains(rendered.String(), "secret-value") || !strings.Contains(rendered.String(), "REDACTED") {
|
||||
t.Fatalf("ancestor redaction = %q, %v", rendered.String(), err)
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[focusedInspectionRoot]{}, func(value *focusedInspectionRoot) *promptkit.Settings { return value.PromptKit })
|
||||
lower := configkit.Input{Source: configkit.Bytes("lower", []byte("promptkit: {historical: old}\n"))}
|
||||
main := configkit.Input{Source: configkit.Bytes("main", []byte("promptkit: null\n"))}
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[focusedInspectionRoot]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{lower}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := configkit.Keys("promptkit", "historical")
|
||||
if len(view.Metadata().History(path)) == 0 {
|
||||
t.Fatal("historical-only field has no native history")
|
||||
}
|
||||
found := false
|
||||
for _, item := range view.HistoricalPaths() {
|
||||
if item.String() == path.String() {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("historical path missing: %+v", view.HistoricalPaths())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFocusedInspectionAbsenceNullAndCrossTypeDiff(t *testing.T) {
|
||||
type noSection struct {
|
||||
Sibling string `yaml:"sibling"`
|
||||
}
|
||||
missing, err := configkit.Load[noSection](configkit.Bytes("absent", []byte("sibling: before\n")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
missingEngine, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
absent, err := missingEngine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output bytes.Buffer
|
||||
if err := absent.WriteEffective(&output); err == nil || output.Len() != 0 {
|
||||
t.Fatalf("absent output = %q, %v", output.String(), err)
|
||||
}
|
||||
nullView, _ := focusedInspection(t, "sibling: before\npromptkit: null\n", nil)
|
||||
if err := nullView.WriteEffective(&output); err != nil || strings.TrimSpace(output.String()) != "null" {
|
||||
t.Fatalf("null output = %q, %v", output.String(), err)
|
||||
}
|
||||
differences, err := absent.Diff(nullView)
|
||||
if err != nil || len(differences) == 0 {
|
||||
t.Fatalf("absent/null diff = %+v, %v", differences, err)
|
||||
}
|
||||
changed, _ := focusedInspection(t, "sibling: before\npromptkit: {label: one}\n", nil)
|
||||
if differences, err := absent.Diff(changed); err != nil || len(differences) == 0 {
|
||||
t.Fatalf("added scope diff = %+v, %v", differences, err)
|
||||
}
|
||||
if differences, err := changed.Diff(absent); err != nil || len(differences) == 0 {
|
||||
t.Fatalf("removed scope diff = %+v, %v", differences, err)
|
||||
}
|
||||
otherSibling, _ := focusedInspection(t, "sibling: after\npromptkit: {label: one}\n", nil)
|
||||
differences, err = changed.Diff(otherSibling)
|
||||
if err != nil || len(differences) != 0 {
|
||||
t.Fatalf("sibling-only diff = %+v, %v", differences, err)
|
||||
}
|
||||
type differentRoot struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
Extra bool `yaml:"extra"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[differentRoot]{}, func(value *differentRoot) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.Load(configkit.Bytes("other", []byte("promptkit: {label: two}\n")), configkit.Options[differentRoot]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
differences, err = changed.Diff(other)
|
||||
if err != nil || len(differences) == 0 {
|
||||
t.Fatalf("cross-root diff = %+v, %v", differences, err)
|
||||
}
|
||||
var zero promptkit.ConfigurationInspection
|
||||
if _, err := zero.Diff(changed); !errors.Is(err, promptkit.ErrInvalidConfig) || zero.HistoricalPaths() != nil {
|
||||
t.Fatalf("zero view: %v", err)
|
||||
}
|
||||
writeErr := errors.New("writer failure")
|
||||
if err := changed.WriteEffective(failingInspectionWriter{err: writeErr}); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("writer error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -38,8 +38,8 @@ func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: inputs},
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
|
||||
{Inputs: mountSettingsInputs(inputs)},
|
||||
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
|
||||
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||
})}}},
|
||||
},
|
||||
@@ -144,10 +144,10 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||
schemas := fstest.MapFS{}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{schemaRootInput(schemas, ".")}}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||
})}}}},
|
||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("structured prompt", map[string]any{
|
||||
"prompts": []any{map[string]any{"id": "structured",
|
||||
"default_profile": "offline",
|
||||
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
||||
@@ -155,7 +155,7 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||
}},
|
||||
})},
|
||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine before request resources exist: %v", err)
|
||||
}
|
||||
|
||||
@@ -78,10 +78,11 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
req := simpleGenerationRequest()
|
||||
req.Validation = &promptkit.OutputContract{
|
||||
req.CaptureExchange = true
|
||||
req.Validation = &promptkit.OutputContractInput{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSON,
|
||||
RepairAttempts: 1,
|
||||
RepairAttempts: new(1),
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), req)
|
||||
|
||||
14
go.mod
14
go.mod
@@ -3,16 +3,16 @@ module go.maximumdirect.net/promptkit
|
||||
go 1.27.0
|
||||
|
||||
require (
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
go.maximumdirect.net/configkit v1.0.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
|
||||
go.maximumdirect.net/configkit v1.4.0
|
||||
go.yaml.in/yaml/v4 v4.0.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||
go.maximumdirect.net/releasekit v0.8.0 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
golang.org/x/mod v0.40.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
github.com/yuin/goldmark/v2 v2.1.16 // indirect
|
||||
go.maximumdirect.net/releasekit v0.10.0 // indirect
|
||||
golang.org/x/mod v0.43.0 // indirect
|
||||
golang.org/x/text v0.44.0 // indirect
|
||||
)
|
||||
|
||||
tool go.maximumdirect.net/releasekit/cmd/releasekit
|
||||
|
||||
30
go.sum
30
go.sum
@@ -1,16 +1,18 @@
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||
go.maximumdirect.net/configkit v1.0.0 h1:cguQnxkaRk3ClQOZNzzhTdc4bfzbraGKv8CkoCp+HLc=
|
||||
go.maximumdirect.net/configkit v1.0.0/go.mod h1:B6fJ5xBLBF1nuywqfLeEI4sLm29UkIcDLu8i6I6gGXE=
|
||||
go.maximumdirect.net/releasekit v0.8.0 h1:hSaHPbUI7tAhnolfbu7UMj2fvmm9FGBY+jhkygdbxR0=
|
||||
go.maximumdirect.net/releasekit v0.8.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/yuin/goldmark/v2 v2.1.16 h1:MXvCVo+Cb2SVJ5cwJUzXetDjI7HECRJCkcqylEQtcJo=
|
||||
github.com/yuin/goldmark/v2 v2.1.16/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||
go.maximumdirect.net/configkit v1.3.0 h1:X3LZnNJL6T2C6DyUugcvIvHMXX6qhtAF+fQEUBeAwuw=
|
||||
go.maximumdirect.net/configkit v1.3.0/go.mod h1:ewu6blSokUDCMovy8JBXAdXRikGmlHXAQpFghnPWRfs=
|
||||
go.maximumdirect.net/configkit v1.4.0 h1:TfOupZbQ23/ZMs+xt5lmyukCF11ekqKep8KbmFQD63o=
|
||||
go.maximumdirect.net/configkit v1.4.0/go.mod h1:ewu6blSokUDCMovy8JBXAdXRikGmlHXAQpFghnPWRfs=
|
||||
go.maximumdirect.net/releasekit v0.10.0 h1:3GOgaIigXTkvYaRkntWZASE//2NTR1+IhkoKmt64aJo=
|
||||
go.maximumdirect.net/releasekit v0.10.0/go.mod h1:cT5NZNEcf7SKDx4MPG40YNK9V4bNyIxcLLFuNlbGQ90=
|
||||
go.yaml.in/yaml/v4 v4.0.0 h1:/LG1pNO84Z4l603j/xMn69CcS5OYhMQ/f7l2LoQUGuw=
|
||||
go.yaml.in/yaml/v4 v4.0.0/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.43.0 h1:MFuq5J6s35W9D3IjkwyWc9vHh0nENOrt5K8Iof4ofmg=
|
||||
golang.org/x/mod v0.43.0/go.mod h1:4ec1JnuyBz5+FIVhg86UsA39YG8BpnrZdJYSNHmGzHY=
|
||||
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||
|
||||
@@ -3,8 +3,6 @@ package artifact
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -153,9 +151,12 @@ func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef,
|
||||
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
||||
if artifact.Hash != wantHash {
|
||||
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(want)})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||
b.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,9 +3,7 @@ package artifact
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -284,9 +282,12 @@ func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
||||
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||
}
|
||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
||||
if artifact.Hash != wantHash {
|
||||
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
||||
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(tc.body)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||
t.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -27,17 +27,13 @@ type Registry struct {
|
||||
// NewEffectiveRegistry stores already-validated effective backend definitions
|
||||
// under their exact configuration keys. Configuration validation and
|
||||
// normalization precede this ownership boundary.
|
||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) *Registry {
|
||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||
for id, definition := range definitions {
|
||||
definition.ID = id
|
||||
cloned, err := domain.CloneBackend(definition)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backend %q: %w", id, err)
|
||||
}
|
||||
registry.backends[id] = cloned
|
||||
registry.backends[id] = domain.CloneBackend(definition)
|
||||
}
|
||||
return registry, nil
|
||||
return registry
|
||||
}
|
||||
|
||||
// GetBackend returns a defensive copy of the backend registered with id.
|
||||
@@ -49,11 +45,7 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
||||
if !ok {
|
||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||
}
|
||||
cloned, err := domain.CloneBackend(definition)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
||||
}
|
||||
return cloned, nil
|
||||
return domain.CloneBackend(definition), nil
|
||||
}
|
||||
|
||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||
|
||||
@@ -22,16 +22,13 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
||||
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||
},
|
||||
}
|
||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||
registry := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||
" local.v1 ": {
|
||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
||||
ConcurrencyLimit: 2, QueueCapacity: 3,
|
||||
SmokeTests: smokeTests,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
params["nested"].(map[string]any)["enabled"] = false
|
||||
defaultProfile = "mutated-profile"
|
||||
reachabilityEndpoint = "https://mutated.example/ready"
|
||||
@@ -79,10 +76,7 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
||||
registry, err := backend.NewEffectiveRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := backend.NewEffectiveRegistry(nil)
|
||||
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("missing backend error = %v", err)
|
||||
}
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
package capacity
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
type client struct {
|
||||
manager *Manager
|
||||
next llm.Client
|
||||
}
|
||||
|
||||
// NewClient wraps next with configured active-generation limits. A nil manager
|
||||
// leaves next unchanged.
|
||||
func NewClient(manager *Manager, next llm.Client) llm.Client {
|
||||
if manager == nil {
|
||||
return next
|
||||
}
|
||||
return &client{
|
||||
manager: manager,
|
||||
next: next,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) Generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
if c.manager.getPool(req.Target.BackendID) == nil {
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
return c.next.Generate(ctx, req)
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package capacity
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -24,6 +23,18 @@ type clientFunc func(
|
||||
domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error)
|
||||
|
||||
// newTestClient exercises one acquisition around a controlled call.
|
||||
func newTestClient(manager *Manager, next llm.Client) llm.Client {
|
||||
return clientFunc(func(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
release, err := manager.Acquire(ctx, req.Target.BackendID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
return next.Generate(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
func (f clientFunc) Generate(
|
||||
ctx context.Context,
|
||||
req domain.GenerateRequest,
|
||||
@@ -167,7 +178,7 @@ func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
|
||||
"second": secondRelease,
|
||||
"third": thirdRelease,
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
client := newTestClient(manager, next)
|
||||
|
||||
first := generateAsync(client, context.Background(), "limited", "first")
|
||||
if got := receiveStarted(t, next.started); got != "first" {
|
||||
@@ -212,7 +223,7 @@ func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
|
||||
releases[string(rune('a'+i))] = gate
|
||||
}
|
||||
next := newBlockingClient(releases)
|
||||
client := NewClient(manager, next)
|
||||
client := newTestClient(manager, next)
|
||||
|
||||
results := make([]<-chan generateResult, 0, len(releases))
|
||||
for id := range releases {
|
||||
@@ -257,7 +268,7 @@ func TestClientRemovesCanceledWaiters(t *testing.T) {
|
||||
"three": make(chan struct{}),
|
||||
}
|
||||
next := newBlockingClient(releases)
|
||||
client := NewClient(manager, next)
|
||||
client := newTestClient(manager, next)
|
||||
|
||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||
if got := receiveStarted(t, next.started); got != "holder" {
|
||||
@@ -321,7 +332,7 @@ func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
|
||||
}
|
||||
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
client := newTestClient(manager, next)
|
||||
|
||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||
waitForActiveCount(t, manager, "limited", 1)
|
||||
@@ -394,7 +405,7 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
||||
"alpha": alphaRelease,
|
||||
"beta": betaRelease,
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
client := newTestClient(manager, next)
|
||||
|
||||
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
||||
beta := generateAsync(client, context.Background(), "beta", "beta")
|
||||
@@ -423,63 +434,6 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
||||
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
||||
}
|
||||
}
|
||||
if got := NewClient(nil, next); got != next {
|
||||
t.Fatal("nil manager did not return the wrapped client unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
|
||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
||||
"limited": {ConcurrencyLimit: 1},
|
||||
})
|
||||
request := domain.GenerateRequest{
|
||||
Prompt: domain.RenderedPrompt{
|
||||
SessionID: "session",
|
||||
Messages: []domain.RenderedMessage{
|
||||
{Role: "user", Content: "content"},
|
||||
},
|
||||
},
|
||||
Target: domain.ExecutionTarget{
|
||||
BackendID: "limited",
|
||||
Model: "model",
|
||||
ExtraParams: map[string]any{"key": "value"},
|
||||
},
|
||||
}
|
||||
response := &domain.GenerateResponse{
|
||||
Content: "output",
|
||||
Usage: domain.TokenUsage{TotalTokens: 7},
|
||||
}
|
||||
collaboratorErr := errors.New("collaborator failure")
|
||||
tests := []struct {
|
||||
name string
|
||||
response *domain.GenerateResponse
|
||||
err error
|
||||
}{
|
||||
{name: "successful response", response: response},
|
||||
{name: "nil response"},
|
||||
{name: "collaborator error", response: response, err: collaboratorErr},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var captured domain.GenerateRequest
|
||||
next := clientFunc(func(
|
||||
_ context.Context,
|
||||
req domain.GenerateRequest,
|
||||
) (*domain.GenerateResponse, error) {
|
||||
captured = req
|
||||
return tc.response, tc.err
|
||||
})
|
||||
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
|
||||
if !reflect.DeepEqual(captured, request) {
|
||||
t.Fatalf("request changed: %#v", captured)
|
||||
}
|
||||
if gotResponse != tc.response || gotErr != tc.err {
|
||||
t.Fatalf("response=(%p, %v), want (%p, %v)",
|
||||
gotResponse, gotErr, tc.response, tc.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||
@@ -496,7 +450,7 @@ func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||
}
|
||||
return &domain.GenerateResponse{Content: "recovered"}, nil
|
||||
})
|
||||
client := NewClient(manager, next)
|
||||
client := newTestClient(manager, next)
|
||||
request := domain.GenerateRequest{
|
||||
Target: domain.ExecutionTarget{BackendID: "limited"},
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
const benchmarkDefinitionCount = 128
|
||||
|
||||
var (
|
||||
constructionResult *loaded
|
||||
constructionResult *testLoaded
|
||||
selectedPrompt *domain.PromptDefinition
|
||||
selectedPromptHash string
|
||||
selectedPromptInspection *domain.PromptInspection
|
||||
|
||||
16
internal/configuration/bundled.go
Normal file
16
internal/configuration/bundled.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
//go:embed bundled/openrouter.yaml
|
||||
var bundledFiles embed.FS
|
||||
|
||||
// BundledOpenRouterUnderlay returns the built-in OpenRouter configuration source.
|
||||
func BundledOpenRouterUnderlay() configkit.Underlay {
|
||||
filesystem := configkit.NewFilesystem("Promptkit bundled OpenRouter", bundledFiles)
|
||||
return configkit.Underlay{Inputs: []configkit.Input{{Source: filesystem.File("bundled/openrouter.yaml"), Destination: configkit.Keys("promptkit")}}}
|
||||
}
|
||||
60
internal/configuration/bundled/openrouter.yaml
Normal file
60
internal/configuration/bundled/openrouter.yaml
Normal file
@@ -0,0 +1,60 @@
|
||||
backends:
|
||||
- id: openrouter
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
smoke_tests:
|
||||
default_profile: gpt-luna-latest
|
||||
authentication:
|
||||
kind: openrouter_key
|
||||
profiles:
|
||||
- id: gpt-astra-latest
|
||||
backend: openrouter
|
||||
model: ~openai/gpt-astra-latest
|
||||
- id: gpt-sol-latest
|
||||
backend: openrouter
|
||||
model: ~openai/gpt-sol-latest
|
||||
- id: gpt-terra-latest
|
||||
backend: openrouter
|
||||
model: ~openai/gpt-terra-latest
|
||||
- id: gpt-luna-latest
|
||||
backend: openrouter
|
||||
model: ~openai/gpt-luna-latest
|
||||
- id: claude-fable-latest
|
||||
backend: openrouter
|
||||
model: ~anthropic/claude-fable-latest
|
||||
- id: claude-opus-latest
|
||||
backend: openrouter
|
||||
model: ~anthropic/claude-opus-latest
|
||||
- id: claude-sonnet-latest
|
||||
backend: openrouter
|
||||
model: ~anthropic/claude-sonnet-latest
|
||||
- id: claude-haiku-latest
|
||||
backend: openrouter
|
||||
model: ~anthropic/claude-haiku-latest
|
||||
- id: gemini-pro-latest
|
||||
backend: openrouter
|
||||
model: ~google/gemini-pro-latest
|
||||
- id: gemini-flash-latest
|
||||
backend: openrouter
|
||||
model: ~google/gemini-flash-latest
|
||||
- id: auto
|
||||
backend: openrouter
|
||||
model: openrouter/auto
|
||||
- id: free
|
||||
backend: openrouter
|
||||
model: openrouter/free
|
||||
- id: auto-low
|
||||
extends: {id: auto}
|
||||
extra_params: {plugins: [{id: auto-router, cost_tier: low}]}
|
||||
- id: auto-medium
|
||||
extends: {id: auto}
|
||||
extra_params: {plugins: [{id: auto-router, cost_tier: medium}]}
|
||||
- id: auto-high
|
||||
extends: {id: auto}
|
||||
extra_params: {plugins: [{id: auto-router, cost_tier: high}]}
|
||||
- id: auto-xhigh
|
||||
extends: {id: auto}
|
||||
extra_params: {plugins: [{id: auto-router, cost_tier: xhigh}]}
|
||||
- id: auto-max
|
||||
extends: {id: auto}
|
||||
extra_params: {plugins: [{id: auto-router, cost_tier: max}]}
|
||||
54
internal/configuration/bundled_test.go
Normal file
54
internal/configuration/bundled_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||
)
|
||||
|
||||
func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||
state, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{BundledOpenRouterUnderlay()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := state.GetBackend("openrouter")
|
||||
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||
t.Fatalf("backend = %+v, %v", backend, err)
|
||||
}
|
||||
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
|
||||
t.Fatalf("bundle inventories: backends=%v prompts=%v collections=%v", state.backendIDs, state.promptIDs, state.collections)
|
||||
}
|
||||
models := map[string]string{
|
||||
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
|
||||
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
|
||||
"gemini-pro-latest": "~google/gemini-pro-latest", "gemini-flash-latest": "~google/gemini-flash-latest", "auto": "openrouter/auto", "free": "openrouter/free",
|
||||
}
|
||||
for _, tier := range []string{"low", "medium", "high", "xhigh", "max"} {
|
||||
models["auto-"+tier] = "openrouter/auto"
|
||||
}
|
||||
if len(state.profileIDs) != len(models) {
|
||||
t.Fatalf("profiles = %v", state.profileIDs)
|
||||
}
|
||||
for id, model := range models {
|
||||
profile, err := state.GetProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
||||
}
|
||||
want := defaults.ExecutionTargetDefault()
|
||||
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
|
||||
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
||||
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
|
||||
}
|
||||
if !reflect.DeepEqual(*profile.ResolvedTarget, want) {
|
||||
t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(fmt.Sprint(state.inspection.Sources()), "Promptkit bundled OpenRouter") {
|
||||
t.Fatalf("source evidence = %+v", state.inspection.Sources())
|
||||
}
|
||||
}
|
||||
151
internal/configuration/capturetest/capture_test.go
Normal file
151
internal/configuration/capturetest/capture_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package capturetest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||
)
|
||||
|
||||
type application struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
Other string `yaml:"other"`
|
||||
}
|
||||
|
||||
func TestCaptureUsesEffectiveSourceContexts(t *testing.T) {
|
||||
baseFiles := fstest.MapFS{
|
||||
"catalog/base.yaml": {Data: []byte("promptkit:\n prompts:\n - id: base\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")},
|
||||
"catalog/original.txt": {Data: []byte("original")},
|
||||
"catalog/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}
|
||||
overrideFiles := fstest.MapFS{
|
||||
"operator/main.yaml": {Data: []byte("promptkit:\n prompts:\n - id: child\n extends: {id: base}\n messages: [{role: user, content_file: replacement.txt}]\n")},
|
||||
"operator/replacement.txt": {Data: []byte("replacement")},
|
||||
}
|
||||
base := configkit.NewFilesystem("base", baseFiles)
|
||||
override := configkit.NewFilesystem("operator", overrideFiles)
|
||||
definition := promptkit.Configure(configkit.Definition[application]{}, func(value *application) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: override.File("operator/main.yaml"), Filesystem: &configkit.FilesystemContext{FS: &override, Root: "."}}, configkit.LayeredOptions[application]{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: base.File("catalog/base.yaml"), Filesystem: &configkit.FilesystemContext{FS: &base, Root: "."}}}}}, Definition: definition,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
|
||||
if !ok {
|
||||
t.Fatal("inspection unavailable")
|
||||
}
|
||||
state, err := configuration.Capture(&result.Config.PromptKit, view, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
child, _, err := state.GetPromptSnapshot(context.Background(), "child")
|
||||
if err != nil || child.Templates[0].Content != "replacement" || child.SchemaFile == nil || child.SchemaFile.Path != "catalog/schema.json" {
|
||||
t.Fatalf("child = %+v, %v", child, err)
|
||||
}
|
||||
data, err := fs.ReadFile(child.SchemaFile.FS, child.SchemaFile.Path)
|
||||
if err != nil || string(data) != `{"type":"object"}` {
|
||||
t.Fatalf("schema = %q, %v", data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapturedInspectionUsesNativeView(t *testing.T) {
|
||||
type anotherApplication struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
Sibling int `yaml:"sibling"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[application]{}, func(value *application) *promptkit.Settings { return &value.PromptKit })
|
||||
first, err := configkit.Load(configkit.Bytes("first", []byte("promptkit: {application_field: first}\n")), configkit.Options[application]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondDefinition := promptkit.Configure(configkit.Definition[anotherApplication]{}, func(value *anotherApplication) *promptkit.Settings { return &value.PromptKit })
|
||||
second, err := configkit.Load(configkit.Bytes("second", []byte("promptkit: {application_field: second}\n")), configkit.Options[anotherApplication]{Definition: secondDefinition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
leftView, _ := first.InspectSubtree(configkit.Keys("promptkit"))
|
||||
rightView, _ := second.InspectSubtree(configkit.Keys("promptkit"))
|
||||
diagnostics := []configkit.Diagnostic{{Message: "warning", Primary: &configkit.Location{SourceID: "first"}, Related: []configkit.Location{{SourceID: "related"}}}}
|
||||
left, err := configuration.Capture(&first.Config.PromptKit, leftView, diagnostics)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
right, err := configuration.Capture(&second.Config.PromptKit, rightView, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
diagnostics[0].Primary.SourceID = "changed"
|
||||
diagnostics[0].Related[0].SourceID = "changed"
|
||||
got := left.Inspection().Diagnostics()
|
||||
if got[0].Primary.SourceID != "first" || got[0].Related[0].SourceID != "related" {
|
||||
t.Fatalf("captured diagnostics = %+v", got)
|
||||
}
|
||||
got[0].Primary.SourceID = "again"
|
||||
got[0].Related[0].SourceID = "again"
|
||||
got = left.Inspection().Diagnostics()
|
||||
if got[0].Primary.SourceID != "first" || got[0].Related[0].SourceID != "related" {
|
||||
t.Fatalf("returned diagnostics aliased: %+v", got)
|
||||
}
|
||||
differences, err := configuration.Diff(left.Inspection(), right.Inspection())
|
||||
if err != nil || len(differences) == 0 || len(left.Inspection().HistoricalPaths()) == 0 {
|
||||
t.Fatalf("diff = %+v, %v", differences, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureOwnsEffectiveResources(t *testing.T) {
|
||||
definition := promptkit.Configure(configkit.Definition[application]{}, func(value *application) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.Load(configkit.Bytes("application", []byte(`other: application
|
||||
promptkit:
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: https://example.test/v1
|
||||
extra_params: {nested: {value: original}}
|
||||
profiles:
|
||||
- id: profile
|
||||
backend: local
|
||||
model: original
|
||||
prompts:
|
||||
- id: prompt
|
||||
inputs: [{name: document, required: false, description: text}]
|
||||
messages: [{role: user, content: hello}]
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema: {type: object, properties: {answer: {type: string}}}
|
||||
`)), configkit.Options[application]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
|
||||
if !ok {
|
||||
t.Fatal("inspection unavailable")
|
||||
}
|
||||
state, err := configuration.Capture(&result.Config.PromptKit, view, result.Diagnostics())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := &result.Config.PromptKit
|
||||
*settings.Backends[0].Endpoint = "https://changed.test/v1"
|
||||
settings.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
||||
*settings.Profiles[0].Model = "changed"
|
||||
*settings.Prompts[0].Messages[0].Content = "changed"
|
||||
settings.Prompts[0].Inputs[0].Required = true
|
||||
settings.Prompts[0].Output.Schema.(map[string]any)["type"] = "changed"
|
||||
backend, err := state.InspectBackend(context.Background(), "local")
|
||||
if err != nil || backend.Endpoint != "https://example.test/v1" || backend.ExtraParams["nested"].(map[string]any)["value"] != "original" {
|
||||
t.Fatalf("backend = %+v, %v", backend, err)
|
||||
}
|
||||
profile, err := state.GetProfile(context.Background(), "profile")
|
||||
if err != nil || profile.ResolvedTarget.Model != "original" {
|
||||
t.Fatalf("profile = %+v, %v", profile, err)
|
||||
}
|
||||
prompt, err := state.InspectPrompt(context.Background(), "prompt")
|
||||
if err != nil || prompt.Messages[0].Content != "hello" || prompt.Output.Schema.(map[string]any)["type"] != "object" || prompt.Inputs == nil || (*prompt.Inputs)[0].Required == nil || *(*prompt.Inputs)[0].Required {
|
||||
t.Fatalf("prompt = %+v, %v", prompt, err)
|
||||
}
|
||||
}
|
||||
@@ -8,24 +8,30 @@ import (
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
// Inspection is the read-only subset of the retained ConfigKit result used by
|
||||
// Inspection is the read-only subset of the native ConfigKit view used by
|
||||
// the facade. It does not expose typed configuration or resource contexts.
|
||||
type Inspection interface {
|
||||
Metadata() configkit.Metadata
|
||||
Sources() []configkit.SourceInfo
|
||||
Diagnostics() []configkit.Diagnostic
|
||||
HistoricalPaths() []configkit.Path
|
||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||
WriteEffective(io.Writer) error
|
||||
WriteExplanation(io.Writer, configkit.Path) error
|
||||
capturedResult() *configkit.Result[document]
|
||||
nativeView() configkit.Inspection
|
||||
}
|
||||
|
||||
type inspectionResult struct {
|
||||
*configkit.Result[document]
|
||||
configkit.Inspection
|
||||
diagnostics []configkit.Diagnostic
|
||||
}
|
||||
|
||||
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
|
||||
return inspection.Result
|
||||
func (inspection inspectionResult) nativeView() configkit.Inspection {
|
||||
return inspection.Inspection
|
||||
}
|
||||
|
||||
func (inspection inspectionResult) Diagnostics() []configkit.Diagnostic {
|
||||
return ownDiagnostics(inspection.diagnostics)
|
||||
}
|
||||
|
||||
// Inspection returns only the retained result's inspection operations.
|
||||
@@ -33,27 +39,25 @@ func (state *loaded) Inspection() Inspection {
|
||||
if state == nil {
|
||||
return nil
|
||||
}
|
||||
return inspectionResult{Result: state.result}
|
||||
return inspectionResult{Inspection: state.view, diagnostics: state.diagnostics}
|
||||
}
|
||||
|
||||
// Diff compares two captured ConfigKit results without reconstructing them.
|
||||
func Diff(left, right Inspection) ([]configkit.Difference, error) {
|
||||
var leftResult, rightResult *configkit.Result[document]
|
||||
if left != nil {
|
||||
leftResult = left.capturedResult()
|
||||
}
|
||||
if right != nil {
|
||||
rightResult = right.capturedResult()
|
||||
}
|
||||
return configkit.Diff(leftResult, rightResult)
|
||||
return left.nativeView().Diff(right.nativeView())
|
||||
}
|
||||
|
||||
func secretProviderPath(path configkit.Path) bool {
|
||||
// Sensitive marks credential-shaped provider values beneath PromptKit settings.
|
||||
func Sensitive(path configkit.Path) bool {
|
||||
pointer := path.String()
|
||||
if pointer == "" {
|
||||
return false
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
|
||||
if len(parts) == 0 || parts[0] != "promptkit" {
|
||||
return false
|
||||
}
|
||||
parts = parts[1:]
|
||||
if !providerExtraParamsPath(parts) {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -12,20 +12,20 @@ func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) {
|
||||
path configkit.Path
|
||||
want bool
|
||||
}{
|
||||
{name: "backend nested secret", path: configkit.Keys("backends").Index(0).Key("extra_params").Key("nested").Key("Token"), want: true},
|
||||
{name: "profile nested secret", path: configkit.Keys("profiles").Index(1).Key("extra_params").Key("authorization"), want: true},
|
||||
{name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends").Index(2).Key("extra_params").Key("nested").Key("Token"), want: true},
|
||||
{name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles").Index(0).Key("extra_params").Key("authorization"), want: true},
|
||||
{name: "backend nested secret", path: configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("nested").Key("Token"), want: true},
|
||||
{name: "profile nested secret", path: configkit.Keys("promptkit", "profiles").Index(1).Key("extra_params").Key("authorization"), want: true},
|
||||
{name: "collection backend nested secret", path: configkit.Keys("promptkit", "collections", "provider", "backends").Index(2).Key("extra_params").Key("nested").Key("Token"), want: true},
|
||||
{name: "collection profile secret", path: configkit.Keys("promptkit", "collections", "assistant", "profiles").Index(0).Key("extra_params").Key("authorization"), want: true},
|
||||
{name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false},
|
||||
{name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false},
|
||||
{name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
|
||||
{name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false},
|
||||
{name: "ordinary provider value", path: configkit.Keys("profiles").Index(0).Key("extra_params").Key("provider").Key("mode"), want: false},
|
||||
{name: "extra params outside resource map", path: configkit.Keys("promptkit", "backends", "provider", "options", "extra_params", "token"), want: false},
|
||||
{name: "collection extra params outside resource map", path: configkit.Keys("promptkit", "collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
|
||||
{name: "missing resource key", path: configkit.Keys("promptkit", "backends", "", "extra_params", "token"), want: false},
|
||||
{name: "ordinary provider value", path: configkit.Keys("promptkit", "profiles").Index(0).Key("extra_params").Key("provider").Key("mode"), want: false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := secretProviderPath(test.path); got != test.want {
|
||||
t.Fatalf("secretProviderPath(%q) = %t, want %t", test.path, got, test.want)
|
||||
if got := Sensitive(test.path); got != test.want {
|
||||
t.Fatalf("Sensitive(%q) = %t, want %t", test.path, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4,11 +4,12 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestTargetInterpretationPreservesComposedNestedValues(t *testing.T) {
|
||||
func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
|
||||
state, err := load(mapping("padded target", map[string]any{
|
||||
"backends": []any{map[string]any{
|
||||
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 1,
|
||||
@@ -32,41 +33,125 @@ func TestTargetInterpretationPreservesComposedNestedValues(t *testing.T) {
|
||||
if err != nil || profile.ResolvedTarget.ServiceTier != "standard" || profile.ResolvedTarget.APIKeyEnv != "TOKEN" {
|
||||
t.Fatalf("interpreted profile = %+v, %v", profile, err)
|
||||
}
|
||||
composed := state.result.Config.Backends[0]
|
||||
if *composed.Endpoint != " HTTPS://example.invalid/v1 " || composed.QueueCapacity != nil ||
|
||||
*composed.SmokeTests.Reachability.Endpoint != " https://example.invalid/health " ||
|
||||
*composed.SmokeTests.Authentication.Kind != " NONE " || *composed.SmokeTests.Authentication.Endpoint != " https://example.invalid/key " ||
|
||||
*state.result.Config.Profiles[0].ServiceTier != " standard " || *state.result.Config.Profiles[0].APIKeyEnv != " TOKEN " {
|
||||
t.Fatalf("composed values were mutated: %+v", state.result.Config)
|
||||
composed := state.backendRecords["local"]
|
||||
if *composed.Endpoint != "https://example.invalid/v1" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 ||
|
||||
*composed.SmokeTests.Reachability.Endpoint != "https://example.invalid/health" ||
|
||||
*composed.SmokeTests.Authentication.Kind != "none" || *composed.SmokeTests.Authentication.Endpoint != "https://example.invalid/key" ||
|
||||
*state.profileRecords["partial"].ServiceTier != "standard" || *state.profileRecords["partial"].APIKeyEnv != "TOKEN" {
|
||||
t.Fatalf("resolved values were not normalized: %+v", composed)
|
||||
}
|
||||
path := configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint")
|
||||
field, found := state.result.Metadata().Lookup(path)
|
||||
if !found || len(field.Origin.Locations) == 0 || field.Origin.Operation == configkit.ChangeNormalization {
|
||||
path := configkit.Keys("promptkit", "backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint")
|
||||
field, found := state.inspection.Metadata().Lookup(path)
|
||||
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||
t.Fatalf("nested field origin changed: %+v", field)
|
||||
}
|
||||
for _, path := range []configkit.Path{
|
||||
configkit.Keys("promptkit", "backends").Index(0).Key("endpoint"),
|
||||
configkit.Keys("promptkit", "backends").Index(0).Key("smoke_tests").Key("reachability").Key("endpoint"),
|
||||
configkit.Keys("promptkit", "backends").Index(0).Key("smoke_tests").Key("authentication").Key("kind"),
|
||||
configkit.Keys("promptkit", "profiles").Index(0).Key("service_tier"),
|
||||
configkit.Keys("promptkit", "profiles").Index(0).Key("api_key_env"),
|
||||
} {
|
||||
field, found := state.inspection.Metadata().Lookup(path)
|
||||
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||
t.Errorf("normalization at %v: %+v", path, field)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptInterpretationPreservesComposedOutputAndRejectsDuplicateNames(t *testing.T) {
|
||||
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
|
||||
definition := map[string]any{
|
||||
"id": "example", "inputs": []any{map[string]any{"name": " document "}},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": " schema.json "},
|
||||
}
|
||||
state, err := load(mapping("prompt metadata", map[string]any{"prompts": []any{definition}}), nil)
|
||||
filesystem := configkit.NewFilesystem("schemas", fstest.MapFS{})
|
||||
input := mapping("prompt metadata", map[string]any{"schema_root": ".", "prompts": []any{definition}})
|
||||
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}
|
||||
state, err := load(input, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, inspection, err := state.prompt("example")
|
||||
if err != nil || selected.Validation.SchemaPath != "schema.json" || inspection.OutputContract.SchemaPath != "schema.json" || selected.Inputs[0].Name != "document" {
|
||||
if err != nil || selected.Validation.SchemaPath != "schema.json" || selected.Inputs[0].Name != "document" {
|
||||
t.Fatalf("interpreted prompt = %+v, %+v, %v", selected, inspection, err)
|
||||
}
|
||||
composed := state.result.Config.Prompts[0]
|
||||
if *composed.Output.SchemaPath != " schema.json " || composed.Inputs[0].Name != " document " {
|
||||
t.Fatalf("composed prompt changed: %+v", composed)
|
||||
composed := state.prompts["example"].resource
|
||||
if *composed.Output.SchemaPath != "schema.json" || composed.Inputs[0].Name != "document" {
|
||||
t.Fatalf("prompt was not normalized: %+v", composed)
|
||||
}
|
||||
definition["inputs"] = []any{map[string]any{"name": " document "}, map[string]any{"name": "document"}}
|
||||
_, err = load(mapping("duplicate inputs", map[string]any{"prompts": []any{definition}}), nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "/prompts/0/inputs/1") || !strings.Contains(err.Error(), "duplicated") {
|
||||
if err == nil || !strings.Contains(err.Error(), "/promptkit/prompts/0/inputs/1") || !strings.Contains(err.Error(), "duplicated") {
|
||||
t.Fatalf("duplicate normalized names were not rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptResolutionRecordsMessageFields(t *testing.T) {
|
||||
loadPrompt := func(role, cacheType, ttl string) *testLoaded {
|
||||
t.Helper()
|
||||
state, err := load(mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||
"id": "example", "inputs": []any{map[string]any{"name": " input ", "content_type": nil, "description": " text "}},
|
||||
"messages": []any{map[string]any{"role": role, "content": "", "cache_control": map[string]any{"type": cacheType, "ttl": ttl}}},
|
||||
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||
}}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return state
|
||||
}
|
||||
padded := loadPrompt(" USER ", " ephemeral ", " 1h ")
|
||||
canonical := loadPrompt("user", "ephemeral", "1h")
|
||||
definition, inspection, err := padded.prompt("example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, otherInspection, err := canonical.prompt("example")
|
||||
if err != nil || inspection.PromptHash != otherInspection.PromptHash || definition.Templates[0].Role != other.Templates[0].Role || definition.Templates[0].Content != "" {
|
||||
t.Fatalf("equivalent prompt mismatch: %+v %+v %v", definition, other, err)
|
||||
}
|
||||
base := configkit.Keys("promptkit", "prompts").Index(0)
|
||||
for _, path := range []configkit.Path{base.Key("inputs").Index(0).Key("name"), base.Key("inputs").Index(0).Key("description"), base.Key("messages").Index(0).Key("role"), base.Key("messages").Index(0).Key("cache_control").Key("type"), base.Key("messages").Index(0).Key("cache_control").Key("ttl")} {
|
||||
field, found := padded.inspection.Metadata().Lookup(path)
|
||||
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||
t.Errorf("normalization at %v: %+v", path, field)
|
||||
}
|
||||
}
|
||||
field, ok := padded.inspection.Metadata().Lookup(base.Key("inputs").Index(0).Key("content_type"))
|
||||
if !ok || !field.Null {
|
||||
t.Fatal("null input metadata was not retained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolutionRecordsTargetDependencies(t *testing.T) {
|
||||
backend := func(id string) map[string]any {
|
||||
return map[string]any{"id": id, "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 2, "queue_capacity": nil}
|
||||
}
|
||||
state, err := load(mapping("targets", map[string]any{
|
||||
"backends": []any{backend("root")},
|
||||
"collections": map[string]any{"scoped": map[string]any{"backends": []any{backend("local")}}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, base := range []configkit.Path{configkit.Keys("promptkit", "backends").Index(0), configkit.Keys("promptkit", "collections", "scoped", "backends").Index(0)} {
|
||||
endpoint, ok := state.inspection.Explain(base.Key("endpoint"))
|
||||
if !ok || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") {
|
||||
t.Fatalf("endpoint explanation: %+v", endpoint)
|
||||
}
|
||||
history := state.inspection.Metadata().History(base.Key("queue_capacity"))
|
||||
found := false
|
||||
for _, change := range history {
|
||||
if change.Origin.Operation == configkit.ChangeDerivedDefault {
|
||||
for _, dependency := range change.Dependencies {
|
||||
if dependency.String() == base.Key("concurrency_limit").String() {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("missing queue dependency: %+v", history)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,36 +2,66 @@ package configuration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// IdentityFilter selects all identities, one namespace, or unqualified IDs.
|
||||
type IdentityFilter struct {
|
||||
All bool
|
||||
Namespace string
|
||||
}
|
||||
|
||||
func selectIdentities[T any](ctx context.Context, entries []T, filter IdentityFilter, id func(T) string) ([]T, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if filter.All {
|
||||
result := append([]T(nil), entries...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
if filter.Namespace != "" {
|
||||
prefix := filter.Namespace + "/"
|
||||
start := sort.Search(len(entries), func(i int) bool { return id(entries[i]) >= prefix })
|
||||
end := start + sort.Search(len(entries)-start, func(i int) bool { return !strings.HasPrefix(id(entries[start+i]), prefix) })
|
||||
result := append([]T(nil), entries[start:end]...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
var result []T
|
||||
for _, entry := range entries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !strings.Contains(id(entry), "/") {
|
||||
result = append(result, entry)
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CollectionIdentity is the captured metadata for one final collection entry.
|
||||
type CollectionIdentity struct {
|
||||
Namespace string
|
||||
ReleaseVersion *string
|
||||
}
|
||||
|
||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
profiles := append([]string(nil), state.profileIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return profiles, nil
|
||||
func (state *loaded) ListProfiles(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||
return selectIdentities(ctx, state.profileIDs, filter, func(id string) string { return id })
|
||||
}
|
||||
|
||||
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backends := append([]string(nil), state.backendIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return backends, nil
|
||||
func (state *loaded) ListBackends(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||
return selectIdentities(ctx, state.backendIDs, filter, func(id string) string { return id })
|
||||
}
|
||||
|
||||
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||
@@ -72,13 +102,6 @@ func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity,
|
||||
return collections, nil
|
||||
}
|
||||
|
||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return prompts, nil
|
||||
func (state *loaded) ListPrompts(ctx context.Context, filter IdentityFilter) ([]domain.PromptIdentity, error) {
|
||||
return selectIdentities(ctx, state.promptIDs, filter, func(entry domain.PromptIdentity) string { return entry.ID })
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
state, err := load(mapping("identities", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": "http://localhost:8080/v1"}},
|
||||
"collections": map[string]any{
|
||||
"z": map[string]any{"profiles": []any{map[string]any{"id": "profile", "backend": "local", "model": "z-model"}}, "prompts": []any{listingPromptDefinition("prompt")}},
|
||||
@@ -23,7 +23,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends, err := state.ListBackends(context.Background())
|
||||
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -32,7 +32,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||
}
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLoadedListsAreIndependent(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
state, err := load(mapping("identities", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.test/v1"}},
|
||||
"profiles": []any{map[string]any{"id": "profile"}},
|
||||
"prompts": []any{listingPromptDefinition("example")},
|
||||
@@ -65,36 +65,36 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles = append(profiles, "caller profile")
|
||||
profiles[0] = "changed profile"
|
||||
backends, err := state.ListBackends(context.Background())
|
||||
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends[0] = "changed backend"
|
||||
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts[0].ID = "changed prompt"
|
||||
|
||||
profiles, err = state.ListProfiles(context.Background())
|
||||
profiles, err = state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||
}
|
||||
backends, err = state.ListBackends(context.Background())
|
||||
backends, err = state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
||||
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
||||
}
|
||||
prompts, err = state.ListPrompts(context.Background())
|
||||
prompts, err = state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -104,27 +104,73 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsSelectNamespaces(t *testing.T) {
|
||||
ids := []string{"a", "a-b/one", "a/one", "a/two", "b", "z/one"}
|
||||
state := &loaded{profileIDs: append([]string(nil), ids...), backendIDs: append([]string(nil), ids...)}
|
||||
for _, id := range ids {
|
||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
filter IdentityFilter
|
||||
want []string
|
||||
}{
|
||||
{"all", IdentityFilter{All: true}, ids},
|
||||
{"unqualified", IdentityFilter{}, []string{"a", "b"}},
|
||||
{"namespace", IdentityFilter{Namespace: "a"}, []string{"a/one", "a/two"}},
|
||||
{"prefix sharing", IdentityFilter{Namespace: "a-b"}, []string{"a-b/one"}},
|
||||
{"missing", IdentityFilter{Namespace: "absent"}, nil},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
profiles, err := state.ListProfiles(context.Background(), tc.filter)
|
||||
if err != nil || !reflect.DeepEqual(profiles, tc.want) {
|
||||
t.Fatalf("profiles = %v, %v; want %v", profiles, err, tc.want)
|
||||
}
|
||||
backends, err := state.ListBackends(context.Background(), tc.filter)
|
||||
if err != nil || !reflect.DeepEqual(backends, tc.want) {
|
||||
t.Fatalf("backends = %v, %v; want %v", backends, err, tc.want)
|
||||
}
|
||||
prompts, err := state.ListPrompts(context.Background(), tc.filter)
|
||||
if err != nil || len(prompts) != len(tc.want) {
|
||||
t.Fatalf("prompts = %v, %v; want %v", prompts, err, tc.want)
|
||||
}
|
||||
for i := range prompts {
|
||||
if prompts[i].ID != tc.want[i] {
|
||||
t.Fatalf("prompt ID = %q; want %q", prompts[i].ID, tc.want[i])
|
||||
}
|
||||
}
|
||||
if len(profiles) > 0 {
|
||||
profiles[0], backends[0], prompts[0].ID = "changed", "changed", "changed"
|
||||
}
|
||||
})
|
||||
}
|
||||
if !reflect.DeepEqual(state.profileIDs, ids) || !reflect.DeepEqual(state.backendIDs, ids) || state.promptIDs[0].ID != ids[0] {
|
||||
t.Fatal("caller mutation changed captured identities")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||
state, err := load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles, err := state.ListProfiles(context.Background())
|
||||
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil || len(profiles) != 0 {
|
||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
backends, err := state.ListBackends(context.Background())
|
||||
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil || len(backends) != 0 {
|
||||
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(context.Background())
|
||||
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
||||
state, err := Load(mapping("identities", map[string]any{
|
||||
state, err := load(mapping("identities", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "partial"}},
|
||||
"prompts": []any{listingPromptDefinition("example")},
|
||||
}), nil)
|
||||
@@ -134,17 +180,19 @@ func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
profiles, err := state.ListProfiles(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
backends, err := state.ListBackends(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
||||
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(ctx)
|
||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||
for _, filter := range []IdentityFilter{{All: true}, {}, {Namespace: "example"}} {
|
||||
profiles, err := state.ListProfiles(ctx, filter)
|
||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||
}
|
||||
backends, err := state.ListBackends(ctx, filter)
|
||||
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
||||
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
||||
}
|
||||
prompts, err := state.ListPrompts(ctx, filter)
|
||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ package configuration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -14,118 +13,26 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||
"go.maximumdirect.net/promptkit/internal/llm"
|
||||
"go.maximumdirect.net/promptkit/internal/settings"
|
||||
)
|
||||
|
||||
type document struct {
|
||||
Version *string `yaml:"version"`
|
||||
Backends []backendConfig `yaml:"backends"`
|
||||
Profiles []profileConfig `yaml:"profiles"`
|
||||
Prompts []promptConfig `yaml:"prompts"`
|
||||
Collections map[string]collectionConfig `yaml:"collections"`
|
||||
}
|
||||
|
||||
type collectionConfig struct {
|
||||
ReleaseVersion *string `yaml:"release_version"`
|
||||
Backends []backendConfig `yaml:"backends"`
|
||||
Profiles []profileConfig `yaml:"profiles"`
|
||||
Prompts []promptConfig `yaml:"prompts"`
|
||||
}
|
||||
|
||||
type backendConfig struct {
|
||||
ID string `yaml:"id"`
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
APIKeyEnv *string `yaml:"api_key_env"`
|
||||
ConcurrencyLimit *int `yaml:"concurrency_limit"`
|
||||
QueueCapacity *int `yaml:"queue_capacity"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
SmokeTests *backendSmokeTestConfig `yaml:"smoke_tests"`
|
||||
}
|
||||
|
||||
type backendSmokeTestConfig struct {
|
||||
DefaultProfile *string `yaml:"default_profile"`
|
||||
Reachability *reachabilityProbeConfig `yaml:"reachability"`
|
||||
Authentication *authenticationProbeConfig `yaml:"authentication"`
|
||||
}
|
||||
|
||||
type reachabilityProbeConfig struct {
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
}
|
||||
|
||||
type authenticationProbeConfig struct {
|
||||
Kind *string `yaml:"kind"`
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
}
|
||||
|
||||
type profileConfig struct {
|
||||
ID string `yaml:"id"`
|
||||
Backend *string `yaml:"backend"`
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
Model *string `yaml:"model"`
|
||||
Temperature *float64 `yaml:"temperature"`
|
||||
TopP *float64 `yaml:"top_p"`
|
||||
MaxTokens *int `yaml:"max_tokens"`
|
||||
TimeoutSeconds *int `yaml:"timeout_seconds"`
|
||||
ServiceTier *string `yaml:"service_tier"`
|
||||
ReasoningEffort *string `yaml:"reasoning_effort"`
|
||||
APIKeyEnv *string `yaml:"api_key_env"`
|
||||
APIKeyRequired *bool `yaml:"api_key_required"`
|
||||
ExtraParams map[string]any `yaml:"extra_params"`
|
||||
}
|
||||
|
||||
type promptConfig struct {
|
||||
ID string `yaml:"id"`
|
||||
DefaultProfile *string `yaml:"default_profile"`
|
||||
Description *string `yaml:"description"`
|
||||
SessionID *string `yaml:"session_id"`
|
||||
Inputs []promptInputConfig `yaml:"inputs"`
|
||||
Messages []promptMessageConfig `yaml:"messages"`
|
||||
Output *promptOutputConfig `yaml:"output"`
|
||||
}
|
||||
|
||||
type promptInputConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Required bool `yaml:"required"`
|
||||
ContentType *string `yaml:"content_type"`
|
||||
Description *string `yaml:"description"`
|
||||
}
|
||||
|
||||
type promptMessageConfig struct {
|
||||
Role *string `yaml:"role"`
|
||||
Content *string `yaml:"content"`
|
||||
ContentFile *string `yaml:"content_file"`
|
||||
CacheControl *cacheControlConfig `yaml:"cache_control"`
|
||||
}
|
||||
|
||||
type cacheControlConfig struct {
|
||||
Type *string `yaml:"type"`
|
||||
TTL *string `yaml:"ttl"`
|
||||
}
|
||||
|
||||
type promptOutputConfig struct {
|
||||
Format *string `yaml:"format"`
|
||||
ValidationMode *string `yaml:"validation_mode"`
|
||||
SchemaPath *string `yaml:"schema_path"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
}
|
||||
|
||||
// fieldState distinguishes omission, an explicit value, and an explicit null.
|
||||
// Inherited values are explicit because ConfigKit has already expanded them.
|
||||
type fieldState uint8
|
||||
|
||||
const (
|
||||
fieldAbsent fieldState = iota
|
||||
fieldValue
|
||||
fieldNull
|
||||
)
|
||||
type collectionConfig = settings.CollectionSettings
|
||||
type backendConfig = settings.BackendSettings
|
||||
type profileConfig = settings.ProfileSettings
|
||||
type promptConfig = settings.PromptSettings
|
||||
type promptInputConfig = settings.PromptInputSettings
|
||||
type promptMessageConfig = settings.PromptMessageSettings
|
||||
type cacheControlConfig = settings.PromptCacheControlSettings
|
||||
type promptOutputConfig = settings.PromptOutputSettings
|
||||
|
||||
type loaded struct {
|
||||
result *configkit.Result[document]
|
||||
interpreted document
|
||||
backends map[string]map[string]fieldState
|
||||
profiles map[string]map[string]fieldState
|
||||
promptMetadata map[string][]promptInputPresence
|
||||
view configkit.Inspection
|
||||
diagnostics []configkit.Diagnostic
|
||||
promptRequired map[string][]bool
|
||||
prompts map[string]*promptEntry
|
||||
registry *backend.Registry
|
||||
backendRecords map[string]backendConfig
|
||||
profileRecords map[string]profileConfig
|
||||
indexedProfiles map[string]domain.ExecutionProfile
|
||||
backendIDs []string
|
||||
profileIDs []string
|
||||
@@ -134,6 +41,9 @@ type loaded struct {
|
||||
collections []CollectionIdentity
|
||||
}
|
||||
|
||||
// Snapshot holds configuration resources captured for engine assembly.
|
||||
type Snapshot = loaded
|
||||
|
||||
type resourceBases struct {
|
||||
backends configkit.Path
|
||||
profiles configkit.Path
|
||||
@@ -151,20 +61,20 @@ type resourceSet struct {
|
||||
|
||||
func rootResourceBases() resourceBases {
|
||||
return resourceBases{
|
||||
backends: configkit.Keys("backends"),
|
||||
profiles: configkit.Keys("profiles"),
|
||||
prompts: configkit.Keys("prompts"),
|
||||
backends: configkit.Keys("promptkit", "backends"),
|
||||
profiles: configkit.Keys("promptkit", "profiles"),
|
||||
prompts: configkit.Keys("promptkit", "prompts"),
|
||||
}
|
||||
}
|
||||
|
||||
func resourceSets(config *document) []resourceSet {
|
||||
func resourceSets(config *settings.Settings) []resourceSet {
|
||||
sets := make([]resourceSet, 0, len(config.Collections)+1)
|
||||
sets = append(sets, resourceSet{
|
||||
bases: rootResourceBases(), backends: config.Backends, profiles: config.Profiles, prompts: config.Prompts,
|
||||
})
|
||||
for _, namespace := range sortedKeys(config.Collections) {
|
||||
collection := config.Collections[namespace]
|
||||
base := configkit.Keys("collections", namespace)
|
||||
base := configkit.Keys("promptkit", "collections", namespace)
|
||||
sets = append(sets, resourceSet{
|
||||
isCollection: true,
|
||||
namespace: namespace,
|
||||
@@ -192,7 +102,7 @@ func indexedResourceID(namespace, localKey string) string {
|
||||
return namespace + "/" + localKey
|
||||
}
|
||||
|
||||
func indexedBackends(config *document) map[string]backendConfig {
|
||||
func indexedBackends(config *settings.Settings) map[string]backendConfig {
|
||||
backends := make(map[string]backendConfig)
|
||||
for _, resources := range resourceSets(config) {
|
||||
for _, definition := range resources.backends {
|
||||
@@ -202,7 +112,7 @@ func indexedBackends(config *document) map[string]backendConfig {
|
||||
return backends
|
||||
}
|
||||
|
||||
func indexedProfiles(config *document) map[string]profileConfig {
|
||||
func indexedProfiles(config *settings.Settings) map[string]profileConfig {
|
||||
profiles := make(map[string]profileConfig)
|
||||
for _, resources := range resourceSets(config) {
|
||||
for _, definition := range resources.profiles {
|
||||
@@ -214,67 +124,61 @@ func indexedProfiles(config *document) map[string]profileConfig {
|
||||
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
|
||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
||||
|
||||
// Load captures one immutable effective configuration for internal consumers.
|
||||
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
return load(main, underlays)
|
||||
// SequenceIdentities returns the component's native sequence registrations.
|
||||
func SequenceIdentities() []configkit.SequenceIdentity {
|
||||
return []configkit.SequenceIdentity{
|
||||
configkit.SequenceIdentityAt(configkit.Keys("promptkit", "backends"), "id"),
|
||||
configkit.SequenceIdentityAt(configkit.Keys("promptkit", "profiles"), "id"),
|
||||
configkit.SequenceIdentityAt(configkit.Keys("promptkit", "prompts"), "id"),
|
||||
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("promptkit"), configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("backends")}, "id"),
|
||||
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("promptkit"), configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("profiles")}, "id"),
|
||||
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("promptkit"), configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("prompts")}, "id"),
|
||||
}
|
||||
}
|
||||
|
||||
func configurationDefinition(state *loaded) configkit.Definition[document] {
|
||||
return configkit.Definition[document]{
|
||||
SequenceIdentities: []configkit.SequenceIdentity{
|
||||
configkit.SequenceIdentityAt(configkit.Keys("backends"), "id"),
|
||||
configkit.SequenceIdentityAt(configkit.Keys("profiles"), "id"),
|
||||
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
|
||||
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("backends")}, "id"),
|
||||
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("profiles")}, "id"),
|
||||
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("prompts")}, "id"),
|
||||
},
|
||||
Version: &configkit.VersionRule{
|
||||
Path: configkit.Keys("version"),
|
||||
Check: checkConfigurationVersion,
|
||||
},
|
||||
Sensitive: secretProviderPath,
|
||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||
for _, resources := range resourceSets(config) {
|
||||
captureFields(resources.backends, func(v backendConfig) string { return v.ID }, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
|
||||
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
||||
// Resolve normalizes the selected effective settings in place.
|
||||
func Resolve(config *settings.Settings, resolution *configkit.Resolution) {
|
||||
if config == nil {
|
||||
return
|
||||
}
|
||||
for _, resources := range resourceSets(config) {
|
||||
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||
resolveProfiles(resources.profiles, resources.bases.profiles, resolution)
|
||||
resolvePrompts(resources.prompts, resources.bases.prompts, resolution)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate checks the selected effective settings.
|
||||
func Validate(config *settings.Settings, validation *configkit.Validation) {
|
||||
if config != nil {
|
||||
validateEffective(config, validation)
|
||||
}
|
||||
}
|
||||
|
||||
// Capture builds owned runtime resources from effective settings and native evidence.
|
||||
func Capture(input *settings.Settings, view configkit.Inspection, diagnostics []configkit.Diagnostic) (*loaded, error) {
|
||||
config := ownSettings(input)
|
||||
state := &loaded{view: view, diagnostics: ownDiagnostics(diagnostics), promptRequired: make(map[string][]bool)}
|
||||
metadata := view.Metadata()
|
||||
for _, resources := range resourceSets(&config) {
|
||||
for index, entry := range resources.prompts {
|
||||
presence := make([]bool, len(entry.Inputs))
|
||||
for i := range entry.Inputs {
|
||||
path := resources.bases.prompts.Index(index).Key("inputs").Index(i)
|
||||
// Typed values capture defaults; explicitness preserves source false.
|
||||
// IsExplicit avoids projecting inheritance origins for a flag.
|
||||
presence[i] = entry.Inputs[i].Required || metadata.IsExplicit(path.Key("required"))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Validate: func(config *document, validation *configkit.Validation) {
|
||||
state.interpreted = interpretConfiguration(config, validation)
|
||||
validateEffective(&state.interpreted, validation)
|
||||
},
|
||||
state.promptRequired[indexedResourceID(resources.namespace, entry.ID)] = presence
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||
state := &loaded{
|
||||
backends: make(map[string]map[string]fieldState),
|
||||
profiles: make(map[string]map[string]fieldState),
|
||||
promptMetadata: make(map[string][]promptInputPresence),
|
||||
}
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||
Underlays: underlays,
|
||||
SourceVersionMode: configkit.SourceVersionModeIfPresent,
|
||||
Definition: configurationDefinition(state),
|
||||
})
|
||||
if err != nil {
|
||||
if err := state.materializePrompts(&config, metadata); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
state.result = result
|
||||
if err := state.materializePrompts(); err != nil {
|
||||
if err := state.buildExecutionProfiles(&config); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
if err := state.buildExecutionProfiles(); err != nil {
|
||||
return nil, fmt.Errorf("load configuration: %w", err)
|
||||
}
|
||||
state.backendIDs = sortedKeys(indexedBackends(&state.interpreted))
|
||||
state.backendIDs = sortedKeys(indexedBackends(&config))
|
||||
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||
for _, id := range sortedKeys(state.prompts) {
|
||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
|
||||
@@ -284,8 +188,8 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
"profile": resourceNamespaces(state.profileIDs),
|
||||
"prompt": promptNamespaces(state.promptIDs),
|
||||
}
|
||||
for _, namespace := range sortedKeys(result.Config.Collections) {
|
||||
collection := result.Config.Collections[namespace]
|
||||
for _, namespace := range sortedKeys(config.Collections) {
|
||||
collection := config.Collections[namespace]
|
||||
state.collections = append(state.collections, CollectionIdentity{
|
||||
Namespace: namespace, ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
|
||||
})
|
||||
@@ -293,6 +197,18 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func ownDiagnostics(values []configkit.Diagnostic) []configkit.Diagnostic {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
result := append([]configkit.Diagnostic(nil), values...)
|
||||
for i := range result {
|
||||
result[i].Primary = copyPointer(result[i].Primary)
|
||||
result[i].Related = append([]configkit.Location(nil), result[i].Related...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func resourceNamespaces(ids []string) []string {
|
||||
namespaces := make(map[string]struct{})
|
||||
for _, id := range ids {
|
||||
@@ -311,64 +227,26 @@ func promptNamespaces(prompts []domain.PromptIdentity) []string {
|
||||
return resourceNamespaces(ids)
|
||||
}
|
||||
|
||||
func checkConfigurationVersion(version configkit.Version, validation *configkit.Validation) error {
|
||||
if !version.Present {
|
||||
return nil
|
||||
}
|
||||
if version.Null || version.Text != "2" {
|
||||
validation.Error(configkit.Keys("version"), "unsupported configuration specification version")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type promptInputPresence struct {
|
||||
contentType fieldState
|
||||
description fieldState
|
||||
}
|
||||
|
||||
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
|
||||
for promptIndex, prompt := range prompts {
|
||||
inputs := prompt.Inputs
|
||||
states := make([]promptInputPresence, len(inputs))
|
||||
base := basePath.Index(promptIndex).Key("inputs")
|
||||
for index := range inputs {
|
||||
path := base.Index(index)
|
||||
states[index] = promptInputPresence{
|
||||
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||
description: captureFieldState(path.Key("description"), resolution),
|
||||
}
|
||||
// eligibleResourceList leaves invalid identities to the existing validation
|
||||
// boundaries, without mutating a list that cannot be recorded by identity.
|
||||
func eligibleResourceList[T any](entries []T, identity func(T) string) bool {
|
||||
seen := make(map[string]struct{}, len(entries))
|
||||
for _, entry := range entries {
|
||||
id := identity(entry)
|
||||
if !domain.IsValidLocalResourceKey(id) {
|
||||
return false
|
||||
}
|
||||
destination[indexedResourceID(namespace, prompt.ID)] = states
|
||||
}
|
||||
}
|
||||
|
||||
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
|
||||
info, exists := resolution.EntryInfo(path)
|
||||
switch {
|
||||
case !exists || !info.Explicit:
|
||||
return fieldAbsent
|
||||
case info.Null:
|
||||
return fieldNull
|
||||
default:
|
||||
return fieldValue
|
||||
}
|
||||
}
|
||||
|
||||
func captureFields[T any](entries []T, identity func(T) string, basePath configkit.Path, namespace string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
||||
for index, entry := range entries {
|
||||
states := make(map[string]fieldState, len(fields))
|
||||
for _, field := range fields {
|
||||
info, exists := resolution.EntryInfo(basePath.Index(index).Key(field))
|
||||
switch {
|
||||
case !exists || !info.Explicit:
|
||||
states[field] = fieldAbsent
|
||||
case info.Null:
|
||||
states[field] = fieldNull
|
||||
default:
|
||||
states[field] = fieldValue
|
||||
}
|
||||
if _, exists := seen[id]; exists {
|
||||
return false
|
||||
}
|
||||
destination[indexedResourceID(namespace, identity(entry))] = states
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func recordStringNormalization(value **string, path configkit.Path, resolution *configkit.Resolution, normalized string) {
|
||||
if *value != nil && **value != normalized {
|
||||
configkit.Normalize(resolution, path, value, &normalized, "Normalize configured value")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,119 +259,12 @@ func sortedKeys[T any](entries map[string]T) []string {
|
||||
return keys
|
||||
}
|
||||
|
||||
func interpretConfiguration(config *document, validation *configkit.Validation) document {
|
||||
interpreted := *config
|
||||
interpreted.Backends = interpretBackends(config.Backends, configkit.Keys("backends"), validation)
|
||||
interpreted.Profiles = interpretProfiles(config.Profiles, configkit.Keys("profiles"), validation)
|
||||
interpreted.Prompts = interpretPrompts(config.Prompts)
|
||||
if config.Collections != nil {
|
||||
interpreted.Collections = make(map[string]collectionConfig, len(config.Collections))
|
||||
for namespace, original := range config.Collections {
|
||||
collection := original
|
||||
base := configkit.Keys("collections", namespace)
|
||||
collection.Backends = interpretBackends(original.Backends, base.Key("backends"), validation)
|
||||
collection.Profiles = interpretProfiles(original.Profiles, base.Key("profiles"), validation)
|
||||
collection.Prompts = interpretPrompts(original.Prompts)
|
||||
interpreted.Collections[namespace] = collection
|
||||
}
|
||||
}
|
||||
return interpreted
|
||||
}
|
||||
|
||||
func interpretBackends(source []backendConfig, basePath configkit.Path, validation *configkit.Validation) []backendConfig {
|
||||
if source == nil {
|
||||
return nil
|
||||
}
|
||||
backends := append([]backendConfig{}, source...)
|
||||
for index := range backends {
|
||||
backend := &backends[index]
|
||||
base := basePath.Index(index)
|
||||
if backend.Endpoint != nil {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
|
||||
if err != nil {
|
||||
validation.Error(base.Key("endpoint"), err.Error())
|
||||
} else {
|
||||
backend.Endpoint = &endpoint
|
||||
}
|
||||
}
|
||||
if backend.APIKeyEnv != nil {
|
||||
value := strings.TrimSpace(*backend.APIKeyEnv)
|
||||
backend.APIKeyEnv = &value
|
||||
}
|
||||
if backend.SmokeTests != nil {
|
||||
smokeTests := *backend.SmokeTests
|
||||
backend.SmokeTests = &smokeTests
|
||||
smokePath := base.Key("smoke_tests")
|
||||
if smokeTests.Reachability != nil {
|
||||
reachability := *smokeTests.Reachability
|
||||
smokeTests.Reachability = &reachability
|
||||
normalizeDiagnosticEndpoint(&reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), validation)
|
||||
}
|
||||
if smokeTests.Authentication != nil {
|
||||
authentication := *smokeTests.Authentication
|
||||
smokeTests.Authentication = &authentication
|
||||
if authentication.Kind != nil {
|
||||
kind := strings.ToLower(strings.TrimSpace(*authentication.Kind))
|
||||
authentication.Kind = &kind
|
||||
}
|
||||
normalizeDiagnosticEndpoint(&authentication.Endpoint, smokePath.Key("authentication").Key("endpoint"), validation)
|
||||
}
|
||||
}
|
||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
|
||||
capacity := 1024
|
||||
backend.QueueCapacity = &capacity
|
||||
}
|
||||
}
|
||||
return backends
|
||||
}
|
||||
|
||||
func normalizeDiagnosticEndpoint(value **string, path configkit.Path, validation *configkit.Validation) {
|
||||
if value == nil || *value == nil {
|
||||
return
|
||||
}
|
||||
normalized := strings.TrimSpace(**value)
|
||||
request, err := http.NewRequest(http.MethodGet, normalized, nil)
|
||||
if normalized == "" || err != nil || request.URL == nil ||
|
||||
(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
|
||||
request.URL.Hostname() == "" {
|
||||
validation.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
|
||||
return
|
||||
}
|
||||
*value = &normalized
|
||||
}
|
||||
|
||||
func interpretProfiles(source []profileConfig, basePath configkit.Path, validation *configkit.Validation) []profileConfig {
|
||||
if source == nil {
|
||||
return nil
|
||||
}
|
||||
profiles := append([]profileConfig{}, source...)
|
||||
for index := range profiles {
|
||||
profile := &profiles[index]
|
||||
base := basePath.Index(index)
|
||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
|
||||
if err != nil {
|
||||
validation.Error(base.Key("endpoint"), err.Error())
|
||||
} else {
|
||||
profile.Endpoint = &endpoint
|
||||
}
|
||||
}
|
||||
for _, value := range []**string{&profile.ServiceTier, &profile.ReasoningEffort, &profile.APIKeyEnv} {
|
||||
if *value != nil {
|
||||
normalized := strings.TrimSpace(**value)
|
||||
*value = &normalized
|
||||
}
|
||||
}
|
||||
}
|
||||
return profiles
|
||||
}
|
||||
|
||||
func validateEffective(config *document, validation *configkit.Validation) {
|
||||
func validateEffective(config *settings.Settings, validation *configkit.Validation) {
|
||||
sets := resourceSets(config)
|
||||
backendIDs := make(map[string]struct{})
|
||||
for _, resources := range sets {
|
||||
if resources.isCollection && !domain.IsValidResourceNamespace(resources.namespace) {
|
||||
validation.Error(configkit.Keys("collections", resources.namespace), "collection namespace must be nonblank and contain no slash")
|
||||
validation.Error(configkit.Keys("promptkit", "collections", resources.namespace), "collection namespace must be nonblank and contain no slash")
|
||||
}
|
||||
for _, backend := range resources.backends {
|
||||
backendIDs[indexedResourceID(resources.namespace, backend.ID)] = struct{}{}
|
||||
@@ -501,95 +272,110 @@ func validateEffective(config *document, validation *configkit.Validation) {
|
||||
}
|
||||
for _, resources := range sets {
|
||||
for index, backend := range resources.backends {
|
||||
base := resources.bases.backends.Index(index)
|
||||
if !domain.IsValidLocalResourceKey(backend.ID) {
|
||||
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
||||
}
|
||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||
}
|
||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||
}
|
||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
||||
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
||||
}
|
||||
if backend.QueueCapacity != nil {
|
||||
if *backend.QueueCapacity < 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
||||
}
|
||||
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
||||
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
||||
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
||||
}
|
||||
}
|
||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||
smokePath := base.Key("smoke_tests")
|
||||
if smokeTests.DefaultProfile != nil {
|
||||
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
||||
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
||||
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
||||
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
||||
}
|
||||
}
|
||||
if authentication := smokeTests.Authentication; authentication != nil {
|
||||
kindPath := smokePath.Key("authentication").Key("kind")
|
||||
if authentication.Kind == nil {
|
||||
validation.Error(kindPath, "authentication kind is required")
|
||||
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||
}
|
||||
}
|
||||
}
|
||||
validateBackendRecord(backend, resources.bases.backends.Index(index), validation)
|
||||
}
|
||||
for index, profile := range resources.profiles {
|
||||
base := resources.bases.profiles.Index(index)
|
||||
if !domain.IsValidLocalResourceKey(profile.ID) {
|
||||
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
||||
}
|
||||
if profile.Backend != nil && *profile.Backend != "" {
|
||||
if !domain.IsValidResourceID(*profile.Backend) {
|
||||
validation.Error(base.Key("backend"), "backend reference is malformed")
|
||||
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
||||
validation.Error(base.Key("backend"), "backend is not defined")
|
||||
}
|
||||
}
|
||||
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
||||
validation.Error(base.Key("model"), "model must not be blank")
|
||||
}
|
||||
if profile.Temperature != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
||||
validation.Error(base.Key("temperature"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TopP != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
||||
validation.Error(base.Key("top_p"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.MaxTokens != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
||||
validation.Error(base.Key("max_tokens"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TimeoutSeconds != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
||||
validation.Error(base.Key("timeout_seconds"), err.Error())
|
||||
}
|
||||
}
|
||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||
validateProfileRecord(profile, resources.bases.profiles.Index(index), backendIDs, validation)
|
||||
}
|
||||
for index, prompt := range resources.prompts {
|
||||
base := resources.bases.prompts.Index(index)
|
||||
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
||||
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
||||
}
|
||||
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
||||
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
||||
validatePromptRecord(prompt, resources.bases.prompts.Index(index), validation)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateBackendRecord(backend backendConfig, base configkit.Path, validation *configkit.Validation) {
|
||||
validateBackendEndpoints(backend, base, validation)
|
||||
if !domain.IsValidLocalResourceKey(backend.ID) {
|
||||
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
||||
}
|
||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||
}
|
||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||
}
|
||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
||||
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
||||
}
|
||||
if backend.QueueCapacity != nil {
|
||||
if *backend.QueueCapacity < 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
||||
}
|
||||
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
||||
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
||||
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
||||
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
||||
}
|
||||
}
|
||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||
smokePath := base.Key("smoke_tests")
|
||||
if smokeTests.DefaultProfile != nil {
|
||||
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
||||
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
||||
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
||||
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
||||
}
|
||||
}
|
||||
if authentication := smokeTests.Authentication; authentication != nil {
|
||||
kindPath := smokePath.Key("authentication").Key("kind")
|
||||
if authentication.Kind == nil {
|
||||
validation.Error(kindPath, "authentication kind is required")
|
||||
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateProfileRecord(profile profileConfig, base configkit.Path, backendIDs map[string]struct{}, validation *configkit.Validation) {
|
||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
|
||||
validation.Error(base.Key("endpoint"), err.Error())
|
||||
}
|
||||
}
|
||||
if !domain.IsValidLocalResourceKey(profile.ID) {
|
||||
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
||||
}
|
||||
if profile.Backend != nil && *profile.Backend != "" {
|
||||
if !domain.IsValidResourceID(*profile.Backend) {
|
||||
validation.Error(base.Key("backend"), "backend reference is malformed")
|
||||
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
||||
validation.Error(base.Key("backend"), "backend is not defined")
|
||||
}
|
||||
}
|
||||
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
||||
validation.Error(base.Key("model"), "model must not be blank")
|
||||
}
|
||||
if profile.Temperature != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
||||
validation.Error(base.Key("temperature"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TopP != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
||||
validation.Error(base.Key("top_p"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.MaxTokens != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
||||
validation.Error(base.Key("max_tokens"), err.Error())
|
||||
}
|
||||
}
|
||||
if profile.TimeoutSeconds != nil {
|
||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
||||
validation.Error(base.Key("timeout_seconds"), err.Error())
|
||||
}
|
||||
}
|
||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||
}
|
||||
|
||||
func validatePromptRecord(prompt promptConfig, base configkit.Path, validation *configkit.Validation) {
|
||||
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
||||
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
||||
}
|
||||
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
||||
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,10 @@ func TestLoadIdentifiedResourceLists(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("load identified resources with header %t: %v", header, err)
|
||||
}
|
||||
if loaded.result.Config.Backends[0].ID != "local" || loaded.result.Config.Collections["team"].Backends[0].ID != "local" {
|
||||
if _, rootErr := loaded.GetBackend("local"); rootErr != nil {
|
||||
t.Fatal(rootErr)
|
||||
}
|
||||
if _, collectionErr := loaded.GetBackend("team/local"); collectionErr != nil {
|
||||
t.Fatal("identified root and collection resources were not retained")
|
||||
}
|
||||
}
|
||||
@@ -153,13 +156,58 @@ func TestIdentifiedResourcesRejectDuplicateStrictPeers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidIdentitiesWithNormalizableFieldsAreRejected(t *testing.T) {
|
||||
for _, kind := range []string{"backends", "profiles", "prompts"} {
|
||||
for _, id := range []string{" ", "bad/id"} {
|
||||
t.Run(kind+"/"+id, func(t *testing.T) {
|
||||
record := map[string]any{"id": id}
|
||||
switch kind {
|
||||
case "backends":
|
||||
record["endpoint"] = " HTTPS://example.invalid/v1 "
|
||||
record["concurrency_limit"] = 1
|
||||
case "profiles":
|
||||
record["service_tier"] = " standard "
|
||||
case "prompts":
|
||||
record["description"] = " padded "
|
||||
}
|
||||
_, err := load(mapping("invalid", map[string]any{kind: []any{record}}), nil)
|
||||
if err == nil || strings.Contains(strings.ToLower(err.Error()), "unrecorded") {
|
||||
t.Fatalf("invalid identity error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateIdentitiesWithNormalizableFieldsAreRejected(t *testing.T) {
|
||||
for _, kind := range []string{"backends", "profiles", "prompts"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
record := map[string]any{"id": "same"}
|
||||
switch kind {
|
||||
case "backends":
|
||||
record["endpoint"] = " HTTPS://example.invalid/v1 "
|
||||
record["concurrency_limit"] = 1
|
||||
case "profiles":
|
||||
record["service_tier"] = " standard "
|
||||
case "prompts":
|
||||
record["description"] = " padded "
|
||||
}
|
||||
_, err := load(mapping("duplicates", map[string]any{kind: []any{record, record}}), nil)
|
||||
if err == nil || strings.Contains(strings.ToLower(err.Error()), "unrecorded") {
|
||||
t.Fatalf("duplicate identity error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
|
||||
for _, kind := range []string{"backends", "profiles", "prompts"} {
|
||||
for _, namespace := range []string{"", "*"} {
|
||||
t.Run(kind+"/"+namespace, func(t *testing.T) {
|
||||
definition := configurationDefinition(&loaded{})
|
||||
definition.Resolve = func(config *document, resolution *configkit.Resolution) error {
|
||||
path := configkit.Keys(kind)
|
||||
definition := testDefinition()
|
||||
definition.Resolve = func(root *testDocument, resolution *configkit.Resolution) error {
|
||||
config := &root.PromptKit
|
||||
path := configkit.Keys("promptkit", kind)
|
||||
if namespace == "" {
|
||||
switch kind {
|
||||
case "backends":
|
||||
@@ -180,7 +228,7 @@ func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
|
||||
collection.Prompts = []promptConfig{{ID: "same"}, {ID: "same"}}
|
||||
}
|
||||
config.Collections = map[string]collectionConfig{namespace: collection}
|
||||
path = configkit.Keys("collections", namespace, kind)
|
||||
path = configkit.Keys("promptkit", "collections", namespace, kind)
|
||||
}
|
||||
resolution.RecordNormalization(path, "supply resolved records")
|
||||
return nil
|
||||
@@ -191,7 +239,8 @@ func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
|
||||
namespace: map[string]any{kind: []any{}},
|
||||
}})
|
||||
}
|
||||
_, err := configkit.LoadLayered(input, configkit.LayeredOptions[document]{Definition: definition})
|
||||
input.Destination = configkit.Keys("promptkit")
|
||||
_, err := configkit.LoadLayered(input, configkit.LayeredOptions[testDocument]{Definition: definition})
|
||||
var loadErr *configkit.LoadError
|
||||
if !errors.As(err, &loadErr) {
|
||||
t.Fatalf("load error = %v, want a final identity diagnostic", err)
|
||||
@@ -228,14 +277,14 @@ func TestIdentifiedResourceLayerOperations(t *testing.T) {
|
||||
values := map[string]any{"profiles": test.records}
|
||||
if test.mode != "" {
|
||||
values["config"] = map[string]any{"sequences": []any{map[string]any{
|
||||
"destination": []any{"profiles"}, "mode": test.mode,
|
||||
"destination": []any{"promptkit", "profiles"}, "mode": test.mode,
|
||||
}}}
|
||||
}
|
||||
state, err := load(mapping("overlay profiles", values), []configkit.Underlay{{Inputs: []configkit.Input{base}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profiles := indexedProfiles(&state.result.Config)
|
||||
profiles := state.profileRecords
|
||||
if len(profiles) != len(test.models) {
|
||||
t.Fatalf("profiles = %+v, want IDs and models %+v", profiles, test.models)
|
||||
}
|
||||
@@ -271,7 +320,7 @@ func TestIdentifiedResourceLayersRejectDisplacedParentAndDuplicateAppend(t *test
|
||||
{
|
||||
name: "append cannot duplicate an identity",
|
||||
values: map[string]any{
|
||||
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"profiles"}, "mode": "append"}}},
|
||||
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "profiles"}, "mode": "append"}}},
|
||||
"profiles": []any{map[string]any{"id": "shared", "model": "other"}},
|
||||
},
|
||||
code: configkit.CodeSequenceConflict,
|
||||
@@ -293,62 +342,11 @@ func TestIdentifiedResourceLayersRejectDisplacedParentAndDuplicateAppend(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadChecksOptionalConfigurationVersion(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
version any
|
||||
include bool
|
||||
valid bool
|
||||
}{
|
||||
{name: "string two", version: "2", include: true, valid: true},
|
||||
{name: "integer two", version: int64(2), include: true, valid: true},
|
||||
{name: "omitted", valid: true},
|
||||
{name: "explicit null", include: true},
|
||||
{name: "unsupported string", version: "1", include: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
values := map[string]any{}
|
||||
if test.include {
|
||||
values["version"] = test.version
|
||||
}
|
||||
loaded, err := load(mapping("main", values), nil)
|
||||
if test.valid {
|
||||
if err != nil {
|
||||
t.Fatalf("load accepted version: %v", err)
|
||||
}
|
||||
if test.include && (loaded.result.Config.Version == nil || *loaded.result.Config.Version != "2") {
|
||||
t.Fatalf("typed version = %v, want 2", loaded.result.Config.Version)
|
||||
}
|
||||
if !test.include && loaded.result.Config.Version != nil {
|
||||
t.Fatalf("omitted version = %q, want nil", *loaded.result.Config.Version)
|
||||
}
|
||||
return
|
||||
}
|
||||
var loadErr *configkit.LoadError
|
||||
if err == nil || !errors.As(err, &loadErr) {
|
||||
t.Fatalf("load error = %v, want ConfigKit load error", err)
|
||||
}
|
||||
found := false
|
||||
for _, diagnostic := range loadErr.Diagnostics() {
|
||||
if diagnostic.Path.String() == "/version" && diagnostic.Message == "unsupported configuration specification version" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("load diagnostics = %+v, want safe version diagnostic", loadErr.Diagnostics())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsMalformedHeadersAndLegacyResourceMappings(t *testing.T) {
|
||||
func TestLoadRejectsLegacyResourceMappings(t *testing.T) {
|
||||
for name, values := range map[string]map[string]any{
|
||||
"boolean header": {"version": true},
|
||||
"fractional header": {"version": 2.5},
|
||||
"sequence header": {"version": []any{2}},
|
||||
"backend mapping": {"backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}}},
|
||||
"profile mapping": {"profiles": map[string]any{"local": map[string]any{}}},
|
||||
"prompt mapping": {"prompts": map[string]any{"local": map[string]any{}}},
|
||||
"backend mapping": {"backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}}},
|
||||
"profile mapping": {"profiles": map[string]any{"local": map[string]any{}}},
|
||||
"prompt mapping": {"prompts": map[string]any{"local": map[string]any{}}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := load(mapping(name, values), nil)
|
||||
@@ -376,23 +374,10 @@ func TestNestedConfigRemainsApplicationData(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConsumesAcceptedUnderlayVersionAndValidatesHeaderlessPayload(t *testing.T) {
|
||||
underlay := mapping("declared underlay", map[string]any{
|
||||
"version": int64(2),
|
||||
})
|
||||
loaded, err := load(mapping("headerless main", map[string]any{}), []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
|
||||
if err != nil {
|
||||
t.Fatalf("load accepted non-main declaration: %v", err)
|
||||
}
|
||||
if loaded.result.Config.Version != nil {
|
||||
t.Fatalf("consumed non-main declaration remained in effective payload: %q", *loaded.result.Config.Version)
|
||||
}
|
||||
|
||||
invalidFragment := mapping("headerless prompt fragment", map[string]any{
|
||||
"prompts": []any{map[string]any{"id": "incomplete", "description": "still validated"}},
|
||||
})
|
||||
if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalidFragment}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
|
||||
t.Fatalf("headerless fragment validation error = %v, want final prompt validation", err)
|
||||
func TestLoadValidatesHeaderlessPromptFragment(t *testing.T) {
|
||||
invalid := mapping("prompt fragment", map[string]any{"prompts": []any{map[string]any{"id": "incomplete", "description": "still validated"}}})
|
||||
if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalid}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
|
||||
t.Fatalf("invalid fragment error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,19 +425,16 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := loaded.result.Config.Collections["base"].Profiles[1]
|
||||
profile := loaded.profileRecords["base/child.v1"]
|
||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||
t.Fatalf("effective values were not retained: %+v", profile)
|
||||
}
|
||||
if loaded.profiles["base/child.v1"]["temperature"] != fieldValue || loaded.profiles["base/child.v1"]["top_p"] != fieldNull || loaded.profiles["base/child.v1"]["api_key_required"] != fieldValue {
|
||||
t.Fatalf("effective presence was lost: %+v", loaded.profiles["base/child.v1"])
|
||||
}
|
||||
provider := profile.ExtraParams["provider"].(map[string]any)
|
||||
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
||||
t.Fatalf("recursive overlay failed: %+v", provider)
|
||||
}
|
||||
backend := loaded.result.Config.Backends[0]
|
||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
||||
backend := loaded.backendRecords["local.v1"]
|
||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 {
|
||||
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
||||
}
|
||||
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
||||
@@ -462,13 +444,11 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
||||
if !exists || collectionProfile.ID != "base/namespaced" || collectionProfile.ResolvedTarget == nil || collectionProfile.ResolvedTarget.BackendID != "base/local.v1" {
|
||||
t.Fatalf("collection profile snapshot = %+v, exists %t", collectionProfile, exists)
|
||||
}
|
||||
if loaded.result.Config.Collections["invoice"].Prompts[0].Messages[0].Content == nil {
|
||||
if loaded.prompts["invoice/summary"].definition.Templates[0].Content != "Summarize." {
|
||||
t.Fatal("literal prompt content was not decoded")
|
||||
}
|
||||
if loaded.result.Config.Version == nil || *loaded.result.Config.Version != "2" ||
|
||||
loaded.result.Config.Collections["base"].ReleaseVersion == nil || *loaded.result.Config.Collections["base"].ReleaseVersion != "" ||
|
||||
loaded.result.Config.Collections["invoice"].ReleaseVersion == nil || *loaded.result.Config.Collections["invoice"].ReleaseVersion != "r7" {
|
||||
t.Fatalf("root or collection version metadata was not retained: %+v", loaded.result.Config)
|
||||
if len(loaded.collections) != 2 || loaded.collections[0].ReleaseVersion == nil || *loaded.collections[0].ReleaseVersion != "" || loaded.collections[1].ReleaseVersion == nil || *loaded.collections[1].ReleaseVersion != "r7" {
|
||||
t.Fatalf("collection version metadata was not retained: %+v", loaded.collections)
|
||||
}
|
||||
if _, _, err := loaded.prompt("invoice/version"); err != nil {
|
||||
t.Fatalf("resource named version was not indexed under its qualified identity: %v", err)
|
||||
@@ -533,18 +513,23 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
|
||||
backend.APIKeyEnv != "TOKEN_NAME" || backend.QueueCapacity != 1024 {
|
||||
t.Fatalf("backend normalization/default was not applied: %+v", backend)
|
||||
}
|
||||
composed := loaded.result.Config.Backends[0]
|
||||
if composed.Endpoint == nil || *composed.Endpoint != " HTTP://localhost:8080/v1 " ||
|
||||
composed.APIKeyEnv == nil || *composed.APIKeyEnv != " TOKEN_NAME " || composed.QueueCapacity != nil {
|
||||
t.Fatalf("composed backend was changed: %+v", composed)
|
||||
composed := loaded.backendRecords["local"]
|
||||
if composed.Endpoint == nil || *composed.Endpoint != "http://localhost:8080/v1" ||
|
||||
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
|
||||
t.Fatalf("resolved backend was not normalized: %+v", composed)
|
||||
}
|
||||
if loaded.backends["local"]["queue_capacity"] != fieldNull {
|
||||
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
|
||||
}
|
||||
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
|
||||
if !found || field.Origin.Operation == configkit.ChangeNormalization || len(field.Origin.Locations) == 0 {
|
||||
field, found := loaded.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("endpoint"))
|
||||
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||
t.Fatalf("endpoint origin was not preserved: %+v", field)
|
||||
}
|
||||
field, found = loaded.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("queue_capacity"))
|
||||
if !found || field.Origin.Operation != configkit.ChangeDerivedDefault {
|
||||
t.Fatalf("queue capacity default was not recorded: %+v", field)
|
||||
}
|
||||
field, found = loaded.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("concurrency_limit"))
|
||||
if !found || field.Origin.Operation == configkit.ChangeNormalization || field.Origin.Operation == configkit.ChangeDerivedDefault {
|
||||
t.Fatalf("unchanged concurrency acquired a resolution record: %+v", field)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
|
||||
@@ -578,7 +563,7 @@ func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := loaded.result.Config.Backends[0].SmokeTests; got != nil {
|
||||
if got := loaded.backendRecords["local"].SmokeTests; got != nil {
|
||||
t.Fatalf("smoke_tests = %+v, want nil", got)
|
||||
}
|
||||
selected, err := loaded.registry.GetBackend("local")
|
||||
@@ -651,32 +636,32 @@ func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
|
||||
{
|
||||
name: "blank default profile",
|
||||
data: map[string]any{"default_profile": " \t "},
|
||||
path: "/backends/0/smoke_tests/default_profile",
|
||||
path: "/promptkit/backends/0/smoke_tests/default_profile",
|
||||
},
|
||||
{
|
||||
name: "authentication kind required",
|
||||
data: map[string]any{"authentication": map[string]any{}},
|
||||
path: "/backends/0/smoke_tests/authentication/kind",
|
||||
path: "/promptkit/backends/0/smoke_tests/authentication/kind",
|
||||
},
|
||||
{
|
||||
name: "unknown authentication kind",
|
||||
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
|
||||
path: "/backends/0/smoke_tests/authentication/kind",
|
||||
path: "/promptkit/backends/0/smoke_tests/authentication/kind",
|
||||
},
|
||||
{
|
||||
name: "non-http reachability URL",
|
||||
data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
|
||||
path: "/backends/0/smoke_tests/reachability/endpoint",
|
||||
path: "/promptkit/backends/0/smoke_tests/reachability/endpoint",
|
||||
},
|
||||
{
|
||||
name: "blank authentication URL",
|
||||
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}},
|
||||
path: "/backends/0/smoke_tests/authentication/endpoint",
|
||||
path: "/promptkit/backends/0/smoke_tests/authentication/endpoint",
|
||||
},
|
||||
{
|
||||
name: "relative authentication URL",
|
||||
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
|
||||
path: "/backends/0/smoke_tests/authentication/endpoint",
|
||||
path: "/promptkit/backends/0/smoke_tests/authentication/endpoint",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -711,7 +696,7 @@ func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := loaded.GetBackend("local")
|
||||
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity != nil || loaded.backends["local"]["queue_capacity"] != fieldAbsent {
|
||||
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.backendRecords["local"].QueueCapacity == nil || *loaded.backendRecords["local"].QueueCapacity != 1024 {
|
||||
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
||||
}
|
||||
}
|
||||
@@ -723,13 +708,12 @@ func TestLoadCapturesProfilePresence(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := loaded.result.Config.Profiles[0]
|
||||
profile := loaded.profileRecords["local"]
|
||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||
t.Fatalf("profile values were not preserved: %+v", profile)
|
||||
}
|
||||
fields := loaded.profiles["local"]
|
||||
if fields["temperature"] != fieldValue || fields["top_p"] != fieldNull || fields["api_key_required"] != fieldValue || fields["model"] != fieldAbsent {
|
||||
t.Fatalf("profile presence was not preserved: %+v", fields)
|
||||
if profile.Model != nil {
|
||||
t.Fatalf("omitted model was not preserved: %+v", profile)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -754,7 +738,7 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("replaced lower-layer domain value should not fail: %v", err)
|
||||
}
|
||||
if loaded.result.Config.Profiles[0].Model != nil {
|
||||
if loaded.profileRecords["partial"].Model != nil {
|
||||
t.Fatal("unused partial profile unexpectedly gained a model")
|
||||
}
|
||||
|
||||
@@ -765,7 +749,7 @@ func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
|
||||
if !errors.As(err, &loadError) {
|
||||
t.Fatalf("expected path-aware effective backend failure, got %v", err)
|
||||
}
|
||||
if diagnostics := loadError.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Path.String() != "/backends/0/endpoint" {
|
||||
if diagnostics := loadError.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Path.String() != "/promptkit/backends/0/endpoint" {
|
||||
t.Fatalf("expected endpoint diagnostic, got %+v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
145
internal/configuration/ownership.go
Normal file
145
internal/configuration/ownership.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package configuration
|
||||
|
||||
import (
|
||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||
"go.maximumdirect.net/promptkit/internal/settings"
|
||||
)
|
||||
|
||||
func copyPointer[T any](value *T) *T {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copy := *value
|
||||
return ©
|
||||
}
|
||||
|
||||
func copyProviderMap(value map[string]any) map[string]any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
return jsonvalue.CloneTrusted(value).(map[string]any)
|
||||
}
|
||||
|
||||
func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings {
|
||||
in.Endpoint = copyPointer(in.Endpoint)
|
||||
in.APIKeyEnv = copyPointer(in.APIKeyEnv)
|
||||
in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit)
|
||||
in.QueueCapacity = copyPointer(in.QueueCapacity)
|
||||
in.ExtraParams = copyProviderMap(in.ExtraParams)
|
||||
in.Application = nil
|
||||
if in.SmokeTests != nil {
|
||||
smoke := *in.SmokeTests
|
||||
smoke.DefaultProfile = copyPointer(smoke.DefaultProfile)
|
||||
if smoke.Reachability != nil {
|
||||
reach := *smoke.Reachability
|
||||
reach.Endpoint = copyPointer(reach.Endpoint)
|
||||
smoke.Reachability = &reach
|
||||
}
|
||||
if smoke.Authentication != nil {
|
||||
auth := *smoke.Authentication
|
||||
auth.Kind = copyPointer(auth.Kind)
|
||||
auth.Endpoint = copyPointer(auth.Endpoint)
|
||||
smoke.Authentication = &auth
|
||||
}
|
||||
in.SmokeTests = &smoke
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func ownProfileSettings(in settings.ProfileSettings) settings.ProfileSettings {
|
||||
in.Backend = copyPointer(in.Backend)
|
||||
in.Endpoint = copyPointer(in.Endpoint)
|
||||
in.Model = copyPointer(in.Model)
|
||||
in.Temperature = copyPointer(in.Temperature)
|
||||
in.TopP = copyPointer(in.TopP)
|
||||
in.MaxTokens = copyPointer(in.MaxTokens)
|
||||
in.TimeoutSeconds = copyPointer(in.TimeoutSeconds)
|
||||
in.ServiceTier = copyPointer(in.ServiceTier)
|
||||
in.ReasoningEffort = copyPointer(in.ReasoningEffort)
|
||||
in.APIKeyEnv = copyPointer(in.APIKeyEnv)
|
||||
in.APIKeyRequired = copyPointer(in.APIKeyRequired)
|
||||
in.ExtraParams = copyProviderMap(in.ExtraParams)
|
||||
in.Application = nil
|
||||
return in
|
||||
}
|
||||
|
||||
func ownPromptSettings(in settings.PromptSettings) settings.PromptSettings {
|
||||
in.DefaultProfile = copyPointer(in.DefaultProfile)
|
||||
in.Description = copyPointer(in.Description)
|
||||
in.SessionID = copyPointer(in.SessionID)
|
||||
in.Application = nil
|
||||
if in.Inputs != nil {
|
||||
in.Inputs = append(make([]settings.PromptInputSettings, 0, len(in.Inputs)), in.Inputs...)
|
||||
for i := range in.Inputs {
|
||||
in.Inputs[i].ContentType = copyPointer(in.Inputs[i].ContentType)
|
||||
in.Inputs[i].Description = copyPointer(in.Inputs[i].Description)
|
||||
}
|
||||
}
|
||||
if in.Messages != nil {
|
||||
in.Messages = append(make([]settings.PromptMessageSettings, 0, len(in.Messages)), in.Messages...)
|
||||
for i := range in.Messages {
|
||||
message := &in.Messages[i]
|
||||
message.Role = copyPointer(message.Role)
|
||||
message.Content = copyPointer(message.Content)
|
||||
message.ContentFile = copyPointer(message.ContentFile)
|
||||
if message.CacheControl != nil {
|
||||
cache := *message.CacheControl
|
||||
cache.Type = copyPointer(cache.Type)
|
||||
cache.TTL = copyPointer(cache.TTL)
|
||||
message.CacheControl = &cache
|
||||
}
|
||||
}
|
||||
}
|
||||
if in.Output != nil {
|
||||
output := *in.Output
|
||||
output.Format = copyPointer(output.Format)
|
||||
output.ValidationMode = copyPointer(output.ValidationMode)
|
||||
output.Schema = jsonvalue.CloneTrusted(output.Schema)
|
||||
output.SchemaFile = copyPointer(output.SchemaFile)
|
||||
output.SchemaPath = copyPointer(output.SchemaPath)
|
||||
output.RepairAttempts = copyPointer(output.RepairAttempts)
|
||||
in.Output = &output
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func ownSettings(in *settings.Settings) settings.Settings {
|
||||
if in == nil {
|
||||
return settings.Settings{}
|
||||
}
|
||||
out := settings.Settings{SchemaRoot: copyPointer(in.SchemaRoot)}
|
||||
copyResources := func(backends []settings.BackendSettings, profiles []settings.ProfileSettings, prompts []settings.PromptSettings) ([]settings.BackendSettings, []settings.ProfileSettings, []settings.PromptSettings) {
|
||||
var b []settings.BackendSettings
|
||||
if backends != nil {
|
||||
b = make([]settings.BackendSettings, len(backends))
|
||||
for i := range backends {
|
||||
b[i] = ownBackendSettings(backends[i])
|
||||
}
|
||||
}
|
||||
var p []settings.ProfileSettings
|
||||
if profiles != nil {
|
||||
p = make([]settings.ProfileSettings, len(profiles))
|
||||
for i := range profiles {
|
||||
p[i] = ownProfileSettings(profiles[i])
|
||||
}
|
||||
}
|
||||
var q []settings.PromptSettings
|
||||
if prompts != nil {
|
||||
q = make([]settings.PromptSettings, len(prompts))
|
||||
for i := range prompts {
|
||||
q[i] = ownPromptSettings(prompts[i])
|
||||
}
|
||||
}
|
||||
return b, p, q
|
||||
}
|
||||
out.Backends, out.Profiles, out.Prompts = copyResources(in.Backends, in.Profiles, in.Prompts)
|
||||
if in.Collections != nil {
|
||||
out.Collections = make(map[string]settings.CollectionSettings, len(in.Collections))
|
||||
for name, collection := range in.Collections {
|
||||
copy := settings.CollectionSettings{ReleaseVersion: copyPointer(collection.ReleaseVersion), SchemaRoot: copyPointer(collection.SchemaRoot)}
|
||||
copy.Backends, copy.Profiles, copy.Prompts = copyResources(collection.Backends, collection.Profiles, collection.Prompts)
|
||||
out.Collections[name] = copy
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user