237 Commits

Author SHA1 Message Date
a0441fb2d1 Prepare v0.17.0 notes and retire completed roadmaps 2026-11-03 02:56:48 +00:00
32547e1e29 Fix the historical profile inheritance link 2026-11-03 02:52:16 +00:00
bcc9ebd2e0 Record the release validation link issue 2026-11-03 02:49:12 +00:00
c7d4318122 Update guidance for Go example tests 2026-11-03 02:31:35 +00:00
175140ae5a Remove the custom example release check 2026-11-03 01:38:30 +00:00
66b98831d3 Test the observation example in Go 2026-11-03 01:38:02 +00:00
3e3cc677fb Test the preparation example in Go 2026-11-03 01:37:44 +00:00
8896d8733b Test the execution example in Go 2026-11-03 01:37:16 +00:00
7bbdb55888 Plan the Go example testing migration 2026-11-03 01:30:30 +00:00
79dfd97afa Plan Go-based testing for maintained examples 2026-11-03 01:00:45 +00:00
995c309560 Adopt ReleaseKit v0.8.0 and saved-attempt recovery 2026-11-03 00:49:47 +00:00
ea4fec9b1b Move Promptkit to its vanity module path 2026-11-02 14:38:00 +00:00
1de3e1fe38 Keep the release runbook focused on Promptkit steps 2026-11-02 14:24:04 +00:00
061f01c081 Run the selected ReleaseKit tool through Go 2026-11-02 14:21:52 +00:00
937b9723d4 Focus release tests on Promptkit wiring 2026-11-02 14:20:28 +00:00
367af5feca Add hosted Gitea release delivery 2026-11-02 14:10:37 +00:00
fd73cd0662 Adopt ReleaseKit v0.7.0 and update recovery guidance 2026-11-02 14:04:42 +00:00
81a55bc62c Adopt ReleaseKit v0.6.0 and its new module path 2026-11-02 13:35:00 +00:00
2bc976de10 Clarify future message normalization opportunities 2026-11-02 13:22:36 +00:00
fc568aa0b2 Complete the ConfigKit resource migration 2026-11-02 13:00:27 +00:00
b0bfe857fe Document the consumer migration to identified resources 2026-11-02 12:58:39 +00:00
2ccbee3758 Document identified resource formats and interpretation 2026-11-02 12:57:20 +00:00
0eee3ec962 Update examples and benchmarks for identified resources 2026-11-02 12:56:05 +00:00
130c4cc447 Update diagnostic and observation contracts 2026-11-02 12:54:10 +00:00
f661176b06 Update public preparation and execution fixtures 2026-11-02 12:53:00 +00:00
3e67743fbf Update internal execution fixtures for identified resources 2026-11-02 12:51:26 +00:00
6b8eefd18b Update public configuration and inventory contracts 2026-11-02 12:50:58 +00:00
682cb84c82 Update configuration and engine fixtures for identified resources 2026-11-02 12:48:47 +00:00
dbc345e566 Verify identified resource composition and inheritance 2026-11-02 12:47:25 +00:00
afcfae2aea Preserve prompt origins while interpreting metadata 2026-11-02 12:47:10 +00:00
eaca50763b Interpret backend and profile settings without changing source values 2026-11-02 12:45:53 +00:00
cb83b8aca6 Include message normalization in future provenance review 2026-11-02 12:43:35 +00:00
0b026861a5 Revise configuration plans to preserve source provenance 2026-11-02 12:20:44 +00:00
da8496b45b Record future provenance-preserving normalization support 2026-11-02 12:11:44 +00:00
e01b23259a Remove public prompt version metadata 2026-11-02 04:04:35 +00:00
500b0aa1c8 Remove internal prompt version metadata 2026-11-02 04:03:39 +00:00
fd6e9742a1 Use explicit IDs in configuration source builders 2026-11-02 04:01:01 +00:00
a6f4d67e15 Preserve inspection redaction for resource lists 2026-11-02 03:58:49 +00:00
c3ce15a4ab Capture prompts from identified lists 2026-11-02 03:56:23 +00:00
8e8f61f53f Index and validate identified resources 2026-11-02 03:55:28 +00:00
b7565affe5 Preserve resource field presence in lists 2026-11-02 03:54:14 +00:00
aba890aa86 Define identified configuration resources 2026-11-02 03:53:09 +00:00
0b271267a0 Update configuration source controls 2026-11-02 03:52:25 +00:00
a4e97bf102 Adopt ConfigKit v1 and Go 1.27 2026-11-02 03:51:39 +00:00
6ae005ed3a Plan the ConfigKit v1 resource migration 2026-11-02 03:46:24 +00:00
07a55e45f2 Settle source builders on explicit-ID slices 2026-11-02 03:33:29 +00:00
2b3875008b Plan ConfigKit v1 and identified resource migration 2026-11-02 03:18:21 +00:00
a51d12d898 Correct collection profile inspection example 2026-10-18 14:42:10 +00:00
9178f3eb91 Repair stale documentation anchors for release 2026-10-17 14:35:52 +00:00
a70fd7bb65 Document the v0.16.0 release and migration 2026-10-17 14:33:22 +00:00
7091b51c87 Retire completed implementation and review roadmaps 2026-10-17 14:31:29 +00:00
8627c751a4 Clarify shared ownership of namespace views 2026-10-17 13:52:50 +00:00
6fff3d822c Detach version labels in prepared execution snapshots 2026-10-17 13:51:30 +00:00
a1f12f3185 Reuse origin metadata across configuration collections 2026-10-17 13:46:58 +00:00
165cef7aec Record configuration collections review findings 2026-10-17 13:37:52 +00:00
0221cd4610 Verify scoped collection behavior and complete roadmap 2026-10-17 06:39:24 +00:00
dfd8da843e Clarify collection contracts and migration docs 2026-10-17 06:35:56 +00:00
29f104fbc0 Add collection migration example and guide 2026-10-17 06:29:57 +00:00
8669294cf6 Add namespace and collection inventories 2026-10-17 06:25:14 +00:00
c9b623b374 Add scoped identity listings and backend inventory 2026-10-17 06:22:22 +00:00
15f79ce7b1 Apply namespace scopes to inspections and diagnostics 2026-10-17 06:17:16 +00:00
aa54b8d1c4 Add immutable namespace views 2026-10-17 06:14:09 +00:00
60c58260ec Use caller profile in inspection test 2026-10-17 06:13:07 +00:00
8dfe35a1f0 Add collection input helper 2026-10-17 06:09:52 +00:00
13b60a72ac Remove bundled catalog integration 2026-10-17 06:08:35 +00:00
ee40a1bdd2 Decouple application tests from bundled profiles 2026-10-17 06:05:27 +00:00
473fc1f6d7 Assemble collection resources by qualified ID 2026-10-17 06:03:28 +00:00
a10a9053b7 Enable shared configuration version checks 2026-10-17 05:56:54 +00:00
ec3245c97e Carry optional prompt labels through results 2026-10-17 05:54:32 +00:00
8cbb7c0daf Remove prompt version selectors 2026-10-17 05:47:14 +00:00
d40a010538 Store one prompt definition per ID 2026-10-17 05:41:21 +00:00
0cd833513c Pass effective paths through configuration loading 2026-10-17 05:33:03 +00:00
5e58b6944b Add shared resource naming primitives 2026-10-17 05:30:26 +00:00
0a85ac8cd1 Adopt ConfigKit v0.8.0 2026-10-17 05:29:24 +00:00
89af2a1281 Plan configuration collections implementation 2026-10-17 05:26:42 +00:00
683689a4f8 Consolidate collection and namespace roadmaps 2026-10-17 05:13:16 +00:00
ddda2f8767 Settle collection configuration decisions around simplicity 2026-10-16 13:50:40 +00:00
190ec69345 Retire unselected collection configuration proposals 2026-10-16 11:59:34 +00:00
b5866dd749 Align collection proposal with implemented ConfigKit capabilities 2026-10-16 11:51:18 +00:00
c7490b783b Propose collections built from mounted configuration fragments 2026-10-14 02:24:50 +00:00
c7d1b0dca7 Propose identity-based collection records for ConfigKit 2026-10-14 01:32:10 +00:00
9e5ed08635 Propose collection configuration using current ConfigKit 2026-10-14 01:15:13 +00:00
4bddbbe8f4 Record collection ownership and ConfigKit boundary decisions 2026-10-14 00:19:05 +00:00
1516631a30 Record accepted namespace scope and filtering decisions 2026-10-13 22:47:21 +00:00
cf997c2034 Explain namespace design recommendations and alternatives 2026-10-13 21:33:21 +00:00
a0ec67c256 Reserve slash for namespace-qualified identifiers 2026-10-13 21:29:36 +00:00
75891b6561 Document namespace and scoped lookup roadmap 2026-10-13 21:25:57 +00:00
5dc6370a97 Avoid redundant smoke report copying 2026-10-13 21:12:17 +00:00
4cda3674c4 Record redundant smoke report copying 2026-10-13 20:50:54 +00:00
5ea6f06a6d Report known zero usage when smoke tests do not dispatch 2026-10-13 20:32:24 +00:00
116bd0e39b Report actual inference smoke test destinations 2026-10-13 20:27:46 +00:00
208ff42915 Preserve caller cancellation in HTTP smoke probes 2026-10-13 20:22:06 +00:00
972a103a97 Record smoke test review findings 2026-10-13 19:59:18 +00:00
2f735aedf9 Verify smoke-test release candidate 2026-10-13 18:38:45 +00:00
f4c721d934 Document smoke-test diagnostics 2026-10-13 18:32:41 +00:00
b7a951685b Verify smoke-test integration boundaries 2026-10-13 18:24:57 +00:00
abb2461244 Expose the public smoke-test API 2026-10-13 18:16:34 +00:00
b268c55783 Integrate inference smoke diagnostics 2026-10-13 18:09:20 +00:00
f0182fb2c9 Assemble HTTP smoke-test reports 2026-10-13 17:48:21 +00:00
8d05b594e2 Add private inference smoke cases 2026-10-13 17:30:45 +00:00
edaaecc982 Add OpenRouter authentication diagnostics 2026-10-13 17:23:47 +00:00
2e81c9e61b Add the unauthenticated reachability probe 2026-10-13 17:12:31 +00:00
2b723760d1 Retain injected client diagnostic evidence 2026-10-13 17:05:40 +00:00
c5aed7a451 Carry evidence through built-in generation retries 2026-10-13 16:56:56 +00:00
d91afc61a4 Retain generation response evidence 2026-10-13 16:47:26 +00:00
6eeb0ee9c0 Capture destination-specific HTTP evidence 2026-10-13 16:38:29 +00:00
b9c232ee63 Adopt tagged diagnostic catalogs 2026-10-13 16:28:41 +00:00
7291d23587 Record RakestrawHome diagnostic catalog candidate 2026-10-13 16:21:20 +00:00
da1d196dc5 Record OpenRouter diagnostic catalog candidate 2026-10-13 16:19:32 +00:00
b0de49a87a Preserve exact smoke-test selections 2026-10-13 16:16:40 +00:00
27ff580dae Resolve smoke-test targets 2026-10-13 16:14:30 +00:00
ece764f19e Load backend smoke-test metadata 2026-10-13 15:53:27 +00:00
f91a02548e Add smoke diagnostic domain values 2026-10-13 15:36:10 +00:00
8fbd058b90 Plan smoke test diagnostics implementation 2026-10-13 15:21:58 +00:00
c8f9a4def9 Plan smoke test integration with locally tagged catalogs 2026-10-13 14:53:55 +00:00
d8b6c0b471 Make smoke test delivery self-contained in one sprint 2026-10-13 14:48:47 +00:00
52cd94fcd1 Finalize smoke test roadmap decisions and evidence contracts 2026-10-13 14:42:24 +00:00
f1f72dfe8f Define factual smoke test diagnostics roadmap 2026-10-13 14:22:35 +00:00
d4676bc870 Record smoke test design decisions and profile precedence 2026-10-13 13:33:12 +00:00
64769883ae Draft smoke test capability proposal 2026-10-13 00:33:21 +00:00
635674c066 Prepare v0.15.0 release documentation 2026-10-12 20:17:52 +00:00
50bc35516a Consolidate request validation parity assertions 2026-10-12 19:31:54 +00:00
18309f50db Record request validation review finding 2026-10-12 19:05:38 +00:00
60784e7875 Document request validation workflow 2026-10-12 17:47:04 +00:00
f3148bfe52 Verify request validation parity 2026-10-12 17:45:18 +00:00
f315620bf7 Add request validation API 2026-10-12 17:43:11 +00:00
97357d62b9 Plan request validation in focused stages 2026-10-12 17:35:25 +00:00
0c3651fceb Clarify request validation roadmap and CLI guidance 2026-10-12 17:26:50 +00:00
5a5d47d36e Define request preflight validation roadmap 2026-10-12 16:56:33 +00:00
143c6e9b3a Prepare v0.14.0 release notes 2026-10-11 01:19:17 +00:00
48e4784fcd Retire completed configuration comparison roadmaps 2026-10-11 01:18:30 +00:00
78af5069c1 Update ConfigKit to v0.7.1 2026-10-11 01:17:10 +00:00
18963266d7 Upgrade ReleaseKit to v0.4.1 2026-10-11 01:11:19 +00:00
d039a019cb Document captured configuration comparison 2026-10-09 16:06:09 +00:00
d1c0281053 Verify captured configuration diff concurrency 2026-10-09 16:01:24 +00:00
870e4ca4e6 Verify configuration diff disclosure semantics 2026-10-09 15:59:58 +00:00
fd82524977 Add captured configuration comparison 2026-10-09 15:58:04 +00:00
3f76c75919 Adopt ConfigKit v0.7.0 2026-10-09 15:56:10 +00:00
fe764f72d8 Plan captured configuration comparison in focused stages 2026-10-09 15:46:41 +00:00
26ab566f38 Define captured configuration comparison roadmap 2026-10-09 15:41:53 +00:00
18ca21fa45 Retire completed roadmap documentation 2026-10-09 04:50:10 +00:00
e32fdc22f4 Document the v0.13.0 release 2026-10-09 04:49:55 +00:00
b02b802427 Upgrade ConfigKit to v0.6.0 2026-10-09 04:45:10 +00:00
301192b974 Document profile and prompt identity listings 2026-10-09 04:34:45 +00:00
f2aeb5eb30 Verify effective listing ownership and concurrency 2026-10-09 04:31:31 +00:00
a27ee8f7da Expose profile and prompt identity listings 2026-10-09 04:28:23 +00:00
4d1895c223 Capture profile and prompt listing identities 2026-10-09 04:27:00 +00:00
d7fe70520a Plan profile and prompt listing in focused stages 2026-10-09 04:23:27 +00:00
7325a34c07 Define profile and prompt listing 2026-10-09 04:19:09 +00:00
03c44923c8 Upgrade the release tooling to ReleaseKit v0.3.0 2026-10-09 03:48:52 +00:00
dc0adf5751 Align the release policy with Go compatibility guidance 2026-10-09 03:45:39 +00:00
eae21797f7 Describe the ReleaseKit upgrade scope 2026-10-09 03:41:56 +00:00
ea6c7d5f0f Release unused file buffers for empty artifacts 2026-10-09 01:52:06 +00:00
389ef26db3 Record the remaining file buffer cleanup issue 2026-10-09 01:38:14 +00:00
8a5c0de162 Finalize computational efficiency evidence 2026-10-09 01:16:14 +00:00
80aecb646d Reuse prepared execution snapshot ownership 2026-10-09 01:08:46 +00:00
b7b4000aab Record prepared snapshot ownership evidence 2026-10-09 00:57:23 +00:00
11ecb923ba Add prepared snapshot ownership benchmarks 2026-10-09 00:56:29 +00:00
f46d5a4591 Record assembled workflow benchmark results 2026-10-09 00:50:41 +00:00
743e78d514 Add assembled workflow benchmark baseline 2026-10-09 00:49:46 +00:00
3743221aa5 Correct roadmap progress summary 2026-10-09 00:46:00 +00:00
a8863d49d0 Record prompt inspection metadata results 2026-10-09 00:45:54 +00:00
c5f733af45 Select prompt metadata for inspection 2026-10-09 00:45:25 +00:00
c09d2c6e20 Record requirement inspection identity results 2026-10-09 00:38:19 +00:00
6c5dec761c Inspect requirements by profile identity 2026-10-09 00:38:01 +00:00
74b91fae99 Record lightweight inspection lookup results 2026-10-09 00:35:59 +00:00
b7ff37118c Add lightweight configuration lookups 2026-10-09 00:35:30 +00:00
ef012ea1e5 Record requirement inspection benchmarks 2026-10-09 00:31:52 +00:00
04d5d695e0 Add requirement inspection benchmarks 2026-10-09 00:30:51 +00:00
78bff91395 Encode generation payloads once on success 2026-10-09 00:27:24 +00:00
c63ca5352f Record request preparation baseline 2026-10-09 00:25:39 +00:00
3f157150e4 Add request preparation baseline and boundary tests 2026-10-09 00:25:21 +00:00
f1e1a45a05 Reserve capacity for regular file artifacts 2026-10-09 00:23:27 +00:00
996295d5c1 Add artifact file-read benchmarks 2026-10-09 00:20:15 +00:00
58a5bbe790 Use local builders for rendered templates 2026-10-09 00:18:39 +00:00
dd89be3b03 Record basic validation allocation savings 2026-10-09 00:15:06 +00:00
ed7c27324d Avoid copying output during basic validation 2026-10-09 00:13:23 +00:00
05fe13d80b Record validation and rendering baselines 2026-10-09 00:11:00 +00:00
0eb009ad5a Add validation and rendering benchmarks 2026-10-09 00:09:15 +00:00
3fde1f6cbb Plan the performance improvements in focused stages 2026-10-09 00:02:21 +00:00
481e2d494c Mark the ConfigKit performance request complete 2026-10-08 23:50:38 +00:00
eb830bb613 Refine performance roadmap after review 2026-10-08 23:44:12 +00:00
dd6d4591ae Plan computational efficiency improvements 2026-10-08 22:02:18 +00:00
de6443f3ae Align completed ReleaseKit documentation 2026-10-08 21:21:37 +00:00
5a63b649f9 Clarify inspected input identifier disclosure 2026-10-08 21:19:24 +00:00
c7e22c0dbe Avoid rescanning unsupported input names 2026-10-08 21:18:45 +00:00
e69a847cf2 Preserve caller scope during requirement inspection 2026-10-08 21:15:56 +00:00
21815ad7c0 Record requirement inspection review findings 2026-10-08 20:59:21 +00:00
a051edd82d Document prompt requirement inspection 2026-10-08 20:48:10 +00:00
ee0ad99e17 Test public prompt requirement inspection 2026-10-08 20:43:48 +00:00
bc5a2da0f2 Expose prompt requirement inspection 2026-10-08 20:40:16 +00:00
089e824a92 Resolve direct prompt input names 2026-10-08 20:36:17 +00:00
93e8874813 Classify simple prompt requirement branches 2026-10-08 20:31:45 +00:00
cc0b6183bf Enumerate prompt input and variable dependencies 2026-10-08 20:25:17 +00:00
bf472dad8a Capture prompt requirement metadata and templates 2026-10-08 20:17:34 +00:00
a6e0ffea8f Simplify requirement inspection plan 2026-10-08 19:58:57 +00:00
e459c7d54b Settle requirement inspection scope 2026-10-08 17:54:20 +00:00
71f6683fda Plan effective requirement inspection implementation 2026-10-08 17:28:04 +00:00
7827ca043b Plan effective prompt requirement inspection 2026-10-08 16:57:58 +00:00
b595dca8c7 Record ReleaseKit adoption review findings 2026-10-08 16:47:23 +00:00
d5f34642d7 Verify ReleaseKit adoption on final source snapshot 2026-10-08 16:42:21 +00:00
b1866946d1 Delegate Promptkit releases to ReleaseKit 2026-10-08 16:40:34 +00:00
05082c6fdf Test ReleaseKit validation failure handling 2026-10-08 16:35:39 +00:00
51b072965c Add successful ReleaseKit publication fixture 2026-10-08 16:33:43 +00:00
d5940856be Add native ReleaseKit bootstrap 2026-10-08 16:23:26 +00:00
0d163f8165 Extract example checks into release profile 2026-10-08 16:20:51 +00:00
1889915a81 Separate the Go minimum from the release toolchain 2026-10-08 16:14:50 +00:00
3ca01ae0b0 Add ReleaseKit as a release tool 2026-10-08 16:10:23 +00:00
6e283d16c3 Plan the ReleaseKit adoption 2026-10-08 13:56:36 +00:00
41b2021d8c Resolve the ReleaseKit adoption decisions 2026-10-08 13:36:01 +00:00
3add7024db Align release preparation and tooling with the current Go version 2026-10-08 13:35:06 +00:00
a148a2b1f4 Define the ReleaseKit adoption roadmap 2026-10-08 13:13:10 +00:00
7310ef34a4 Assess ReleaseKit adoption and shared release validation 2026-10-07 16:08:26 +00:00
4b8169da28 Prepare Promptkit v0.12.0 release 2026-10-03 01:26:43 +00:00
62a4a20529 Preserve message resource origins and profile credential lookup names 2026-10-03 01:21:48 +00:00
8c68b28421 Record ConfigKit adoption review findings 2026-10-03 01:10:29 +00:00
66c55b7397 Benchmark configuration workflows and reuse captured content origins 2026-09-30 04:25:48 +00:00
48343300e2 Remove superseded configuration loaders and consolidate runtime tests 2026-09-30 04:18:27 +00:00
f17a38f60b Document keyed configuration and migrated consumer workflows 2026-09-30 03:34:15 +00:00
aba5e960aa Expose captured configuration inspection and redacted reports 2026-09-30 03:29:28 +00:00
d818e0d162 Render captured prompt templates without reloading sources 2026-09-30 03:24:12 +00:00
ba9c972faa Cut over public engine construction to ConfigKit sources 2026-09-30 03:18:49 +00:00
17b6c7bca5 Integrate released catalog sources with effective configuration 2026-09-30 02:46:33 +00:00
2969a83a71 Record published Rakestrawhome catalog migration 2026-09-30 02:40:22 +00:00
7f58261498 Record published OpenRouter catalog migration 2026-09-30 02:37:50 +00:00
defb240b6e Build presence-aware execution targets from effective configuration 2026-09-30 02:32:54 +00:00
b158a8ec2d Capture keyed prompts and source-backed content at load time 2026-09-30 02:19:52 +00:00
2b847ac2ae Add ConfigKit typed configuration loading core 2026-09-30 02:08:20 +00:00
5f7b19403f Accept the ConfigKit configuration boundary and align the implementation plan 2026-09-30 01:59:30 +00:00
80131fa160 Plan ConfigKit adoption and configuration migration 2026-09-30 00:50:46 +00:00
521291d80f Finalize ConfigKit adoption decisions 2026-09-30 00:40:16 +00:00
0f43460f84 Plan ConfigKit adoption for PromptKit 2026-09-30 00:29:09 +00:00
08409dd472 Align the assessment with ConfigKit v0.5.0 2026-09-30 00:25:10 +00:00
34f5980973 Clarify configuration inspection and execution boundaries 2026-09-29 16:56:50 +00:00
7b012033a6 Align the adoption assessment with ConfigKit source tiers 2026-09-29 16:49:18 +00:00
37f8dba6c3 Clarify ConfigKit prerequisites and efficiency goals 2026-09-29 14:52:38 +00:00
c7bdaa076b Assess ConfigKit adoption for PromptKit 2026-09-29 14:11:30 +00:00
263 changed files with 21581 additions and 14252 deletions

4
.gitignore vendored
View File

@@ -22,6 +22,9 @@
go.work go.work
go.work.sum go.work.sum
# Local ReleaseKit delivery credentials
.releasekit/
# env file # env file
.env .env
@@ -67,4 +70,3 @@ Icon
.AppleDesktop .AppleDesktop
Network Trash Folder Network Trash Folder
Temporary Items Temporary Items
.apdisk

View File

@@ -4,11 +4,12 @@ Promptkit is a reusable Go library for preparing and executing prompt-defined
LLM workflows. Its module path is: LLM workflows. Its module path is:
```text ```text
gitea.maximumdirect.net/eric/promptkit go.maximumdirect.net/promptkit
``` ```
The root `promptkit` package provides the supported public engine. Consumers The root `promptkit` package provides the supported public engine. Consumers
can configure filesystem or in-memory prompt, profile, and schema sources, can configure file, embedded, or programmatic backend, profile, and prompt
definitions alongside filesystem or in-memory schemas,
prepare requests without generation, run requests with the built-in prepare requests without generation, run requests with the built-in
OpenAI-compatible client, or inject their own model client and artifact reader. OpenAI-compatible client, or inject their own model client and artifact reader.
@@ -20,12 +21,21 @@ Run the maintained offline preparation example from the repository root:
go run ./examples/go-library/prepare go run ./examples/go-library/prepare
``` ```
It loads a repository-local prompt, supplies an in-memory profile and inline It composes two explicitly selected synthetic collections, imports
input, and prints deterministic preparation metadata without contacting a source-relative prompt content, supplies an inline input, and prepares through
provider or requiring credentials. Read the a scoped view. The example also demonstrates collection inventories and
[example source](examples/go-library/prepare/main.go), the namespace-filtered discovery, without contacting a provider or requiring
credentials.
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
ordered file/embedded sources, programmatic mappings, and how to replace an
engine when source files change. Applications adapting the prior keyed
configuration or versioned-prompt format can consult the
[configuration migration guide](docs/consumers/configuration-migration.md).
See the [example source](examples/go-library/prepare/main.go), the
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the [Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
[framework format reference](docs/formats.md) to build a consumer workflow. [framework format reference](docs/formats.md) to build a consumer workflow.
The consumer guide also covers explicit, live smoke diagnostics and interpreting
their factual reports; they are separate from the offline quickstart.
Contributors should start with the [development guide](docs/development.md). Contributors should start with the [development guide](docs/development.md).
The [architecture policy](docs/policy/architecture.md) defines the library The [architecture policy](docs/policy/architecture.md) defines the library
@@ -33,6 +43,25 @@ boundary and constraints that framework work must preserve.
## Release Guidance ## Release Guidance
The [v0.17.0 release notes](docs/releases/v0.17.0.md) cover the vanity module
path, ConfigKit v1, identified resource lists, removal of prompt-version
metadata, and the Go 1.27 minimum. This is a breaking pre-v1 migration.
Consumers using the former Go module path should follow the
[module-path migration guidance](docs/consumers/configuration-migration.md#update-go-imports).
The [v0.16.0 release notes](docs/releases/v0.16.0.md) cover explicit smoke
diagnostics, configuration collections, scoped engine views, and discovery.
Upgrading consumers should also review the current identified-resource and
ConfigKit v1 changes in the [migration guide](docs/consumers/configuration-migration.md).
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
validation of concrete requests with `Engine.ValidateRequest`.
Consumers upgrading to `v0.12.0` should read the
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
configuration and public construction API migration.
The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment
and copied-policy attribution updates. and copied-policy attribution updates.

View File

@@ -7,7 +7,7 @@ import (
"sync/atomic" "sync/atomic"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) { func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
@@ -29,15 +29,7 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl}, {Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
} }
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
engine, err := promptkit.NewEngine( engine := newSimpleEngine(t, "configured prompt", promptkit.WithLLMClient(client))
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended} request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
@@ -101,14 +93,9 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
} }
} }
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) { func TestInvalidAppendedMessagesFailBeforeModelWork(t *testing.T) {
promptSource := &inspectionCountingFS{}
var modelCalls atomic.Int64 var modelCalls atomic.Int64
engine, err := promptkit.NewEngine( engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
promptkit.Config{},
promptkit.WithPromptFS(promptSource, "."),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
)
if err != nil { if err != nil {
t.Fatalf("construct engine: %v", err) t.Fatalf("construct engine: %v", err)
} }
@@ -136,9 +123,6 @@ func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
} }
}) })
} }
if promptSource.opens.Load() != 0 {
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
}
if modelCalls.Load() != 0 { if modelCalls.Load() != 0 {
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load()) t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
} }

View File

@@ -4,8 +4,8 @@ import (
"context" "context"
"errors" "errors"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact" artifactadapter "go.maximumdirect.net/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error") var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error")

View File

@@ -4,7 +4,7 @@ import (
"context" "context"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) { func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) {

View File

@@ -0,0 +1,237 @@
package promptkit_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
var (
assembledPreparedSink *promptkit.PreparedRun
assembledResultSink *promptkit.RunResult
assembledDetailsSink promptkit.PreparedRun
assembledHandleSink *promptkit.PreparedExecution
)
type assembledWorkflowClient struct{ content string }
func (client assembledWorkflowClient) Generate(
context.Context,
promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: client.content}, nil
}
type assembledWorkflowFixture struct {
engine *promptkit.Engine
request promptkit.RunRequest
expectedMessage string
response string
}
func assembledWorkflowSchema(b *testing.B) []byte {
b.Helper()
properties := make(map[string]any, 256)
for index := range 256 {
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
}
data, err := json.Marshal(map[string]any{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": properties,
"additionalProperties": false,
})
if err != nil {
b.Fatal(err)
}
return data
}
func assembledWorkflowExtras(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
b.Helper()
message := "Hello {{.name}}"
response := "ok"
output := map[string]any{"format": "text", "validation_mode": "none"}
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
var expectedMessage string
switch name {
case "small":
expectedMessage = "Hello Ada"
case "large_message":
prefix := strings.Repeat("x", 256*1024)
message = prefix + "{{.name}}"
expectedMessage = prefix + "Ada"
case "mutable_payload":
message = `{"subject":"{{.name}}"}`
expectedMessage = `{"subject":"Ada"}`
response = "{}"
profile["extra_params"] = assembledWorkflowExtras(64)
output = map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
}
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
}, "."))
default:
b.Fatalf("unknown assembled workflow fixture %q", name)
}
config := map[string]any{
"profiles": []any{profile},
"prompts": []any{map[string]any{"id": "workflow",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": message}},
"output": output,
}},
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("assembled workflow benchmark", config),
}}}, options...)
if err != nil {
b.Fatal(err)
}
request := promptkit.RunRequest{
PromptID: "workflow", Vars: map[string]string{"name": "Ada"},
}
return assembledWorkflowFixture{
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
}
}
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
b.Helper()
ctx := context.Background()
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
if prepared.PromptID != "workflow" || len(prepared.Messages) != 1 ||
prepared.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
}
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
details := handle.Details()
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
handle.Discard()
b.Fatalf("prepared execution details are invalid: %+v", details)
}
handle.Discard()
result, err := fixture.engine.Run(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
if result.RawOutput != fixture.response {
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
}
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
}
}
func BenchmarkEnginePreparation(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
ctx := context.Background()
b.Run("prepare", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
assembledPreparedSink = prepared
}
b.StopTimer()
assembledPreparedSink = nil
})
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
handle.Discard()
assembledHandleSink = handle
}
b.StopTimer()
assembledHandleSink = nil
})
})
}
}
func BenchmarkEngineRun(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
ctx := context.Background()
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
result, err := fixture.engine.Run(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
assembledResultSink = result
}
b.StopTimer()
assembledResultSink = nil
})
}
}
func BenchmarkPreparedExecutionDetails(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
if err != nil {
b.Fatal(err)
}
defer handle.Discard()
initial := handle.Details()
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared details fixture is invalid: %+v", initial)
}
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
assembledDetailsSink = handle.Details()
}
b.StopTimer()
assembledDetailsSink = promptkit.PreparedRun{}
})
}
}

View File

@@ -1,107 +1,134 @@
package promptkit package promptkit
import ( import (
"gitea.maximumdirect.net/eric/promptkit/internal/backend" "go.maximumdirect.net/configkit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/backend"
) )
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter // BackendOpenRouter is the conventional ID for an OpenRouter backend.
// backend.
const BackendOpenRouter = backend.OpenRouterID const BackendOpenRouter = backend.OpenRouterID
// BackendRakestrawHome is the reserved ID of Promptkit's built-in // BackendRakestrawHome is the conventional ID for a Rakestrawhome backend.
// Rakestrawhome backend.
const BackendRakestrawHome = backend.RakestrawHomeID const BackendRakestrawHome = backend.RakestrawHomeID
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend]. // BackendLocal is a conventional ID for a caller-configured local backend.
// It is not a built-in or reserved backend and must be registered with
// [WithBackend].
const BackendLocal = "local" const BackendLocal = "local"
// Backend configures one engine-scoped OpenAI-compatible backend. // BackendConfig is a presence-aware convenience value for an identified backend
// // definition. Nil pointer fields are omitted, while non-nil pointers preserve
// Backend has no stable JSON representation. Use keyed literals so additions // explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
// to this configuration value do not break source compatibility. // empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
type Backend struct { // non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
// ID is the stable, case-sensitive registry key. NewEngine trims it and // inputs for explicit null. BackendConfig has no stable JSON representation.
// requires a non-blank value. Built-in backend IDs are reserved. type BackendConfig struct {
// ID is the local backend identity within its resource list.
ID string ID string
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and // Endpoint is the required absolute HTTP(S) base URL after all layers merge.
// requires an absolute HTTP or HTTPS URL with a host and without user Endpoint *string
// information, a query string, or a fragment. Paths are allowed. // APIKeyEnv optionally names an environment variable, not a credential.
Endpoint string APIKeyEnv *string
// APIKeyEnv optionally names an environment lookup source for an API key. // ConcurrencyLimit is zero for unlimited generation; negative is invalid.
// NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*. ConcurrencyLimit *int
// A direct RunRequest.APIKey takes precedence. When no usable credential is // QueueCapacity defaults to 1024 for positive concurrency when omitted.
// available, the built-in client omits Authorization; injected clients own // An explicit value, including zero, requires positive concurrency.
// their own credential-resolution behavior. Store only the name, never a
// credential value.
APIKeyEnv string
// ExtraParams contains backend-wide request defaults. Values must be
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
// must not be model, session_id, messages, temperature, max_tokens, top_p,
// service_tier, reasoning_effort, or response_format. An empty map supplies
// no defaults. NewEngine deeply copies the map and rejects excessively deep
// or large values for safety.
ExtraParams map[string]any
// ConcurrencyLimit is the maximum number of simultaneous model-generation
// calls allowed for this backend within one Engine. Zero leaves the backend
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
ConcurrencyLimit int
// QueueCapacity controls how many additional Run or RunPrepared calls may
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
// is positive; a pointer uses its exact value, including zero. The pointed-to
// value must be non-negative, and QueueCapacity must be nil when
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
// the value and does not retain the pointer.
QueueCapacity *int QueueCapacity *int
// ExtraParams supplies JSON-compatible provider request defaults.
ExtraParams map[string]any
// SmokeTests supplies optional diagnostic defaults and HTTP probe settings.
SmokeTests *BackendSmokeTestConfig
} }
// LocalBackend returns a caller-owned Backend for a conventional local // BackendSmokeTestConfig contains optional diagnostic settings associated with
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint // a backend. Nil nested fields are omitted by BackendSource.
// and concurrencyLimit into Endpoint and ConcurrencyLimit without type BackendSmokeTestConfig struct {
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep // DefaultProfile is the exact profile ID selected as the backend's
// their zero values. // diagnostic inference default. A supplied value must not be blank; the ID
// // is resolved only when that default is selected.
// LocalBackend does not read environment variables, register the value, or DefaultProfile *string
// mutate engine or package state. Supply the returned value through // Reachability optionally selects a full HTTP destination for reachability
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and // diagnostics. A nil endpoint uses the effective backend endpoint.
// concurrency semantics, including default queue capacity for a positive Reachability *ReachabilityProbeConfig
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative // Authentication optionally declares a named authentication diagnostic.
// limit. // A supplied object requires Kind.
func LocalBackend(endpoint string, concurrencyLimit int) Backend { Authentication *AuthenticationProbeConfig
return Backend{
ID: BackendLocal,
Endpoint: endpoint,
ConcurrencyLimit: concurrencyLimit,
}
} }
// WithBackend adds one Backend registration to the constructed Engine. // ReachabilityProbeConfig configures an optional reachability destination.
// type ReachabilityProbeConfig struct {
// Registrations accumulate in option order. Every normalized ID must be unique // Endpoint is an absolute HTTP(S) URL, including any desired path or query.
// across consumer registrations and built-ins; a duplicate or invalid // Nil selects the effective backend endpoint; an explicit blank value is
// definition makes NewEngine fail with ErrInvalidConfig. Built-in IDs, // invalid.
// including [BackendOpenRouter] and [BackendRakestrawHome], cannot be Endpoint *string
// replaced. The immutable registration is scoped to the resulting Engine and }
// cannot be enumerated, replaced, removed, or mutated after construction.
// WithBackend does not install package-global state. // AuthenticationProbeConfig configures an optional named authentication
func WithBackend(backend Backend) Option { // diagnostic.
queueCapacity := 0 type AuthenticationProbeConfig struct {
queueCapacitySet := backend.QueueCapacity != nil // Kind selects the authentication adapter. Supported values are "none" and
if queueCapacitySet { // "openrouter_key"; surrounding whitespace and case are normalized.
queueCapacity = *backend.QueueCapacity Kind *string
} // Endpoint is an absolute HTTP(S) URL, including any desired path or query.
return optionFunc(func(options *engineOptions) error { // Nil selects the adapter's backend-relative destination; an explicit blank
options.internal.AddBackend(domain.Backend{ // value is invalid.
ID: backend.ID, Endpoint *string
Endpoint: backend.Endpoint, }
APIKeyEnv: backend.APIKeyEnv,
ExtraParams: backend.ExtraParams, // BackendSource creates a labeled ConfigKit mapping containing backend records
ConcurrencyLimit: backend.ConcurrencyLimit, // in caller order. IDs are explicit and duplicates are left for loading to reject.
QueueCapacity: queueCapacity, // It encodes fields without validating them; NewEngine validates the final
QueueCapacitySet: queueCapacitySet, // effective configuration after all selected source layers are combined.
}) // Mapping captures a copy of supported input values at creation; unsupported
return nil // values fail when the source is loaded.
}) func BackendSource(label string, backends []BackendConfig) configkit.Source {
definitions := make([]any, 0, len(backends))
for _, config := range backends {
fields := map[string]any{"id": config.ID}
if config.Endpoint != nil {
fields["endpoint"] = *config.Endpoint
}
if config.APIKeyEnv != nil {
fields["api_key_env"] = *config.APIKeyEnv
}
if config.ConcurrencyLimit != nil {
fields["concurrency_limit"] = *config.ConcurrencyLimit
}
if config.QueueCapacity != nil {
fields["queue_capacity"] = *config.QueueCapacity
}
if config.ExtraParams != nil {
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
}
if config.SmokeTests != nil {
smokeTests := make(map[string]any)
if config.SmokeTests.DefaultProfile != nil {
smokeTests["default_profile"] = *config.SmokeTests.DefaultProfile
}
if config.SmokeTests.Reachability != nil {
reachability := make(map[string]any)
if config.SmokeTests.Reachability.Endpoint != nil {
reachability["endpoint"] = *config.SmokeTests.Reachability.Endpoint
}
smokeTests["reachability"] = reachability
}
if config.SmokeTests.Authentication != nil {
authentication := make(map[string]any)
if config.SmokeTests.Authentication.Kind != nil {
authentication["kind"] = *config.SmokeTests.Authentication.Kind
}
if config.SmokeTests.Authentication.Endpoint != nil {
authentication["endpoint"] = *config.SmokeTests.Authentication.Endpoint
}
smokeTests["authentication"] = authentication
}
fields["smoke_tests"] = smokeTests
}
definitions = append(definitions, fields)
}
return configkit.Mapping(label, map[string]any{"backends": definitions})
}
// LocalBackend supplies endpoint and concurrency pointers for a caller-chosen
// backend ID; it does not select an ID or validate either value.
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig {
return BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrencyLimit}
} }

View File

@@ -7,7 +7,8 @@ import (
"testing" "testing"
"time" "time"
"gitea.maximumdirect.net/eric/promptkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
) )
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) { func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
@@ -150,17 +151,7 @@ func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
{ {
name: "endpoint-only profile", name: "endpoint-only profile",
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
engine, err := promptkit.NewEngine(promptkit.Config{}, return newSimpleEngine(t, "message", promptkit.WithLLMClient(client))
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct endpoint-only engine: %v", err)
}
return engine
}, },
}, },
} }
@@ -250,27 +241,36 @@ func newBackendCapacityEngine(
reader promptkit.ArtifactReader, reader promptkit.ArtifactReader,
) *promptkit.Engine { ) *promptkit.Engine {
t.Helper() t.Helper()
promptFS := contractPromptFS("prompt", "profile", "message") promptID, content := "prompt", "message"
var inputs []any
if reader != nil { if reader != nil {
promptFS = contractInputPromptFS() promptID, content = "input-prompt", `{{input "input"}}`
inputs = []any{map[string]any{"name": "input", "required": true}}
} }
options := []promptkit.Option{ backend := map[string]any{"id": "limited", "endpoint": "http://backend.example/v1", "concurrency_limit": limit}
promptkit.WithPromptFS(promptFS, "."), if queueCapacity != nil {
promptkit.WithBackend(promptkit.Backend{ backend["queue_capacity"] = *queueCapacity
ID: "limited",
Endpoint: "http://backend.example/v1",
ConcurrencyLimit: limit,
QueueCapacity: queueCapacity,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", BackendID: "limited", Model: "model",
}),
promptkit.WithLLMClient(client),
} }
definition := map[string]any{
"id": promptID,
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
if inputs != nil {
definition["inputs"] = inputs
}
options := []promptkit.Option{promptkit.WithLLMClient(client)}
if reader != nil { if reader != nil {
options = append(options, promptkit.WithArtifactReader(reader)) options = append(options, promptkit.WithArtifactReader(reader))
} }
engine, err := promptkit.NewEngine(promptkit.Config{}, options...) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("capacity fixture", map[string]any{
"backends": []any{backend},
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
"prompts": []any{definition},
}),
}}}, options...)
if err != nil { if err != nil {
t.Fatalf("construct capacity engine: %v", err) t.Fatalf("construct capacity engine: %v", err)
} }

View File

@@ -0,0 +1,233 @@
package promptkit_test
import (
"errors"
"strings"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
left := comparisonInspection(t, "earlier-model")
right := comparisonInspection(t, "later-model")
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare captured configurations: %v", err)
}
if len(differences) != 1 {
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
}
difference := differences[0]
if got, want := difference.Path.String(), "/profiles"; got != want {
t.Fatalf("difference path = %q, want %q", got, want)
}
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
}
reversed, err := right.Diff(left)
if err != nil {
t.Fatalf("compare reversed configurations: %v", err)
}
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
t.Fatalf("reversed difference = %+v", reversed)
}
equal := comparisonInspection(t, "earlier-model")
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
}
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
}
}
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
valid := comparisonInspection(t, "current-model")
var zero promptkit.ConfigurationInspection
for _, comparison := range []struct {
name string
left, right promptkit.ConfigurationInspection
}{
{name: "zero left", left: zero, right: valid},
{name: "zero right", left: valid, right: zero},
{name: "both zero", left: zero, right: zero},
} {
t.Run(comparison.name, func(t *testing.T) {
differences, err := comparison.left.Diff(comparison.right)
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
}
})
}
}
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
t.Fatal("comparison fixtures did not retain their distinct source labels")
}
differences, err := left.Diff(right)
if err != nil || len(differences) != 0 {
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
}
}
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"sequence": []any{"earlier", "unchanged"},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"nullable": nil,
"sequence": []any{"later", "unchanged"},
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare presence and sequence values: %v", err)
}
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
t.Fatalf("atomic backend list difference = %+v", differences)
}
difference := differences[0]
if !difference.Left.Present || !difference.Right.Present ||
strings.Contains(difference.Left.YAML, "nullable:") || !strings.Contains(difference.Right.YAML, "nullable: null") ||
!strings.Contains(difference.Left.YAML, "earlier") || !strings.Contains(difference.Right.YAML, "later") {
t.Fatalf("atomic sequence snapshots = %+v", difference)
}
}
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"token": "left-secret-sentinel",
"visible": "before-value",
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"token": "right-secret-sentinel",
"visible": "after-value",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive values: %v", err)
}
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
t.Fatalf("sensitive backend list difference = %+v", differences)
}
secret := differences[0]
if !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
t.Fatalf("sensitive difference = %+v", secret)
}
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
}
if !strings.Contains(secret.Left.YAML, "before-value") || !strings.Contains(secret.Right.YAML, "after-value") {
t.Fatalf("public sibling missing from snapshots: %+v", secret)
}
differences[0].Path = configkit.Keys("caller", "mutation")
differences[0].Left.YAML = "caller mutation"
differences[0].Right.YAML = "caller mutation"
again, err := left.Diff(right)
if err != nil {
t.Fatalf("repeat comparison after mutating returned records: %v", err)
}
if len(again) != 1 {
t.Fatalf("repeat difference count = %d, want 1: %+v", len(again), again)
}
againByPath := differencesByPath(again)
if _, exists := againByPath["/caller/mutation"]; exists {
t.Fatal("mutating returned path reached retained inspection")
}
if _, exists := againByPath["/backends"]; !exists {
t.Fatalf("original backend path was lost after mutation: %+v", again)
}
for _, difference := range again {
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
}
}
}
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"details": map[string]any{
"token": "nested-secret-sentinel",
"label": "public detail",
},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"details": "replacement",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive shape replacement: %v", err)
}
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
t.Fatalf("shape replacement differences = %+v", differences)
}
difference := differences[0]
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
t.Fatalf("shape replacement masking = %+v", difference)
}
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
}
}
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
t.Helper()
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
}
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
t.Helper()
backend := map[string]any{"id": "local", "endpoint": "https://example.invalid/v1"}
if extraParams != nil {
backend["extra_params"] = extraParams
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
"backends": []any{backend},
"profiles": []any{map[string]any{
"id": "offline", "backend": "local",
"model": model,
}},
})}}}},
}})
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison engine: %v", err)
}
return inspection
}
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
byPath := make(map[string]configkit.Difference, len(differences))
for _, difference := range differences {
byPath[difference.Path.String()] = difference
}
return byPath
}
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
for _, source := range view.Sources() {
if source.Label == label {
return true
}
}
return false
}

View File

@@ -0,0 +1,99 @@
package promptkit
import (
"fmt"
"io"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/configuration"
)
// ConfigurationInspection is a read-only view of one Engine's captured
// effective configuration and source history. It does not expose typed
// configuration values, filesystem contexts, or runtime credentials.
// Report content may include application prompt and provider data, so callers
// choose where and to whom to disclose it.
type ConfigurationInspection struct {
source configuration.Inspection
}
// InspectConfiguration returns the ConfigKit inspection view captured by this
// Engine. It does not reload sources, resolve credentials, or contact a model.
// A nil Engine returns ErrInvalidConfig.
func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
if e == nil || e.internal == nil {
return ConfigurationInspection{}, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
}
// Diff compares this view's captured effective configuration with other. This
// view is the earlier (left) value and other is the later (right) value. The
// comparison delegates to ConfigKit's frozen effective-value semantics, so
// source or resolution history alone does not create differences. It does not
// reread external sources or compare referenced file contents, resolved
// credential contents, later execution preparation, or request-specific
// overrides. Either zero view returns ErrInvalidConfig and no records; other
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
// equivalent engine behavior or execution. Records can contain ordinary
// application data, so callers choose where and to whom to disclose them.
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
if v.source == nil || other.source == nil {
return nil, ErrInvalidConfig
}
return configuration.Diff(v.source, other.source)
}
// Metadata returns detached effective presence and history records.
func (v ConfigurationInspection) Metadata() configkit.Metadata {
if v.source == nil {
return configkit.Metadata{}
}
return v.source.Metadata()
}
// Sources returns a detached list of loaded source descriptions.
func (v ConfigurationInspection) Sources() []configkit.SourceInfo {
if v.source == nil {
return nil
}
return v.source.Sources()
}
// Diagnostics returns detached nonfatal load diagnostics.
func (v ConfigurationInspection) Diagnostics() []configkit.Diagnostic {
if v.source == nil {
return nil
}
return v.source.Diagnostics()
}
// Explain returns detached effective and historical values for a known path.
// Dynamic credential-shaped provider keys are redacted by ConfigKit; ordinary
// prompt and provider content is not automatically safe to disclose.
func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explanation, bool) {
if v.source == nil {
return configkit.Explanation{}, false
}
return v.source.Explain(path)
}
// WriteEffective writes ConfigKit's redacted effective YAML. Writer errors
// remain available through errors.Is and errors.As. The zero view returns
// ErrInvalidConfig.
func (v ConfigurationInspection) WriteEffective(w io.Writer) error {
if v.source == nil {
return ErrInvalidConfig
}
return v.source.WriteEffective(w)
}
// WriteExplanation writes ConfigKit's redacted history for path. Unknown paths
// produce no output; writer errors remain inspectable. The zero view returns
// ErrInvalidConfig.
func (v ConfigurationInspection) WriteExplanation(w io.Writer, path configkit.Path) error {
if v.source == nil {
return ErrInvalidConfig
}
return v.source.WriteExplanation(w, path)
}

View File

@@ -0,0 +1,316 @@
package promptkit_test
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
"backends": []any{map[string]any{
"id": "local", "endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
}},
"collections": map[string]any{"literal": map[string]any{
"backends": []any{
map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "namespaced-private-sentinel"}},
map[string]any{"id": "child", "extends": map[string]any{"id": "local"}},
},
"profiles": []any{
map[string]any{"id": "base.v1", "backend": "local", "model": "default-model"},
map[string]any{"id": "id", "extends": map[string]any{"id": "base.v1"}},
},
}},
})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
"collections": map[string]any{"literal": map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
}},
})}}},
},
}})
if err != nil {
t.Fatalf("construct layered engine: %v", err)
}
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model")
model, found := view.Explain(modelPath)
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
}
inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model")
inherited, found := view.Explain(inheritedPath)
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
}
if inheritedPath.String() != "/collections/literal/profiles/1/model" {
t.Fatalf("literal-key path = %q", inheritedPath)
}
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token")
secret, found := view.Explain(secretPath)
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
}
namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
namespacedSecret, found := view.Explain(namespacedSecretPath)
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
}
inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
inheritedSecret, found := view.Explain(inheritedSecretPath)
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
}
metadata := fmt.Sprintf("%+v", view.Metadata().History(inheritedSecretPath))
if strings.Contains(metadata, "replacement-private-sentinel") || strings.Contains(metadata, "namespaced-private-sentinel") {
t.Fatal("metadata disclosed a sensitive provider value")
}
var effective bytes.Buffer
if err := view.WriteEffective(&effective); err != nil {
t.Fatalf("write effective configuration: %v", err)
}
if strings.Contains(effective.String(), "private-sentinel") || !strings.Contains(effective.String(), "OPTIONAL_KEY") || !strings.Contains(effective.String(), "public-value") {
t.Fatalf("unexpected effective report: %s", effective.String())
}
var explanation bytes.Buffer
if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") {
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
}
sources := view.Sources()
if len(sources) == 0 {
t.Fatal("inspection returned no sources")
}
originalLabel := sources[0].Label
sources[0].Label = "changed"
if view.Sources()[0].Label != originalLabel {
t.Fatal("source mutation reached retained inspection")
}
model.History[0].After.YAML = "changed"
if again, ok := view.Explain(modelPath); !ok || again.History[0].After.YAML == "changed" {
t.Fatal("explanation mutation reached retained inspection")
}
history := view.Metadata().History(modelPath)
if len(history) == 0 {
t.Fatal("metadata history is empty")
}
history[0].Reasons = []string{"changed"}
if again := view.Metadata().History(modelPath); len(again) == 0 || (len(again[0].Reasons) > 0 && again[0].Reasons[0] == "changed") {
t.Fatal("metadata mutation reached retained inspection")
}
}
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("invalid backend", map[string]any{
"collections": map[string]any{"private": map[string]any{"backends": []any{
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
}}},
}),
}}})
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("invalid backend error = %v, want load error", err)
}
if text := fmt.Sprintf("%v %+v", err, loadErr.Diagnostics()); strings.Contains(text, "failure-private-sentinel") {
t.Fatalf("failure diagnostics disclosed a sensitive value: %s", text)
}
}
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("main", map[string]any{"version": "2"}),
}}})
if err != nil {
t.Fatalf("construct engine with accepted declaration: %v", err)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
path := configkit.Keys("version")
field, found := inspection.Explain(path)
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
}
var effective bytes.Buffer
if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") {
t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err)
}
}
type failingInspectionWriter struct{ err error }
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
var engine *promptkit.Engine
if _, err := engine.InspectConfiguration(); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine inspection = %v", err)
}
var empty promptkit.ConfigurationInspection
path := configkit.Keys("profiles", "missing")
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
t.Fatal("zero inspection reported configuration")
}
if _, found := empty.Explain(path); found {
t.Fatal("zero inspection explained an unknown path")
}
if err := empty.WriteEffective(io.Discard); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("zero effective writer = %v", err)
}
if err := empty.WriteExplanation(io.Discard, path); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("zero explanation writer = %v", err)
}
profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
})}
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}},
}})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
view, err := valid.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
writeErr := errors.New("report writer failed")
writer := failingInspectionWriter{err: writeErr}
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
t.Fatalf("effective writer error = %v", err)
}
if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
t.Fatalf("explanation writer error = %v", err)
}
}
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
const promptDocument = "prompts:\n - id: captured\n default_profile: offline\n messages:\n - role: user\n content_file: body.tmpl\n output:\n format: text\n validation_mode: none\n"
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
}}
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
if err != nil {
t.Fatalf("select inputs: %v", err)
}
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
}}
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
if err != nil {
t.Fatalf("select comparison inputs: %v", err)
}
var modelCalls atomic.Int64
client := countingLLMClient{calls: &modelCalls}
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
})}}}},
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct engine: %v", err)
}
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
})}}}},
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
source.closed.Store(true)
otherSource.closed.Store(true)
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
otherView, err := otherEngine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison configuration: %v", err)
}
var expected configkit.Difference
for index := 0; index < 3; index++ {
differences, err := view.Diff(otherView)
if err != nil || len(differences) != 1 {
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
}
if differences[0].Path.String() != "/profiles" ||
!strings.Contains(differences[0].Left.YAML, model) ||
!strings.Contains(differences[0].Right.YAML, otherModel) {
t.Fatalf("captured difference = %+v", differences[0])
}
if index == 0 {
expected = differences[0]
} else if differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
}
}
if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
t.Fatal("captured prompt content path was not inspectable")
}
var report bytes.Buffer
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
}
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
const calls = 24
var wait sync.WaitGroup
errs := make(chan error, calls)
for index := 0; index < calls; index++ {
wait.Add(1)
go func(index int) {
defer wait.Done()
switch index % 3 {
case 0:
_, err := engine.InspectConfiguration()
errs <- err
case 1:
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
errs <- err
case 2:
differences, err := view.Diff(otherView)
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
err = errors.New("concurrent comparison returned an unexpected difference")
}
errs <- err
}
}(index)
}
wait.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
}

100
configuration_sources.go Normal file
View File

@@ -0,0 +1,100 @@
package promptkit
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/filecatalog"
)
// CollectionInput mounts source at the selected collection's configuration
// destination, collections/<namespace>, and enables ConfigKit imports for that
// input. An invalid namespace returns an error matching ErrInvalidConfig. The
// source is not read or validated until ConfigKit loads it. Callers may adjust
// the returned input, including its filesystem context or imports setting.
func CollectionInput(namespace string, source configkit.Source) (configkit.Input, error) {
if !domain.IsValidResourceNamespace(namespace) {
return configkit.Input{}, fmt.Errorf("%w: invalid collection namespace %q", ErrInvalidConfig, namespace)
}
return configkit.Input{
Source: source,
Destination: configkit.Keys("collections", namespace),
Imports: true,
}, nil
}
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
// directory as one strict-composition group. The returned order is deterministic;
// it does not assign precedence among those files. Discovery is recursive over
// .yaml and .yml files. An unavailable root fails; an empty directory yields
// no inputs. Imports are disabled unless callers enable them on the inputs.
func DirectoryInputs(directory string) ([]configkit.Input, error) {
root, err := filepath.Abs(directory)
if err != nil {
return nil, fmt.Errorf("resolve configuration directory: %w", err)
}
info, err := os.Stat(root)
if err != nil {
return nil, fmt.Errorf("inspect configuration directory: %w", err)
}
if !info.IsDir() {
return nil, fmt.Errorf("configuration root is not a directory: %s", root)
}
root, err = filepath.EvalSymlinks(root)
if err != nil {
return nil, fmt.Errorf("resolve configuration directory: %w", err)
}
files, err := filecatalog.FindYAMLFiles(context.Background(), root)
if err != nil {
return nil, fmt.Errorf("discover configuration files: %w", err)
}
inputs := make([]configkit.Input, 0, len(files))
for _, file := range files {
inputs = append(inputs, configkit.Input{
Source: configkit.File(file),
Filesystem: &configkit.FilesystemContext{Root: root},
})
}
return inputs, nil
}
// FilesystemInputs selects the YAML files beneath root in a caller-owned
// filesystem as one strict-composition group. Discovery is recursive over
// .yaml and .yml files, with deterministic ordering and no precedence among
// files. The caller retains ownership of the filesystem; an unavailable root
// fails and an empty directory yields no inputs. Imports remain disabled unless
// callers enable them on inputs.
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
if filesystem == nil {
return nil, fmt.Errorf("configuration filesystem is nil")
}
root = filecatalog.CleanFSRoot(root)
if !fs.ValidPath(root) {
return nil, fmt.Errorf("configuration root is not a valid filesystem path: %s", root)
}
info, err := fs.Stat(filesystem, root)
if err != nil {
return nil, fmt.Errorf("inspect configuration root: %w", err)
}
if !info.IsDir() {
return nil, fmt.Errorf("configuration root is not a directory: %s", root)
}
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
if err != nil {
return nil, fmt.Errorf("discover configuration files: %w", err)
}
namespace := configkit.NewFilesystem(label, filesystem)
inputs := make([]configkit.Input, 0, len(files))
for _, file := range files {
inputs = append(inputs, configkit.Input{
Source: namespace.File(file),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
})
}
return inputs, nil
}

View File

@@ -0,0 +1,157 @@
package promptkit_test
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type sourceDocument struct {
First string `yaml:"first"`
Second string `yaml:"second"`
}
func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
input, err := promptkit.CollectionInput("docs", configkit.File(filepath.Join(t.TempDir(), "not-read.yaml")))
if err != nil {
t.Fatal(err)
}
if input.Destination.String() != "/collections/docs" || !input.Imports {
t.Fatalf("collection input = %+v, want destination /collections/docs with imports enabled", input)
}
for _, namespace := range []string{"", " ", "docs/api"} {
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Errorf("namespace %q error = %v, want ErrInvalidConfig", namespace, err)
}
}
}
func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
directory := t.TempDir()
mainPath := filepath.Join(directory, "collection.yaml")
defaultsPath := filepath.Join(directory, "defaults.yaml")
if err := os.WriteFile(mainPath, []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(defaultsPath, []byte("prompts:\n - id: greeting\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n"), 0o600); err != nil {
t.Fatal(err)
}
input, err := promptkit.CollectionInput("docs", configkit.File(mainPath))
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
Main: &configkit.Input{
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
Destination: configkit.Keys("collections", "docs"),
},
}})
if err != nil {
t.Fatalf("construct engine with native collection import: %v", err)
}
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
}
}
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
files := fstest.MapFS{
"collection.yaml": {Data: []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n")},
"defaults.yaml": {Data: []byte("prompts:\n - id: greeting\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n")},
}
filesystem := configkit.NewFilesystem("embedded collection", files)
input, err := promptkit.CollectionInput("docs", filesystem.File("collection.yaml"))
if err != nil {
t.Fatal(err)
}
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: "."}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
Main: &configkit.Input{
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
Destination: configkit.Keys("collections", "docs"),
},
}})
if err != nil {
t.Fatalf("construct engine with embedded collection import: %v", err)
}
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
}
}
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
t.Fatal(err)
}
for name, data := range map[string]string{
"a.yaml": "first: one\n", "nested/b.yml": "second: two\n", "nested/body.md": "message",
} {
if err := os.WriteFile(filepath.Join(root, name), []byte(data), 0o600); err != nil {
t.Fatal(err)
}
}
inputs, err := promptkit.DirectoryInputs(root)
if err != nil || len(inputs) != 2 || inputs[0].Imports || inputs[1].Imports {
t.Fatalf("directory inputs: %+v, %v", inputs, err)
}
assertSourceInputs(t, inputs, "message")
if _, err := promptkit.DirectoryInputs(filepath.Join(root, "missing")); err == nil {
t.Fatal("missing selected directory succeeded")
}
if empty, err := promptkit.DirectoryInputs(t.TempDir()); err != nil || len(empty) != 0 {
t.Fatalf("empty directory: %+v, %v", empty, err)
}
}
func TestFilesystemInputsSelectsEmbeddedFilesWithRootContext(t *testing.T) {
files := fstest.MapFS{
"settings/a.yaml": {Data: []byte("first: one\n")},
"settings/nested/b.yml": {Data: []byte("second: two\n")},
"settings/nested/body.md": {Data: []byte("message")},
}
inputs, err := promptkit.FilesystemInputs("embedded", files, "settings")
if err != nil || len(inputs) != 2 || inputs[0].Imports || inputs[1].Imports {
t.Fatalf("filesystem inputs: %+v, %v", inputs, err)
}
assertSourceInputs(t, inputs, "message")
if _, err := promptkit.FilesystemInputs("embedded", files, "missing"); err == nil {
t.Fatal("missing selected root succeeded")
}
}
func assertSourceInputs(t *testing.T, inputs []configkit.Input, expected string) {
t.Helper()
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[sourceDocument]{
Underlays: []configkit.Underlay{{Inputs: inputs}},
})
if err != nil {
t.Fatal(err)
}
if result.Config.First != "one" || result.Config.Second != "two" {
t.Fatalf("configuration files did not compose: %+v", result.Config)
}
field, found := result.Metadata().Lookup(configkit.Keys("second"))
if !found || len(field.Origin.Locations) != 1 {
t.Fatalf("missing second-field origin: %+v", field)
}
context, found := result.SourceContext(field.Origin.Locations[0].SourceID)
if !found || context.BaseDir != "nested" {
t.Fatalf("unexpected rooted source context: %+v", context)
}
data, err := fs.ReadFile(context.FS, "nested/body.md")
if err != nil || string(data) != expected {
t.Fatalf("rooted resource read: %q, %v", data, err)
}
}

View File

@@ -0,0 +1,317 @@
package promptkit_test
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
const name = "promptkit.test-api-key"
for _, test := range []struct {
name, value, direct, want string
unset, clear bool
}{
{name: "set", value: " synthetic-key ", want: "Bearer synthetic-key"},
{name: "empty"},
{name: "whitespace", value: " "},
{name: "unset", unset: true},
{name: "direct precedence", value: "synthetic-key", direct: "direct-key", want: "Bearer direct-key"},
{name: "explicit clear", value: "synthetic-key", clear: true},
} {
t.Run(test.name, func(t *testing.T) {
t.Setenv(name, test.value)
if test.unset {
if err := os.Unsetenv(name); err != nil {
t.Fatal(err)
}
}
cfg := simpleConfig("hello")
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
endpoint, model := "http://example.test/v1", "model"
lowerName := name
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
}),
}}})
env := " " + name + " "
if test.clear {
env = ""
}
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
})}
var calls int
var auth string
cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
calls++
auth = request.Header.Get("Authorization")
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
})}
engine, err := promptkit.NewEngine(cfg)
if err != nil {
t.Fatal(err)
}
_, err = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", APIKey: test.direct})
if err != nil || calls != 1 || auth != test.want {
t.Fatalf("run: err=%v calls=%d authorization=%q, want %q", err, calls, auth, test.want)
}
})
}
}
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
backendID, model := "workstation", "offline-model"
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
"prompts": []any{map[string]any{
"id": "hello",
"default_profile": "offline",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}},
})}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
localBackend := promptkit.LocalBackend("http://localhost:8000/v1", 2)
localBackend.ID = backendID
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
localBackend,
})},
{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
{ID: "offline", Backend: &backendID, Model: &model},
})},
}}},
Main: &prompt,
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct engine: %v", err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "hello"})
if err != nil {
t.Fatalf("run prompt: %v", err)
}
if result.RawOutput != "reply" || result.SelectedBackendID != backendID || result.EffectiveModelParams.Model != model {
t.Fatalf("unexpected result: %+v", result)
}
if len(client.requests) != 1 || client.requests[0].Target.Endpoint != "http://localhost:8000/v1" {
t.Fatalf("unexpected model request: %+v", client.requests)
}
}
func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
openRouterID, rakestrawhomeID := promptkit.BackendOpenRouter, promptkit.BackendRakestrawHome
openRouterEndpoint, rakestrawhomeEndpoint := "https://openrouter.example/v1", "https://inference.example/v1"
model := "configured-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
{ID: openRouterID, Endpoint: &openRouterEndpoint},
{ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint},
})},
{Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
{ID: "openrouter-profile", Backend: &openRouterID, Model: &model},
{ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model},
})},
}}},
}})
if err != nil {
t.Fatalf("construct engine with explicitly configured provider IDs: %v", err)
}
for profileID, backendID := range map[string]string{
"openrouter-profile": openRouterID,
"rakestrawhome-profile": rakestrawhomeID,
} {
inspection, err := engine.InspectProfile(context.Background(), profileID)
if err != nil || inspection.EffectiveModelParams.BackendID != backendID {
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
}
}
}
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
"config/default.yaml": &fstest.MapFile{Data: []byte(`
profiles:
- id: application
endpoint: https://example.invalid/v1
model: default-model
prompts:
- id: hello
default_profile: application
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
`)},
}, "config")
if err != nil {
t.Fatalf("select embedded defaults: %v", err)
}
operatorDirectory := t.TempDir()
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n - id: application\n endpoint: https://example.invalid/v1\n model: operator-model\n"), 0o644); err != nil {
t.Fatalf("write operator input: %v", err)
}
operator, err := promptkit.DirectoryInputs(operatorDirectory)
if err != nil {
t.Fatalf("select operator inputs: %v", err)
}
programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1"
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
{ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel},
})}
for _, test := range []struct {
name string
apply bool
want string
}{
{name: "embedded", want: "default-model"},
{name: "operator", apply: true, want: "operator-model"},
{name: "programmatic", apply: true, want: "programmatic-model"},
} {
t.Run(test.name, func(t *testing.T) {
underlays := []configkit.Underlay{{Inputs: embedded}}
if test.apply {
underlays = append(underlays, configkit.Underlay{Inputs: operator})
}
if test.name == "programmatic" {
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"})
if err != nil {
t.Fatalf("prepare prompt: %v", err)
}
if prepared.EffectiveModelParams.Model != test.want {
t.Fatalf("model = %q, want %q", prepared.EffectiveModelParams.Model, test.want)
}
})
}
}
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("invalid", map[string]any{"profiles": []any{map[string]any{"id": "broken", "unknown": true}}}),
}}})
var loadErr *configkit.LoadError
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err)
}
}
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
main := configkit.Input{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("construction = (%#v, %v), want invalid supplied main input", engine, err)
}
}
func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T) {
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
{ID: backendID, Endpoint: &endpoint},
})},
{Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
{ID: "application-profile", Backend: &backendID, Model: &model},
})},
}}},
}})
if err != nil {
t.Fatalf("construct engine with backend overlay: %v", err)
}
inspection, err := engine.InspectProfile(context.Background(), "application-profile")
if err != nil {
t.Fatalf("inspect configured profile: %v", err)
}
if inspection.EffectiveModelParams.Endpoint != endpoint {
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
}
}
func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
backendEndpoint := "https://backend.example/v1"
defaultProfile := "not-yet-listed"
reachabilityEndpoint := " https://probe.example/ready?check=1 "
authenticationKind := " OPENROUTER_KEY "
authenticationEndpoint := "https://probe.example/key?scope=current"
source := promptkit.BackendSource("diagnostic backend", []promptkit.BackendConfig{
{
ID: "local",
Endpoint: &backendEndpoint,
SmokeTests: &promptkit.BackendSmokeTestConfig{
DefaultProfile: &defaultProfile,
Reachability: &promptkit.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
Authentication: &promptkit.AuthenticationProbeConfig{
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
},
},
},
})
backendEndpoint = "https://mutated.example/v1"
defaultProfile = "mutated-profile"
reachabilityEndpoint = "https://mutated.example/ready"
authenticationKind = "none"
authenticationEndpoint = "https://mutated.example/key"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}},
}})
if err != nil {
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatal(err)
}
var effective bytes.Buffer
if err := inspection.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
contents := effective.String()
for _, want := range []string{
"https://backend.example/v1",
"not-yet-listed",
"https://probe.example/ready?check=1",
" OPENROUTER_KEY ",
"https://probe.example/key?scope=current",
} {
if !strings.Contains(contents, want) {
t.Errorf("effective configuration does not contain %q:\n%s", want, contents)
}
}
for _, unwanted := range []string{
"mutated.example",
"mutated-profile",
} {
if strings.Contains(contents, unwanted) {
t.Errorf("effective configuration unexpectedly contains %q:\n%s", unwanted, contents)
}
}
}
func TestPreparationSelectsSinglePromptDefinitionByID(t *testing.T) {
engine := newSimpleEngine(t, "selected")
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil || prepared == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Content != "selected" {
t.Fatalf("prepare prompt by ID = (%+v, %v)", prepared, err)
}
}

View File

@@ -0,0 +1,73 @@
package promptkit_test
import (
"context"
"fmt"
"net/http"
"sync/atomic"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type fakeLLMClient struct {
response *promptkit.GenerateResponse
responses []*promptkit.GenerateResponse
err error
requests []promptkit.GenerateRequest
}
func (f *fakeLLMClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
f.requests = append(f.requests, request)
if f.err != nil {
return nil, f.err
}
if len(f.responses) > 0 {
index := len(f.requests) - 1
if index >= len(f.responses) {
return nil, fmt.Errorf("no response configured for generation %d", index+1)
}
return f.responses[index], nil
}
return f.response, nil
}
type countingLLMClient struct{ calls *atomic.Int64 }
func (client countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if client.calls != nil {
client.calls.Add(1)
}
return &promptkit.GenerateResponse{Content: "ok"}, nil
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
// newSimpleEngine exercises the public constructor with a complete keyed
// document. Tests that need layering or resources select their own inputs.
func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(simpleConfig(content), options...)
if err != nil {
t.Fatalf("construct consumer test engine: %v", err)
}
return engine
}
func simpleConfig(content string) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
"profiles": []any{map[string]any{"id": "profile",
"endpoint": "http://example.test/v1", "model": "model",
}},
"prompts": []any{map[string]any{"id": "prompt",
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}},
})},
}}
}

View File

@@ -5,8 +5,8 @@ import (
"reflect" "reflect"
"unicode/utf8" "unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/jsonvalue"
) )
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) { func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
@@ -25,7 +25,6 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
} }
return domain.RunRequest{ return domain.RunRequest{
PromptID: req.PromptID, PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID, ProfileID: req.ProfileID,
SessionID: req.SessionID, SessionID: req.SessionID,
APIKey: req.APIKey, APIKey: req.APIKey,
@@ -73,7 +72,6 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
} }
return &PreparedRun{ return &PreparedRun{
PromptID: prepared.PromptID, PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash, PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID, SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID, SelectedBackendID: prepared.SelectedBackendID,
@@ -100,7 +98,6 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
RawOutput: result.RawOutput, RawOutput: result.RawOutput,
Validation: fromDomainValidationResult(result.Validation), Validation: fromDomainValidationResult(result.Validation),
PromptID: result.PromptID, PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptHash: result.PromptHash, PromptHash: result.PromptHash,
SessionID: result.SessionID, SessionID: result.SessionID,
RenderedPromptHash: result.RenderedPromptHash, RenderedPromptHash: result.RenderedPromptHash,
@@ -211,6 +208,12 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
} }
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget { func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
return fromDomainExecutionTargetWithOwnedParams(target, copyAnyMap(target.ExtraParams))
}
// fromDomainExecutionTargetWithOwnedParams projects scalar settings and takes
// ownership of extraParams, which the caller has already copied.
func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, extraParams map[string]any) ExecutionTarget {
return ExecutionTarget{ return ExecutionTarget{
BackendID: target.BackendID, BackendID: target.BackendID,
Endpoint: target.Endpoint, Endpoint: target.Endpoint,
@@ -222,7 +225,7 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
ServiceTier: target.ServiceTier, ServiceTier: target.ServiceTier,
ReasoningEffort: target.ReasoningEffort, ReasoningEffort: target.ReasoningEffort,
APIKeyEnv: target.APIKeyEnv, APIKeyEnv: target.APIKeyEnv,
ExtraParams: copyAnyMap(target.ExtraParams), ExtraParams: extraParams,
} }
} }
@@ -252,7 +255,6 @@ func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInsp
} }
return &PromptInspection{ return &PromptInspection{
PromptID: inspection.PromptID, PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion,
PromptHash: inspection.PromptHash, PromptHash: inspection.PromptHash,
DefaultProfileID: inspection.DefaultProfileID, DefaultProfileID: inspection.DefaultProfileID,
Inputs: inputs, Inputs: inputs,

View File

@@ -5,7 +5,7 @@ import (
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
func TestToDomainAppendedMessages(t *testing.T) { func TestToDomainAppendedMessages(t *testing.T) {

37
doc.go
View File

@@ -1,27 +1,32 @@
// Package promptkit provides an embeddable engine for preparing and executing // Package promptkit provides an embeddable engine for preparing and executing
// prompt-defined LLM workflows. // prompt-defined LLM workflows.
// //
// Applications construct an [Engine] with [NewEngine], select filesystem or // Applications construct an [Engine] with [NewEngine], select ConfigKit
// in-memory sources and optional engine-scoped [Backend] registrations, and // sources for prompts, profiles, and backends, and
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare], // call [Engine.InspectConfiguration], [Engine.InspectPrompt],
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete // [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
// registries, repositories, validators, and the built-in OpenAI-compatible // [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
// client remain internal implementation details. // [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
// validators, and the built-in OpenAI-compatible client remain internal
// implementation details.
// //
// # Concurrency and ownership // # Concurrency and ownership
// //
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare, // An Engine supports concurrent InspectConfiguration, InspectPrompt,
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies // PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
// bound admitted Run and RunPrepared calls and model generations where // bound admitted Run and RunPrepared calls and model generations where
// configured, while different backend pools and unlimited backends continue // configured, while different backend pools and unlimited backends continue
// independently. An injected [LLMClient] or [ArtifactReader] can therefore // independently. An injected [LLMClient] or [ArtifactReader] can therefore
// still receive concurrent calls and must be safe for that use. // still receive concurrent calls and must be safe for that use.
// //
// NewEngine copies in-memory profiles and backend definitions. Prepare, // ConfigKit captures the selected configuration when NewEngine constructs an
// PrepareExecution, and Run copy request maps, slices, pointer values, and // Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
// JSON-compatible extra parameters before using them. InspectPrompt and // values, and JSON-compatible extra parameters before using them. InspectPrompt,
// InspectProfile return copied inspection values. Returned values and values // InspectRequirements, and InspectProfile return copied inspection values.
// passed to extension interfaces are likewise isolated from engine state. // ListProfiles and ListPrompts return caller-owned identity slices.
// Returned values and values passed to extension interfaces are likewise
// isolated from engine state.
// Callers own those copies and may mutate them after the call that supplied or // Callers own those copies and may mutate them after the call that supplied or
// returned them. [CapacityError] values are caller-owned and may be mutated // returned them. [CapacityError] values are caller-owned and may be mutated
// without affecting engine state or another error. Immutable [GenerationError] // without affecting engine state or another error. Immutable [GenerationError]
@@ -52,9 +57,11 @@
// used by those values. // used by those values.
// //
// Construction, inspection, handle, and error values, including [Config], // Construction, inspection, handle, and error values, including [Config],
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile], // [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
// [OpenAICompatibleProfileConfig], [ProfileInspection], // [ProfileConfig], [RunRequest],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], // [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
// [CapacityError], [GenerationError], [RetryExhaustedError], and // [CapacityError], [GenerationError], [RetryExhaustedError], and
// [ExecutionEvent], do not have stable JSON // [ExecutionEvent], do not have stable JSON
// representations. Direct API keys are nevertheless excluded from JSON for // representations. Direct API keys are nevertheless excluded from JSON for

View File

@@ -0,0 +1,60 @@
# 0001: Configuration Ownership At The ConfigKit Handoff
## Status
Accepted.
## Date
2026-09-30
## Context
PromptKit is adopting ConfigKit to standardize configuration semantics and
simplify its loading machinery. Configuration includes prompts, profiles, and
backend definitions contributed by maintained catalogs and consuming applications.
ConfigKit supplies loading, composition, precedence, inheritance, and inspection
capabilities. PromptKit supplies LLM-specific interpretation and execution.
A clear handoff assigns each library responsibility for the configuration it
owns and keeps validation aligned with the values an application actually uses.
## Decision
ConfigKit owns configuration processing up to the handoff of the final effective
configuration. PromptKit takes ownership at that handoff and applies its domain
interpretation and validation to the final configuration.
The handoff includes effective presence/null information and resource context
supplied by ConfigKit. PromptKit uses this information to interpret effective
settings and resolve referenced resources.
PromptKit passes through ConfigKit's inspection capabilities for explicit
operator use. Configuration history and source explanations serve that
inspection purpose; domain validation operates on the final effective values.
This boundary applies uniformly to application configuration and maintained
catalog configuration, including backend definitions.
## Alternatives Considered
- **Independent source or catalog preflight validation:** preserves checks on
individual contributions, but duplicates configuration processing and evaluates
values that may be replaced before handoff. Raw-source and intermediate-state
validation are outside PromptKit's scope, including equivalent test-only checks.
- **History-based source auditing:** can identify overwritten contributions, but
couples domain enforcement to configuration history. Historical inspection is
an operator capability rather than a validation mechanism.
## Consequences
PromptKit's configuration logic and tests focus on the final effective values,
their domain meaning, and resulting execution behavior. ConfigKit owns failures
arising during its processing, which PromptKit preserves through its error
boundary. Catalog maintainers own their repositories' publishing checks.
The adoption work routes backend configuration through the same handoff and
retires PromptKit's source-level and intermediate-configuration validation.
Effective resource interpretation and explicit operator inspection remain
supported. This concentrates configuration semantics in ConfigKit and reduces
duplicated machinery and maintenance in PromptKit.

View File

@@ -0,0 +1,231 @@
# Configuration And API Migration
This breaking pre-v1 minor migration adopts Promptkit's vanity module path,
changes the v0.16-era keyed resources to identified lists, adopts ConfigKit v1,
and removes prompt-version labels.
Promptkit does not install a provider catalog; applications choose configuration
inputs explicitly, and execution selects one prompt by its exact ID. This guide
explains the shape changes and decisions an application must make; it is not an
automatic migration tool. The [framework format reference](../formats.md) owns
the current configuration contract, and the root package GoDoc owns exact API
signatures and errors.
## Update Go Imports
Promptkit's module and public package import path is now
`go.maximumdirect.net/promptkit`, replacing
`gitea.maximumdirect.net/eric/promptkit`. Update application, library, test, and
generated imports, then select a release using the new declaration:
```sh
go get go.maximumdirect.net/promptkit@VERSION
go mod tidy
```
Replace `VERSION` with a release that declares the vanity module path.
Earlier tags, including v0.16.0, retain their original module declaration and
use the former import path. No permanent `replace` directive is needed for the
cutover. Source browser, licensing, and hosted-release links continue to use
the Forge repository URLs, independently of Go's module identifier.
The package name remains `promptkit`. The path change itself does not rename
exported APIs or change their behavior, but Go treats packages imported through
the two paths as distinct. Update dependent packages that exchange Promptkit
types together and review any old-path dependencies retained after tidying.
The configuration and API adaptations below are also part of this migration.
## Select And Compose The Configuration You Need
Supply each backend, profile, prompt, schema, and collection source that the
application intends to use. Promptkit does not discover directories or install
provider definitions implicitly. An application may select an explicit
collection input with `CollectionInput`:
```go
input, err := promptkit.CollectionInput(
"openrouter",
configkit.File("config/openrouter.yaml"),
)
if err != nil {
return err
}
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
},
})
```
Collection resource IDs are local IDs, while their runtime identities are
qualified. For example, a profile with `id: gemini-flash-latest` in collection
`openrouter` is selected as `openrouter/gemini-flash-latest`. Stored backend,
prompt-default, and other resource references must use their exact intended IDs;
Promptkit does not rewrite references when a source is mounted. `extends`
selects a sibling ID in the same list, not a cross-collection reference.
Review collisions and all references when moving resources into collections.
The optional source-root `version` header now accepts `2` or `"2"`; headerless
sources remain valid. A collection's `release_version` remains an optional
label. Prompt-version labels have been removed: choose one definition per ID
and store any application-specific release label outside Promptkit. In a
standalone imported prompt item, root `version` is a source header, not a label.
For example, change a version-keyed definition:
```yaml
prompts:
report:
"2.0":
messages:
- role: user
content: Review this report.
output:
format: text
validation_mode: none
```
to one identified definition:
```yaml
prompts:
- id: report
messages:
- role: user
content: Review this report.
output:
format: text
validation_mode: none
```
The same conversion applies to backends and profiles. Sibling inheritance uses
identity references, and a later same-ID record replaces the whole lower record:
```yaml
profiles:
- id: base
endpoint: https://example.invalid/v1
model: example-model
- id: fast
extends: {id: base}
timeout_seconds: 30
```
To customize a lower resource's fields, add a distinct inherited variant as
above; when replacing the same ID, supply the complete intended record. Generic
append, replace, imports, and inheritance are described by the
[ConfigKit v1 configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md).
Source-root controls are now `config:`, not `configkit:`. Imports are optional
unless the selected import declares `required: true`.
Update ConfigKit imports to `go.maximumdirect.net/configkit` at v1.0.0: its Go
types are incompatible with the old module path. Promptkit now requires Go
1.27.0 or newer. Source builders take ordered slices with explicit IDs:
```go
endpoint, model := "https://example.invalid/v1", "example-model"
base := "base"
source := promptkit.ProfileSource("application", []promptkit.ProfileConfig{
{ID: "base", Endpoint: &endpoint, Model: &model},
{ID: "fast", Extends: &base},
})
```
Replace former maps passed to `BackendSource` in the same way, setting each
`BackendConfig.ID`. Duplicates are retained for loading to reject. Resource
inspection paths now address list indexes, for example
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
and diffs expose composed values and provenance, while `InspectProfile` and
other interpreted views expose Promptkit normalization and defaults; the views
are not promised to be identical.
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
`prompt_version` JSON members. Prompt hashes are opaque equality values and may
change when the removed field no longer participates in hashing.
## Identity Mapping
The reviewed catalog identities have slash-free profile keys. Their documented
mapping is:
| Previous identity | Namespaced identity or action |
| --- | --- |
| Backend `openrouter` | `openrouter/default` |
| Backend `rakestrawhome` | `rakestrawhome/default` |
| Each OpenRouter catalog profile `p` | `openrouter/p`, retaining the exact local key `p` |
| Profile `rakestrawhome-gemma-4-31b` | `rakestrawhome/rakestrawhome-gemma-4-31b` |
| Built-in `promptkit-openrouter-standard` | No automatic replacement; optionally define caller profile `openrouter/standard` |
| Built-in `promptkit-openrouter-auto` | No automatic replacement; optionally define caller profile `openrouter/auto` |
These are conversion examples, not aliases or bundled resources. Mounting old
`/v3` assets does not update backend references, profile inheritance, prompt
defaults, or diagnostic defaults inside them. Edit those values to the exact
qualified IDs you chose.
For example, an optional caller-owned standard profile may extend a selected
Gemini profile in the same `openrouter` collection:
```yaml
collections:
openrouter:
profiles:
- id: standard
extends: {id: gemini-flash-latest}
```
An optional auto profile may instead select the exact backend and model values:
```yaml
collections:
openrouter:
profiles:
- id: auto
backend: openrouter/default
model: openrouter/auto
```
Neither optional profile name is a Promptkit contract. Keep provider model
strings and settings as selected by the application, including any meaningful
prefixes. If a backend declares diagnostic defaults, update them to exact
qualified IDs; an unselected diagnostic default is not automatically made
resolvable by migration.
## Prompt Selection API
Prompt selection is ID-only. Remove version selectors from calls to `Run`,
`Prepare`, `PrepareExecution`, `ValidateRequest`, `InspectRequirements`, and
`SmokeTest`. Remove the `PromptVersion` field from `RunRequest`,
`RequirementInspectionRequest`, and `SmokeTestRequest`; `InspectPrompt` now
has the form `InspectPrompt(ctx, promptID)`. For example, select the prompt
explicitly in the request rather than setting a version selector:
```go
request := promptkit.RunRequest{
PromptID: "support/report",
ProfileID: "support/standard",
}
_, err = engine.Run(ctx, request)
if err != nil {
return err
}
```
If the former format contained multiple versions under one ID, choose the one
definition the application should install. Promptkit no longer retains or
returns its former descriptive label.
The public inventory APIs can help an operator inspect a captured effective
configuration: `ListProfiles`, `ListPrompts`, and `ListBackends` return exact
identities, while `ListNamespaces` and `ListCollections` expose resource-bearing
namespaces and effective collection declarations. Inventory does not select
which configuration version the application should deploy.
## Deployment Selection Remains With The Application
Applications decide which checked-in, embedded, operator-supplied, or
programmatic inputs to compose for each deployment. They also decide when to
construct a replacement engine after changing those sources. Promptkit does
not fetch catalog updates, select provider availability, validate upstream
catalog release status, or automatically migrate arbitrary consumer files.
The maintained [offline collection example](../../examples/go-library/prepare/main.go)
shows explicit source selection, local imports, layering, a scoped view, and
operator inventory without a provider call.

View File

@@ -12,63 +12,327 @@ file contracts.
Import the package as: Import the package as:
```go ```go
import "gitea.maximumdirect.net/eric/promptkit" import "go.maximumdirect.net/promptkit"
``` ```
For migration from the former module path, see
[Update Go Imports](configuration-migration.md#update-go-imports).
Configuration APIs use `go.maximumdirect.net/configkit` v1.0.0. Its types are
not interchangeable with the old ConfigKit module path.
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
demonstrates explicit collection composition and inventory. Applications
adapting older provider or versioned-prompt configuration can use the
[configuration migration guide](configuration-migration.md) for the required
caller-owned choices.
The following Go fragments are illustrative and omit surrounding package, The following Go fragments are illustrative and omit surrounding package,
import, and error-handling code. Use the maintained examples for complete import, and error-handling code. Use the maintained examples for complete
programs. programs.
## Go Compatibility
The `go` directive in the selected release's [module metadata](../../go.mod)
declares its minimum supported consumer Go version, currently Go 1.27.0.
Maintainers build and
validate releases with the locally selected Go patch release; the minimum
does not simply track that patch version.
Successful release validation on the selected patch is sufficient to presume
support across its Go series, subject to the declared minimum and concrete
evidence of incompatibility. Specific compatibility concerns are investigated
as they arise. Minor releases may raise the minimum; patch releases preserve
it. Release notes explain increases and any required consumer action. See the
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
## Construct An Engine ## Construct An Engine
Create an engine with Create an engine with [`NewEngine`](../../engine.go). Select each directory
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt explicitly and place its inputs in an ordered ConfigKit underlay. A group
directory and may supply profile and schema directories: strictly composes its files; later groups can overlay earlier ones:
```go ```go
promptInputs, err := promptkit.DirectoryInputs("prompts")
if err != nil { return err }
profileInputs, err := promptkit.DirectoryInputs("profiles")
if err != nil { return err }
engine, err := promptkit.NewEngine(promptkit.Config{ engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts", Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
ProfileDir: "profiles", {Inputs: promptInputs},
SchemaDir: "schemas", {Inputs: profileInputs},
}},
SchemaDir: "schemas",
}) })
``` ```
Options support single-file or `fs.FS` sources, in-memory profiles, For a single highest-priority document, set `Sources.Main` to a
engine-scoped backends, and injected artifact or model clients. Consult the `*configkit.Input`; omit it for ordinary layered construction. A single file
[constructor and option GoDoc](../../engine.go) for composition, precedence, can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
validation, and default transport behavior. Source discovery, format `configkit.Mapping` supply programmatic definitions. Schema and runtime
validation, and profile precedence are defined by the collaborator options remain available. Construction validates the final
[framework format reference](../formats.md). effective configuration and captures referenced prompt content; construct a
new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go)
for the exact public contract and the [framework format reference](../formats.md)
for definition fields.
### Use Scoped Engine Views
Create a view when an application wants short request IDs to resolve within a
selected collection:
```go
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
Prompt: "support",
Profile: "support",
})
if err != nil { return err }
prepared, err := scoped.Prepare(ctx, promptkit.RunRequest{
PromptID: "answer",
ProfileID: "concise",
})
```
Execution, preparation, prompt/profile inspection, requirement inspection, and
smoke diagnostics qualify slashless caller selections with their respective
namespaces. Already-qualified IDs pass through unchanged; configured defaults
and references are not rewritten. Each call replaces the view's scope defaults,
including clearing a scope with an empty field, without changing the receiver
or rebuilding its captured configuration and execution collaborators.
Namespace syntax is checked when creating the view; resource existence is
checked when selected. Configuration inspection remains global. See
[`WithNamespaces`](../../namespaces.go) for the exact public contract.
## Supply Embedded Application Defaults ## Supply Embedded Application Defaults
Use `WithFallbackProfileFS` when an application packages profile definitions Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
that should apply unless an operator provides an ordinary configured profile explicitly chosen operator directory. A later programmatic group can override
with the same ID. For example, an application can embed its defaults while both. No directory name has built-in precedence:
continuing to use `ProfileDir` for operator overrides:
```go ```go
//go:embed profiles/*.yaml defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS, "config")
var applicationProfiles embed.FS if err != nil { return err }
operator, err := promptkit.DirectoryInputs(operatorDirectory)
engine, err := promptkit.NewEngine(promptkit.Config{ if err != nil { return err }
PromptDir: "prompts", profile := promptkit.ProfileSource("application choices", []promptkit.ProfileConfig{
ProfileDir: operatorProfileDir, {ID: "summary", Endpoint: &selectedEndpoint, Model: &selectedModel},
}, })
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"), engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
) Underlays: []configkit.Underlay{
{Inputs: defaults},
{Inputs: operator},
{Inputs: []configkit.Input{{Source: profile}}},
},
}})
``` ```
Keep application-owned profile IDs and definitions in the embedded source. Each later same-ID resource replaces the complete earlier record; use a distinct
Use the ordinary configured profile source for operator overrides. Leave inherited sibling for field-level customization. Only add the operator group
`operatorProfileDir` empty when the operator did not configure an override when the application selects that directory; an
directory; a non-empty path names an authoritative higher-precedence source, explicitly selected missing or unreadable root is an error. Each helper finds
so an unavailable or unreadable directory is an error rather than a reason to `.yaml` and `.yml` files recursively and disables imports by default. The
fall back. The caller may enable imports on individual inputs. A nil `Sources.Main` is an
[framework format reference](../formats.md#source-and-profile-precedence) empty main source, not another directory lookup.
owns the exact profile format and lookup order; the
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and For a programmatic document with explicit null or a prompt definition, use a
validation rules. labeled mapping as an input. A mapping owns a snapshot of supported values:
```go
document := configkit.Mapping("application configuration", map[string]any{
"profiles": []any{
map[string]any{"id": "local-summary",
"backend": "local", "model": "example-model",
"temperature": nil, // Suppress an inherited provider control.
},
},
})
input := configkit.Input{Source: document}
// Add input to a later underlay, or set Sources.Main = &input.
```
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
is not needed. Backend and profile definitions are supplied by the caller;
there is no registration-only backend namespace.
When configuration or message files change, build and validate a new engine,
then switch the application to that engine for subsequent work. Existing
engines and prepared handles retain their captured definitions; schema and
request artifact resources still follow their runtime lifecycle. Inspection
does not watch or reload sources automatically.
## Discover Available Resources And Collections
Use [`Engine.ListProfiles`](../../listing.go),
[`Engine.ListPrompts`](../../listing.go), and
[`Engine.ListBackends`](../../listing.go) to discover exact identities from the
engine's final captured configuration. A scoped view filters by its respective
namespace by default; pass `NamespaceNamed`, `NamespaceUnqualified`, or
`NamespaceAll` to override that listing filter without changing execution
scope. Returned IDs remain full identities and can be used for exact selection:
```go
profiles, err := engine.ListProfiles(ctx)
if err != nil {
return err
}
if len(profiles) > 0 {
_, err := engine.InspectProfile(ctx, profiles[0])
if err != nil {
return err
}
}
prompts, err := engine.ListPrompts(ctx)
if err != nil {
return err
}
if len(prompts) > 0 {
selected := prompts[0]
inspection, err := engine.InspectPrompt(ctx, selected.ID)
if err != nil {
return err
}
_ = inspection
}
```
Use [`Engine.ListNamespaces`](../../listing.go) to discover resource-bearing
namespaces across the entire captured configuration, independent of view
scopes. [`Engine.ListCollections`](../../listing.go) reports effective
collection declarations, including metadata-only entries; it does not infer
collection metadata from resource membership or source history.
These lists describe effective membership, not ConfigKit contribution history,
and do not promise that a profile is ready to execute. Use inspection for
details and preparation to resolve request resources. The linked method GoDoc
owns ordering, ownership, concurrency, and error behavior; identifiers may be
sensitive application data.
## Inspect The Captured Configuration
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
operator-selected report of composed configuration, source descriptions,
diagnostics, and path history. Configuration inspection uses list indexes in
the collection's nested path rather than qualified runtime IDs. For a profile
at index `i` in collection `team`, construct the path with `configkit.Keys`:
```go
view, err := engine.InspectConfiguration()
if err != nil { return err }
path := configkit.Keys("collections", "team", "profiles").Index(i).Key("model")
explanation, found := view.Explain(path)
if found && explanation.Effective.Redacted {
// Apply the application's disclosure policy before sharing the report.
}
```
`WriteEffective` and `WriteExplanation` produce ConfigKit-redacted reports;
the application decides whether those reports may be disclosed. Credential-
shaped provider keys are redacted, but arbitrary prompt and provider content
is not necessarily safe to share. Inspection uses the engine's captured result
and does not reopen sources or resolve credentials. It retains composed values
and provenance rather than Promptkit's normalized/defaulted interpretation.
`InspectProfile` instead shows backend-completed execution settings for one profile, and `Prepare` or
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
and validation resources. None of those views is a substitute for the others.
To review changes between two independently captured configurations, compare
their inspection views:
```go
earlier, err := earlierEngine.InspectConfiguration()
if err != nil {
return err
}
later, err := candidateEngine.InspectConfiguration()
if err != nil {
return err
}
differences, err := earlier.Diff(later)
if err != nil {
return err
}
for _, difference := range differences {
if difference.HiddenChanged {
// Report a protected change without inferring its value.
}
if difference.Left.Redacted || difference.Right.Redacted {
// Apply the application's disclosure policy before presenting snapshots.
}
}
```
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
and ConfigKit's [v1.0.0 effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md#effective-configuration-diffs)
for the exact comparison contract. The operation compares only frozen effective
configuration values: it does not reread external files or compare their
contents, resolved credential contents, later execution preparation, or
request-specific overrides. An empty diff is not proof of equivalent execution.
Records can contain ordinary application data, so use the application's
disclosure policy when presenting them.
## Run Explicit Smoke Diagnostics
[`Engine.SmokeTest`](../../smoke_diagnostics.go) runs one explicit diagnostic
using a target resolved from the engine's captured configuration. It does not
execute the selected application prompt or its inputs, session, artifacts, or
output schema. Unlike the offline examples and default test suite, calling it
can contact the selected provider and use configured credentials. It is a
consumer operation, not an automatic readiness verdict or a release check.
For example, a caller can request the built-in text inference check and treat
its optional fact as an observation rather than assuming that a nil Go error
means success:
```go
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
Kind: promptkit.SmokeTestTextInference,
ProfileID: profileID,
APIKey: apiKey,
})
if err != nil {
if report != nil {
retainDisclosureSafeSummary(report)
}
return err
}
if report != nil {
matched := report.Facts.ExpectedTextMatched
switch {
case matched == nil:
// The comparison was not evaluated or is unknown/inapplicable.
case *matched:
// The private expected-text comparison matched.
default:
// It did not match; the operation may still have concluded normally.
}
}
```
A nil Go error means the diagnostic concluded, not that its finding was
positive. Fact pointers are tri-state: nil means unknown, not evaluated, or
inapplicable. Reports contain factual observations and effective settings, not
generated text or provider bodies; identifiers and settings can still be
sensitive. The
[`SmokeTest` GoDoc](../../smoke_diagnostics.go) owns selection precedence,
exact fields, timeout and retry behavior, supported transports, and disclosure
details.
Caller cancellation can return both an error and a partial report. Preserve the
context error identity and inspect any available observations before returning:
```go
report, err := engine.SmokeTest(ctx, request)
if report != nil {
retainDisclosureSafeSummary(report)
}
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
// The report may describe work completed before interruption.
}
return err
}
```
## Inspect A Prompt Before Preparation ## Inspect A Prompt Before Preparation
@@ -77,7 +341,7 @@ declared inputs and output workflow without creating placeholder inputs or
resolving a profile: resolving a profile:
```go ```go
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "") inspection, err := engine.InspectPrompt(ctx, "meeting.summary")
if err != nil { if err != nil {
return err return err
} }
@@ -94,6 +358,85 @@ rendered messages, and use prepared execution when that work must remain tied
to later execution. The method's [GoDoc](../../engine.go) owns exact fields, to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
hash, ownership, and error semantics. hash, ownership, and error semantics.
## Inspect Requirements Before Preparation
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
needs the prompt's recognizable input and caller-variable dependencies for a
particular context before it has input artifacts. Unlike `InspectPrompt`, it
also walks captured template syntax and selects a profile identity; unlike
`Prepare`, it does not load artifacts or render messages.
```go
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
PromptID: "meeting.summary",
Vars: map[string]string{"audience": "project leads"},
})
if err != nil {
return err
}
for _, input := range requirements.Inputs {
if input.Kind == promptkit.RequirementRequired {
// Plan to bind this input before preparation.
}
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
// A recognizable reference may occur only on a known skipped path.
}
}
for _, variable := range requirements.Variables {
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
// The key is required even though this inspection did not receive it.
}
}
if !requirements.Complete {
// Preserve the partial inventory and account for its discovery issues.
}
```
Optional entries can include recognizable inputs and variables found only in
inactive branches. A missing required variable is reported separately from
discovery completeness: missing a value used only for direct text substitution
does not by itself make the inventory incomplete. Conversely, an unresolved
dependency-bearing branch can make discovery incomplete while retaining known
conditional entries. Do not treat an empty partial inventory as proof that no
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
the exact classifications, supported syntax, ownership, and error contract.
## Validate A Concrete Request
Use [`Engine.ValidateRequest`](../../engine.go) when the application has a
complete `RunRequest` and needs to know whether that concrete invocation passes
preparation:
```go
request := promptkit.RunRequest{
PromptID: "meeting.summary",
Vars: map[string]string{"audience": "project leads"},
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
}
if err := engine.ValidateRequest(ctx, request); err != nil {
return err
}
```
Unlike `InspectRequirements`, this checks supplied values, the selected template
branches, and request bindings through the existing preparation path. It checks
only this invocation, does not aggregate errors, and loads every supplied input
according to ordinary preparation behavior, even if a branch does not use it.
It makes no model call and returns no prepared snapshot, but artifact and schema
resource I/O and its time and memory costs remain. Success does not certify
provider acceptance or later execution success. See the method's GoDoc for its
exact error and context behavior.
Scriptorium can map the returned error to its own diagnostics and exit status.
Its `validate` and `run` commands should construct equivalent requests for
equivalent caller inputs so their preparation behavior agrees. Separate
subprocess calls prepare independently; an opaque `PrepareExecution` handle
cannot be transferred from `validate` to `run`. Use `PrepareExecution` when one
process and engine must execute the exact prepared snapshot.
## Prepare Without Model Execution ## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile, [`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
@@ -114,8 +457,11 @@ prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
The maintained The maintained
[offline preparation example](../../examples/go-library/prepare/main.go) [offline preparation example](../../examples/go-library/prepare/main.go)
shows a complete runnable setup with a prompt file, in-memory profile, and shows a complete runnable setup with a prompt file, in-memory profile, and
inline input. Exact request requirements and prepared-result fields belong to inline input. It demonstrates validation and preparation with the same request
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go). to make the two operations concrete; this performs preparation twice and is
illustrative, not a requirement to validate before `Prepare` or `Run`. Exact
request requirements and prepared-result fields belong to the
[`RunRequest` and `PreparedRun` GoDoc](../../types.go).
## Prepare Now And Execute The Same Snapshot Later ## Prepare Now And Execute The Same Snapshot Later
@@ -268,8 +614,8 @@ result, err := engine.Run(ctx, request)
These messages are already rendered: Promptkit does not template or resolve These messages are already rendered: Promptkit does not template or resolve
files in them, and they can contain sensitive model output or application files in them, and they can contain sensitive model output or application
feedback. Promptkit remains stateless; every `Run` call re-resolves its current feedback. Every `Run` call uses the engine's captured configuration; the
sources and the application owns any semantic retry budget. When a application owns any semantic retry budget. When a
pre-execution equality check is required, use `PrepareExecution` and compare pre-execution equality check is required, use `PrepareExecution` and compare
its opaque rendered-prompt hash before invoking `RunPrepared`. its opaque rendered-prompt hash before invoking `RunPrepared`.
@@ -282,84 +628,48 @@ replace execution settings or the complete output contract.
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement, The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
copy, and credential behavior. The copy, and credential behavior. The
[framework format reference](../formats.md) defines how those request values [framework format reference](../formats.md) defines how those request values
interact with prompt definitions, file-backed and application fallback interact with prompt definitions, layered profiles, built-ins, schemas, and
profiles, built-ins, schemas, and framework defaults. framework defaults.
For programmatic profiles, For programmatic profiles, use [`ProfileSource`](../../profiles.go) with identified
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary `ProfileConfig` values. Nil fields are omitted; pointers preserve explicit
OpenAI-compatible settings into a value accepted by `WithProfiles`. zero, false, and blank values. Raw `configkit.Mapping` expresses explicit null.
### Migrate From The Previous Configuration API
Replace `Config.PromptDir` and `Config.ProfileDir` with selected ConfigKit
inputs in `Config.Sources`. The removed `WithPromptFS`, `WithPromptFile`,
`WithProfileFS`, `WithProfileFile`, `WithFallbackProfileFS`, `WithProfiles`,
and `WithBackend` options have no compatibility wrappers. Use
`DirectoryInputs` or `FilesystemInputs` for a strict-composition file group,
ConfigKit file inputs for individual files, and `ProfileSource`,
`BackendSource`, or `configkit.Mapping` for programmatic values. Backend IDs
are fields in configuration, not registrations. Choose caller underlays in
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
highest-priority input.
Move resource IDs into identified `backends`, `profiles`, and `prompts` lists;
each prompt ID identifies one installed definition without a version label.
Convert `base_profile` to ConfigKit `extends: {id: parent}`. For the current
collection identity mapping, ID-only prompt selection, and required
caller-owned choices, see the
[configuration migration guide](configuration-migration.md). Exact current
format, reference, inheritance, and presence semantics belong to the
[framework format reference](../formats.md). Prompt text and static templates
are captured at construction, not automatically reloaded between runs. Replace
the engine to pick up changed definitions or message files.
### Choose A Profile Ownership Model ### Choose A Profile Ownership Model
Choose the profile source that matches the application's continuity needs: Supply backend and profile definitions through the configuration sources your
application selects. PromptKit does not install provider catalogs or built-in
- Select `promptkit-openrouter-standard` or `promptkit-openrouter-auto` when profile inventories; conventional backend ID constants are names only. Define
the maintained literal name is the desired consumer contract. Their current the model, endpoint, credentials lookup, and settings that your application
selections are in the [format reference](../formats.md#built-in-profile-catalog). intends to support. A profile may inherit a sibling profile by its local key in
- Select an external catalog profile directly when the application accepts that the same root or collection list; its backend reference uses the exact
profile's catalog volatility. The selected external inventories are linked configured identity. Caller-selected source layers determine the final values. The
from the [format reference](../formats.md#external-profile-inventories). [format reference](../formats.md) describes the definition fields and
- Define a complete application-owned profile with an explicit backend or composition rules.
endpoint, model, and settings when the application owns those choices.
An application profile that inherits an external base follows later changes to
that base and fails if the base disappears; it is not a frozen copy. Promptkit
tests the catalog versions selected in its module graph. Independently advancing
a catalog dependency can remove an inherited base and produce an untested
combination. Pinning the resolved module graph preserves the embedded data for
a build, but cannot preserve provider availability. Running binaries neither
download catalog updates nor automatically replace withdrawn models. See the
[release policy](../policy/release.md#catalog-profile-compatibility) for the exact
cross-version guarantees.
### Alias A Built-In Profile
Give an application-owned profile ID a built-in base when prompts should select
the application ID while inheriting the built-in target. The child can override
only the setting it owns:
```go
promptkit.WithProfiles(promptkit.Profile{
ID: "weather-light",
BaseProfileID: "promptkit-openrouter-standard",
ReasoningEffort: "high",
})
```
Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
reported selected profile. See the [profile inheritance format
reference](../formats.md#profile-inheritance) and the
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
behavior. If the base is an external profile, apply the volatility guidance
above when deciding whether to use an application-owned complete definition
instead.
### Use The Rakestrawhome Built-In Profile
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
prepared result identifies the selected built-in through
`BackendRakestrawHome`:
```go
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
ProfileID: "rakestrawhome-gemma-4-31b",
Inputs: inputs,
})
if err != nil {
return err
}
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
}
```
Do not register `rakestrawhome` manually. When adopting this built-in, remove
an existing `WithBackend` registration with that exact ID; retaining it causes
the intentional duplicate-ID configuration error. Direct request credentials
and runtime endpoint overrides remain supported under their ordinary GoDoc and
format contracts.
### Inspect A Profile Before Prompt Work ### Inspect A Profile Before Prompt Work
@@ -424,19 +734,15 @@ Choose the smallest configuration that fits how the endpoint will be reused.
#### Use An Endpoint-Only Profile #### Use An Endpoint-Only Profile
Put the endpoint directly on an in-memory profile when only that profile needs Put the endpoint directly on a programmatic profile when only that profile needs
it and shared backend identity or capacity policy is unnecessary: it and shared backend identity or capacity policy is unnecessary:
```go ```go
engine, err := promptkit.NewEngine(promptkit.Config{ endpoint, model := "http://localhost:8000/v1", "example-model"
PromptDir: "prompts", profile := promptkit.ProfileSource("local profile", []promptkit.ProfileConfig{
}, {ID: "local-summary", Endpoint: &endpoint, Model: &model},
promptkit.WithProfiles(promptkit.Profile{ })
ID: "local-summary", // Add configkit.Input{Source: profile} to a caller underlay.
Endpoint: "http://localhost:8000/v1",
Model: "example-model",
}),
)
``` ```
Endpoint-only profiles have an empty backend ID and remain unrestricted by Endpoint-only profiles have an empty backend ID and remain unrestricted by
@@ -448,59 +754,47 @@ Use `LocalBackend` when profiles should share the conventional `local`
identity, endpoint, and concurrency limit: identity, endpoint, and concurrency limit:
```go ```go
engine, err := promptkit.NewEngine(promptkit.Config{ backendID, model := promptkit.BackendLocal, "example-model"
PromptDir: "prompts", localBackend := promptkit.LocalBackend("http://localhost:8000/v1", 2)
}, localBackend.ID = backendID
promptkit.WithBackend( backend := promptkit.BackendSource("local backend", []promptkit.BackendConfig{
promptkit.LocalBackend("http://localhost:8000/v1", 2), localBackend,
), })
promptkit.WithProfiles(promptkit.Profile{ profile := promptkit.ProfileSource("local profile", []promptkit.ProfileConfig{
ID: "local-summary", {ID: "local-summary", Backend: &backendID, Model: &model},
BackendID: promptkit.BackendLocal, })
Model: "example-model", // Add both sources as inputs in a caller underlay.
}),
)
``` ```
The helper is explicit: it does not pre-register a backend or read environment The helper does not choose an ID or read environment variables. A positive
variables. Supplying a positive limit leaves queue capacity omitted, so normal concurrency limit with omitted queue capacity selects the default waiting
backend registration selects the existing default waiting capacity of 1024. capacity of 1024.
The returned value still enters the engine through `WithBackend`.
#### Configure A Complete Backend #### Configure A Complete Backend
Use a keyed `Backend` value for authentication, extra request parameters, an Use an identified `BackendConfig` value for authentication, extra request parameters, an
explicit queue capacity, a custom ID, or multiple local endpoints: explicit queue capacity, a custom ID, or multiple local endpoints:
```go ```go
noWaiting := 0 endpoint, keyName, limit, noWaiting := "http://gpu-host:8000/v1", "LOCAL_GPU_API_KEY", 2, 0
engine, err := promptkit.NewEngine(promptkit.Config{ backend := promptkit.BackendSource("GPU backend", []promptkit.BackendConfig{
PromptDir: "prompts", {
}, ID: "local-gpu",
promptkit.WithBackend(promptkit.Backend{ Endpoint: &endpoint, APIKeyEnv: &keyName,
ID: "local-gpu", ConcurrencyLimit: &limit, QueueCapacity: &noWaiting,
Endpoint: "http://gpu-host:8000/v1", ExtraParams: map[string]any{"provider_option": "enabled"},
APIKeyEnv: "LOCAL_GPU_API_KEY", },
ExtraParams: map[string]any{"provider_option": "enabled"}, })
ConcurrencyLimit: 2,
QueueCapacity: &noWaiting,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "gpu-summary",
BackendID: "local-gpu",
Model: "example-model",
}),
)
``` ```
Use distinct custom IDs when registering multiple local endpoints. Place `backend` in a caller underlay and select `"local-gpu"` from a
Registrations belong to one engine and custom IDs cannot replace built-ins. `ProfileConfig.Backend` pointer or an identified file profile. All backend
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go) contributions use the same ConfigKit layering rules, including overlays of
defines exact construction, validation, copying, uniqueness, concurrency, and maintained definitions. The [`BackendConfig` and `LocalBackend` GoDoc](../../backends.go)
request-default behavior. defines exact omission and construction behavior.
Both file-backed and in-memory profiles select a registration through Both file-backed and programmatic profiles select a backend through
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain `backend` or `ProfileConfig.Backend`. Profile and request endpoint overrides retain
that routing and capacity identity. `PreparedRun.SelectedBackendID`, that routing and capacity identity. `PreparedRun.SelectedBackendID`,
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID` `RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
expose it to consumers and injected model clients. Endpoint-only profiles expose it to consumers and injected model clients. Endpoint-only profiles
@@ -508,7 +802,7 @@ remain supported and expose an empty backend ID.
### Limit Backend Concurrency ### Limit Backend Concurrency
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many Set `BackendConfig.ConcurrencyLimit` when a backend needs protection from too many
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
example above, selects the default waiting capacity of 1024. example above, selects the default waiting capacity of 1024.
@@ -517,24 +811,23 @@ zero:
```go ```go
noWaiting := 0 noWaiting := 0
backend := promptkit.Backend{ backend := promptkit.BackendConfig{
ID: "local-gpu", Endpoint: &endpoint,
Endpoint: "http://gpu-host:8000/v1", ConcurrencyLimit: &limit,
ConcurrencyLimit: 2, QueueCapacity: &noWaiting,
QueueCapacity: &noWaiting,
} }
``` ```
The pointer distinguishes an explicit zero from omission. Keep using keyed The pointer distinguishes an explicit zero from omission. Use identified
`Backend` literals so additive configuration fields remain source-compatible. `BackendConfig` literals inside `BackendSource` for programmatic definitions.
Capacity belongs to one engine and the selected backend ID; endpoint-only Capacity belongs to one engine and the selected backend ID; endpoint-only
profiles and custom backends without a configured limit remain unrestricted. profiles and custom backends without a configured limit remain unrestricted.
Exact validation, defaulting, ownership, and concurrency semantics belong to Exact validation, defaulting, ownership, and concurrency semantics belong to
the [`Backend` GoDoc](../../backends.go). the [`BackendConfig` GoDoc](../../backends.go).
## Credentials ## Credentials
File-backed profiles name an environment variable; in-memory profiles can File-backed and programmatic profiles can name an environment variable or
require a direct request key. Direct keys are request-scoped and are excluded require a direct request key. Direct keys are request-scoped and are excluded
from supported JSON values and the package's `String` and `GoString` from supported JSON values and the package's `String` and `GoString`
summaries. The exact precedence and redaction guarantees belong to summaries. The exact precedence and redaction guarantees belong to
@@ -568,12 +861,13 @@ the application policy that motivated the injection.
Use `errors.Is` with the Use `errors.Is` with the
[public error sentinels and operation GoDoc](../../engine.go). The declarations [public error sentinels and operation GoDoc](../../engine.go). The declarations
distinguish invalid construction, invalid requests, absent sources, distinguish invalid construction, invalid requests, absent selections,
source-loading failures, collaborator failures, and operational validation collaborator failures, and operational validation
failures. Specific request conditions may also match the broader failures. Specific request conditions may also match the broader
`ErrInvalidRequest`, and injected collaborator identities are preserved where `ErrInvalidRequest`, and injected collaborator identities are preserved where
documented. Invalid or duplicate backend registrations match documented. Invalid effective definitions, including unknown backend
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`. references, match `ErrInvalidConfig` during construction. ConfigKit load
errors remain available through `errors.As`.
When a limited backend has admitted all active and waiting calls, handle When a limited backend has admitted all active and waiting calls, handle
`ErrCapacityExceeded` separately from request errors and provider failures: `ErrCapacityExceeded` separately from request errors and provider failures:

View File

@@ -43,13 +43,13 @@ Before starting work:
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. | | Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). | | Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. | | Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. | | Caller-supplied configuration sources | The [internal source document](internal/sources.md), [framework format reference](formats.md), and [testing policy](policy/testing.md). |
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. | | Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. | | Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. | | Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). | | Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. | | Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. | | Licensing, dependencies, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). | | Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
For cross-cutting changes, follow every applicable row. Do not create For cross-cutting changes, follow every applicable row. Do not create
@@ -59,37 +59,42 @@ exist.
## Maintainer Validation ## Maintainer Validation
For ordinary work, select validation proportionate to the change. Do not repeat For ordinary work, select validation proportionate to the change. Do not repeat
successful checks for unchanged content merely to create a commit. The full successful checks for unchanged content merely to create a commit. During
release-candidate checks have one executable entry point from the repository root: development, run relevant Go tests and vet, format changed Go files, and execute
the maintained examples when they are affected:
```sh ```sh
./scripts/release-check.sh go test ./...
go vet ./...
gofmt -w path/to/changed.go
git diff --check
``` ```
The [script](../scripts/release-check.sh) owns the executable checks: Release candidates use `./scripts/release.sh VERSION`, which runs the selected
race-enabled tests, vet, the three offline consumer examples with semantic output profile checks and publishes the tag. The [release procedure](release.md)
assertions, tracked Go formatting, local Markdown links, module/workspace describes its requirements and recovery. For an optional diagnostic of a
hygiene, whitespace, and the credential-pattern scan. It does not publish, committed candidate without publication, run `./scripts/releasekit.sh check
change files, validate release-note structure, or retire roadmaps. It requires Go, VERSION`; this is not a required pre-publication step.
Git, a POSIX shell, and Python 3. Tests and examples require no provider
The example workflows are covered by the Go suite; use
`go test ./examples/go-library/...` to run them separately during development.
The focused `python3 scripts/test-release.py` command exercises the release
tooling against disposable local repositories; use it when changing that
tooling, not as a recurring release gate. Tests and examples require no provider
credentials or live inference calls. credentials or live inference calls.
The script can run on intended uncommitted changes during development. Ordinary validation can run on intended uncommitted work. Release validation
Formatting and documentation checks use tracked files; stage intended new uses committed candidate contents and the selected local Go installation.
files before final validation so those checks include them. The release
procedure separately enforces a clean, committed candidate.
### Change Review ### Change Review
Inspect `git status --short --untracked-files=all` and the complete diff before Inspect `git status --short --untracked-files=all` and the complete diff before
accepting a change. Reject unintended files, credentials, private data, accepting a change. Reject unintended files, credentials, private data,
generated binaries, test output, downloaded assets, and template residue. generated binaries, test output, downloaded assets, and template residue.
The automated credential scan does not replace review of the actual change.
For releases, follow the [release procedure](release.md). Its publication For releases, follow the [release procedure](release.md). Do not repeat its
command runs the full local checks once on the final committed candidate and candidate checks as a separate sign-off. Roadmap retirement remains preparation
automates release-note structure and Git checks. Do not repeat ordinary change housekeeping.
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
commits. Ordinary commits may leave unrelated work in the workspace; preserve commits. Ordinary commits may leave unrelated work in the workspace; preserve

View File

@@ -2,394 +2,392 @@
## Purpose ## Purpose
This document is the canonical contract for Promptkit prompt-definition, This is the canonical reference for Promptkit's backend, profile, prompt, and
profile, and schema files. The [Go package consumer guide](consumers/pkg-promptkit.md) schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
explains how to select these sources and invoke the engine. The how to select sources and use an engine; the
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md) [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
owns the resulting outbound wire behavior. owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md)
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
This reference describes what Promptkit does with the final effective values.
Prompt and profile sources recursively discover files ending in `.yaml` or A configuration document may contain top-level `backends`, `profiles`, and
`.yml`. Each prompt-definition and profile file contains exactly one YAML `prompts` lists, plus a `collections` mapping of namespaced resource groups.
document; comments and trailing whitespace are allowed. YAML decoding is Each resource has an explicit literal `id`, not a filename-derived identity.
strict: unknown fields are errors for the selected definition. Definitions are IDs and references must be nonblank and remain exact: Promptkit does not trim or
selected by their YAML `id`, not their file name or directory. re-key them. Unknown fields are errors. Promptkit installs no provider or
profile catalog; callers select the definitions they need.
```yaml
backends:
- id: local
endpoint: http://localhost:8000/v1
concurrency_limit: 2
profiles:
- id: local-summary
backend: local
model: example-model
prompts:
- id: meeting.summary
default_profile: local-summary
messages:
- role: user
content: Summarize this note.
output:
format: text
validation_mode: none
collections:
support:
release_version: "1.4.0"
backends:
- id: offline
endpoint: https://example.invalid/v1
profiles:
- id: base
backend: support/offline
model: offline-model
- id: fast
extends: {id: base}
timeout_seconds: 30
prompts:
- id: answer
default_profile: support/fast
messages:
- role: user
content: Answer the selected caller input.
output:
format: text
validation_mode: none
```
## Source Headers, Collections, And Resource IDs
An input may omit a source header. When present, the source-root `version` is
reserved for ConfigKit's specification header and accepts only the exact string
`"2"` or strict decimal integer `2`; null and other values are invalid. This
header is distinct from a collection's `release_version` and ConfigKit's own
software version. ConfigKit checks and consumes accepted headers on non-main
inputs; a declaration on the main input remains visible in explicit
configuration inspection. Headerless inputs still undergo ordinary final-value
validation. ConfigKit owns generic import and source-history behavior.
For a standalone prompt item, a root `version` is only a source header, not
prompt metadata.
The root resource lists contain unqualified IDs. Each `collections` entry
has a namespace key and may contain `release_version` plus the same three
resource lists. Collection resource IDs are local keys; their runtime IDs
are `<namespace>/<local-key>`. Root and collection keys may be the same. A
namespace and local key must be nonblank and slash-free, and references must
name their intended exact ID; mounting a source does not rewrite references.
For example, the `support/fast` profile above stores `support/offline` as its
backend ID. `extends: {id: base}` refers to a sibling in the same resource
list. It is not a qualified or cross-collection reference.
`release_version` is an optional opaque collection label. Absent or null means
no label; an explicitly empty string is retained. It does not select a resource
or change execution. Runtime lookups
use the captured effective resource indexes. `InspectConfiguration` is an
explicit operator view of composed settings and provenance. Normalization and
defaults appear in Promptkit's interpreted execution views, not as changes to
that composed tree. Source history is not a second runtime resolver, and the
retained effective shape is not flattened or rewritten into those indexes.
For examples of explicitly selecting native collection inputs and inspecting
inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
[maintained preparation example](../examples/go-library/prepare/main.go). For
breaking identity and prompt-selection changes, see the
[configuration migration guide](consumers/configuration-migration.md).
Every effective backend, profile, and prompt definition is validated when
`NewEngine` constructs the engine, even if it is never selected. A profile may
be partial: an endpoint and model need only be complete after request overrides
are applied. A prompt's default profile similarly need not be runnable without
an overriding request. Missing request data, artifacts, schemas, credentials,
and provider work remain runtime concerns.
The engine captures effective definitions, referenced message-file contents,
and static template syntax at construction. It does not reread configuration or
message files on ordinary runs; construct a new engine to observe edits.
Schema and request artifact reads still happen during preparation. Use
`InspectConfiguration` for an explicit effective-configuration or provenance
report, `InspectProfile` for backend-completed profile settings, and preparation
for request-specific rendered details. For identity discovery before inspection,
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
## Prompt Definitions ## Prompt Definitions
A prompt definition describes inputs, Go-template messages, an optional The `prompts` list contains definitions selected by their literal IDs:
default profile, and an output contract.
```yaml ```yaml
id: meeting.summary prompts:
version: "1.0.0" - id: meeting.summary
default_profile: local-summary default_profile: local-summary
description: Summarize a synthetic meeting note. description: Summarize a synthetic meeting note.
session_id: '{{.session}}' session_id: '{{.session}}'
inputs: inputs:
- name: note - name: note
required: true required: true
content_type: text/plain content_type: text/plain
description: Meeting note to summarize. description: Meeting note to summarize.
messages: messages:
- role: system - role: system
content: Return a concise summary. content: Return a concise summary.
cache_control: cache_control:
type: ephemeral type: ephemeral
ttl: 1h ttl: 1h
- role: user - role: user
content_file: ./summary.user.md content_file: ./summary.user.md
output: output:
format: markdown format: markdown
validation_mode: basic validation_mode: basic
repair_attempts: 0 repair_attempts: 0
``` ```
| Field | Required | Meaning | `messages` must be nonempty and `output` must be a valid mapping. Optional
| --- | --- | --- | `default_profile`, `description`, `session_id`, `inputs`, and cache-control
| `id` | yes | Non-empty prompt identifier used by `RunRequest.PromptID`. | fields may be absent or cleared with null where appropriate. A null whole
| `version` | yes | Non-empty version selected by an optional `RunRequest.PromptVersion`. | definition, required role, output format, or validation mode is invalid.
| `default_profile` | no | Non-empty profile ID used when the request omits `ProfileID`. | `required` and `repair_attempts` default to false and zero, respectively, and
| `description` | no | Human-readable description. | are not nullable. `InspectPrompt` selects the same captured definition without
| `session_id` | no | Go template rendered from request variables and input helpers. | rendering or resolving a profile.
| `inputs` | no | Declared input metadata. |
| `messages` | yes | One or more chat-message templates. |
| `output` | yes | Output format and validation settings. |
When a request omits a version, the selected prompt ID must identify exactly
one definition. When it supplies a version, the ID and version pair must be
unique.
Exact prompt inspection uses this same configured source, strict decoding,
referenced content-file resolution, and ID/version selection. It reports the
selected definition's declared metadata without changing the prompt format or
executing the definition.
### Inputs ### Inputs
Each `inputs` item has these fields: Each `inputs` item may contain `name`, `required`, `content_type`, and
`description`. `name` is required, nonblank, and unique within the prompt.
| Field | Required | Meaning | Omitted `required` means false. Null `inputs` means no declarations. Requests
| --- | --- | --- | supply inline or file-backed `ArtifactRef` values; declared required inputs
| `name` | yes | Non-empty name used by the request input map and `input` template helper. Names must be unique. | must be present. A template reference also needs its named input, even when
| `required` | no | When true, preparation fails if the request omits the input. The default is false. | undeclared. Extra request inputs are allowed.
| `content_type` | no | Expected media-type metadata. |
| `description` | no | Human-readable input description. |
Requests supply inputs as inline or file-backed `ArtifactRef` values. Declared
required inputs must be present. A template reference also requires the named
input to exist, whether or not it was declared. Extra request inputs are
allowed.
### Messages And Templates ### Messages And Templates
Each message has a `role` that Promptkit trims and lowercases. It must then be Each message needs a role and exactly one nonnull `content` or `content_file`.
exactly one of `developer`, `system`, `user`, or `assistant`; blank, custom, An explicitly empty inline `content` is valid. Null clears either content
`tool`, and `function` roles are invalid. This intentionally tightens the selection. Roles are trimmed and lowercased, then must be `developer`,
previous nonblank-string rule. Consumers migrating to the next minor release `system`, `user`, or `assistant`. Static template syntax is checked during
must update any nonstandard prompt-definition roles before upgrading. construction; missing variables and named inputs fail when rendered.
Request variables are template data (`{{.audience}}`), and
`{{input "note"}}` renders an input body.
Each message also has exactly one of: `content_file` uses the effective field's own source context, including when
inherited or imported. Its relative path is resolved against that source's
base directory within its rooted filesystem. Absolute paths and lexical
escapes from the selected root fail. Host-file sources also prevent symlink
escapes. Promptkit preserves the path text, including meaningful whitespace,
after checking that it is not blank. A source without a unique usable resource
context must supply one explicitly or override the field. Content is captured
at construction, not reopened for each request.
- `content`, containing an inline Go template; or Optional `session_id` uses the same template variables and input helper. Its
- `content_file`, naming a file whose contents are the Go template. rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
points. A nonblank direct request session ID bypasses the template.
`content_file` must be a relative path. It resolves from the directory that `InspectRequirements` can enumerate recognizable dependencies from captured
contains the prompt file and must remain within the configured prompt source templates before artifact content is available. Its conservative syntax
root; parent components are allowed only when the resolved target remains coverage does not change template validity or rendering; see the [consumer
inside that root. Absolute paths and paths that escape the root are rejected. guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
Operating-system directory and single-file sources also reject symlink targets the method's [GoDoc](../requirements.go) for that inspection boundary.
outside the root, while injected `fs.FS` sources apply containment in that
filesystem's relative path namespace. For `WithPromptFile`, the source root is
the directory containing the selected prompt file. Promptkit uses the parsed
path text exactly after checking separately that it is not blank, so leading
and trailing whitespace can name real filesystem entries.
Request variables are the template data, so a variable named `audience` is
referenced as `{{.audience}}`. The `{{input "note"}}` helper renders the body
of a named input. Missing variables and input references are errors.
The optional `session_id` uses the same template data and input helper. Its
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
points. A nonblank direct request session ID bypasses this template completely;
a blank direct value leaves the template behavior unchanged.
### Cache Control ### Cache Control
`cache_control` is optional and has these fields: Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
which may be empty or `1h`. Promptkit retains this metadata on rendered
| Field | Required | Values | messages; the integration contract defines its wire representation.
| --- | --- | --- |
| `type` | yes | `ephemeral` |
| `ttl` | no | Empty or `1h` |
Promptkit preserves cache-control metadata on the rendered message. The
outbound integration determines its wire representation.
### Output Contract ### Output Contract
| Field | Required | Values or behavior | | Field | Required | Values or behavior |
| --- | --- | --- | | --- | --- | --- |
| `format` | yes | `text`, `markdown`, or `json`. | | `format` | yes | `text`, `markdown`, or `json` |
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. | | `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` |
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. | | `schema_path` | for `json_schema` | Path in the selected schema source |
| `repair_attempts` | no | Integer from zero through three; omitted means zero. A positive value requires `basic`, `json`, or `json_schema` validation. | | `repair_attempts` | no | Integer 0–3; default 0. Positive values need `basic`, `json`, or `json_schema` |
The validation modes behave as follows: `none` skips content validation; `basic` requires nonempty content; `json`
accepts any valid JSON value; `json_schema` checks the selected schema.
`format` determines output artifact metadata. JSON Schema mode also supplies
structured-output metadata to compatible clients. Plain `json` does not
request a provider-native JSON-object constraint. A positive repair budget
counts additional generations after failed validation; exhaustion returns the
last candidate and its failed validation result. Operational errors remain
errors. A request-level `OutputContract` replaces the whole configured one,
not individual fields; an empty request format defaults to `text`.
- `none` skips content validation; ## Backend Definitions
- `basic` requires non-empty generated content;
- `json` requires valid JSON; and
- `json_schema` requires valid JSON that satisfies the selected schema.
`format` controls output artifact metadata. JSON Schema mode also supplies the Backend definitions are supplied through caller-selected configuration
schema to compatible model clients as structured-output metadata. Plain `json` sources. The conventional backend IDs are names only; PromptKit does not
validation accepts every valid JSON value and does not request a provider-native install definitions for them.
JSON-object constraint.
`repair_attempts` counts additional generation calls after a failed validation. A backend definition may have `endpoint`, `api_key_env`,
Zero is single-pass. With a positive eligible budget, Promptkit stops at the `concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
first valid candidate. If the budget is exhausted, it returns the final final endpoint is required and must be an absolute HTTP(S) URL with a host, no
candidate and its complete failed validation result; generation and operational user information, query, or fragment. An omitted or null API-key environment
validation failures remain errors. `none` never permits repair. name means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
negative value is invalid. For positive concurrency, omitted/null queue
capacity defaults to 1024. An explicit queue capacity, including zero,
requires positive concurrency and must be nonnegative. Unlimited concurrency
has no queue. Provider maps contain JSON-compatible values; null entries are
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
A request-level `OutputContract` replaces the complete prompt output contract. `smoke_tests` is optional backend metadata for explicit diagnostic operations.
It does not merge individual fields. If its format is empty, Promptkit uses It may contain `default_profile`, `reachability`, and `authentication`:
`text`.
## Built-In Backends ```yaml
backends:
- id: openrouter
endpoint: https://openrouter.ai/api/v1
smoke_tests:
default_profile: application-profile
reachability:
endpoint: https://status.example/ready?check=1
authentication:
kind: openrouter_key
endpoint: https://openrouter.ai/api/v1/key
```
Every engine provides these reserved OpenAI-compatible backend IDs. Consumers An omitted or null `smoke_tests` block disables these metadata defaults.
must not register either ID with `WithBackend`; exact registration and `default_profile` is preserved as an exact, nonblank profile ID; its existence
reservation behavior belongs to the [`Backend` GoDoc](../backends.go). is checked only if an inference diagnostic selects that default. Omitted/null
`default_profile` means there is no inference default. Omitted/null
`reachability` or its endpoint uses the effective backend endpoint. An explicit
reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
host; unlike the backend base endpoint, its path and query are allowed. Null
means fallback, while an explicit blank value is invalid.
| ID | Base endpoint | API-key environment variable | Active generation limit | Default queue capacity | Omitted/null `authentication` means no authentication adapter is declared. A
| --- | --- | --- | ---: | ---: | non-null object requires `kind`, which is trimmed and case-normalized; the
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 | 1024 | recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 | 1024 | uses the adapter's backend-relative destination (`openrouter_key` appends
`/key` to the effective backend base); an explicit endpoint follows the same
full-URL rules as reachability. `none` declares authentication inapplicable.
These values are configuration only: engine construction does not contact a
backend.
These fields are resolved through the ordinary ConfigKit source layers before
Promptkit validates the final effective values. Sibling inheritance can overlay
individual nested fields; a later same-ID resource replaces the whole record.
An explicit null can clear the whole `smoke_tests` block or one of its nested
objects. An endpoint set to null uses
its documented fallback, while an explicit blank endpoint is invalid. The
metadata does not change ordinary generation credential requirements or
provider settings.
## Profile Definitions ## Profile Definitions
A profile supplies model execution settings: Profiles have literal IDs and may inherit a sibling parent with
`extends`:
```yaml ```yaml
id: local-summary profiles:
backend: openrouter - id: local-summary
model: example-model backend: local
temperature: 0.2 model: example-model
max_tokens: 500 temperature: 0.2
top_p: 0.95 max_tokens: 500
timeout_seconds: 90 timeout_seconds: 90
service_tier: flex extra_params:
reasoning_effort: medium provider_option: enabled
extra_params: - id: local-summary-fast
provider_option: enabled extends: {id: local-summary}
timeout_seconds: 30
reasoning_effort: low
``` ```
A derived profile can use a named base and override only the settings it owns: Profiles may also supply `endpoint`, `top_p`, `service_tier`, `api_key_env`,
and `api_key_required`. Promptkit does not infer a backend from an endpoint or
```yaml model. A profile with only an endpoint has no backend capacity identity.
id: local-summary-fast Nonblank endpoints follow the backend URL rules; temperature is 0–2, top_p
base_profile: local-summary is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept
timeout_seconds: 30 JSON-compatible values with nonempty keys and cannot collide with the built-in
reasoning_effort: low client's [standard request fields](integrations/openai-compatible-chat.md#request-body).
``` Credential values do not belong in profiles: raw `api_key` is invalid.
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
| `base_profile` | no | One optional parent profile ID. A derived profile may inherit target fields from it. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. |
| `max_tokens` | no | Integer zero or greater. |
| `top_p` | no | Number from 0 through 1. |
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
| `service_tier` | no | Provider-specific request tier. |
| `reasoning_effort` | no | Provider-specific reasoning setting. |
| `api_key_env` | no | Optional environment-variable lookup source for an API key. |
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
Raw `api_key` is prohibited in profile YAML. Store only an environment
variable name in `api_key_env`.
A standalone profile must provide a model and at least one of `backend` or
`endpoint`. A derived profile may omit those target fields because its selected
base chain can provide them. Local parsing still validates a derived profile's
own ID, supplied endpoint, execution-setting bounds, and `extra_params`.
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
profiles remain supported and have no effective backend ID.
The engine always provides the built-in `openrouter` and `rakestrawhome` IDs.
Consumers can add engine-scoped IDs with
[`WithBackend`](../backends.go); exact registration validation belongs to its
GoDoc.
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
objects with string keys. Keys must be non-empty. With the built-in client,
they also cannot collide with the standard fields listed in the
[outbound request contract](integrations/openai-compatible-chat.md#request-body).
Excessively deep or large JSON-shaped values are rejected for safety.
### Defaults And Overrides ### Defaults And Overrides
Execution settings resolve in this order: After effective profile inheritance, Promptkit applies the selected backend's
defaults once, then request `ExecutionTargetOverride` values. Backend-derived
settings are not inheritance input: changing a child profile's backend does
not carry the parent's derived endpoint or credential source into it. Changing
only a model or backend does not clear unrelated explicitly inherited fields.
1. the framework timeout baseline; | Profile field | Absent | Explicit value | Explicit null |
2. the selected backend, when the profile names one; | --- | --- | --- | --- |
3. the selected profile; and | `backend` | No backend | Exact backend ID; blank means none | No backend |
4. request `ExecutionTargetOverride` values. | `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
| `api_key_required` | False | Boolean, including false | False |
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
The framework baseline is: An empty mapping does not clear inherited map entries; null clears the whole
map. Nested maps merge recursively, while scalar and array values replace.
| Setting | Default | Runtime nonempty `ExtraParams` overrides replace the entire configured map.
| --- | --- | Request overrides otherwise follow the [public value GoDoc](../types.go).
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. | For example, a runtime `extra_params["extends"]` key is ordinary provider
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. | data, not a configuration inheritance instruction. An explicit zero timeout
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. | does not disable the caller context or HTTP transport timeout.
| `timeout_seconds` | `600` |
Numeric zero in a file or in-memory profile does not select a numeric value.
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
request overrides use pointers, so an explicit zero is retained and sent to
compatible providers. In particular, an explicit request `timeout_seconds` of
zero disables the per-generation deadline while leaving the caller context and
transport timeout intact.
Non-empty profile strings replace backend defaults, and non-empty request
strings replace both. Request reasoning is the exception: a nil
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
trims and replaces it, and a pointer to a blank string clears it. Backend
identity is retained when either layer overrides the endpoint, so the override
also retains any engine-local capacity policy configured for that backend.
Capacity configuration belongs to the Go
[`Backend` API](../backends.go), not prompt or profile YAML. A non-empty
`extra_params` map at each layer replaces the entire lower-precedence map
rather than merging keys.
The [outbound integration contract](integrations/openai-compatible-chat.md)
defines how the effective settings are serialized.
### Source And Profile Precedence ### Source And Profile Precedence
An explicit request profile ID takes precedence over the prompt's `NewEngine` composes caller underlays in their given order, followed by an
`default_profile`. If neither is present, preparation fails. Exact profile optional main input. Later layers overlay earlier ones. Inputs within one
inspection instead takes one explicit profile ID and does not use a prompt underlay group are strictly composed: conflicting contributions are errors,
default. not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
in sorted order, but that order does not give one file higher precedence.
An explicitly selected missing/unreadable root fails; an empty root contributes
nothing. Imports are disabled by these helpers unless the caller enables them
on an input. Single files can use ConfigKit inputs directly.
Profile sources resolve matching IDs in this order: Root backends, profiles, and prompts, as well as collection declarations and
their resource lists, follow these same source rules. `CollectionInput`
mounts one explicitly selected source at a collection destination and enables
ConfigKit imports for that input; it does not discover or select sources.
A later same-ID record replaces the lower definition in full. Use a distinct
inherited sibling when a variant should retain its parent's fields.
A request `ProfileID` takes precedence over a prompt `default_profile`; if
neither is selected, preparation fails. Exact profile inspection selects one
explicit ID and does not consult a prompt default.
1. in-memory profiles supplied with `WithProfiles`; ### Resource Inheritance
2. the ordinary configured source selected by a profile file, `fs.FS`, or
configured profile directory;
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
4. Promptkit-owned profiles; and
5. maintained external catalog profiles.
A profile source supplies a complete definition; definitions and their fields ConfigKit resolves `extends: {id: parent}` for backends, profiles, and prompts.
are not merged across sources. A higher-precedence source falls back only when Each child inherits a surviving sibling in the same root or collection list;
the requested profile ID is absent. An invalid matching profile is an error and collection inheritance cannot cross namespaces. Parents are ordinary validated
does not fall back. In-memory `Profile` values follow the same ranges as YAML resources, and profiles may remain partial until request completion. Presence, explicit
profiles. They use `APIKeyRequired` for request-scoped credentials instead of null, and recursively merged maps determine the final fields. Other resource
`api_key_env`. Preparation and exact profile inspection use this same source references are exact IDs and use their qualified form when they select a
precedence. collection resource. There is no Promptkit-specific depth limit or second
base-chain resolver. Missing references and cycles fail at construction. For
When a selected definition names `base_profile`, every profile ID in that generic inheritance and layer mechanics, consult
chain is looked up through this same precedence order. A higher-precedence [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md).
definition therefore shadows a lower-precedence definition of the same base
ID, including a built-in. References are not source-qualified.
### Profile Inheritance
Promptkit resolves one linear base chain of at most 32 profiles, including the
selected profile. It merges settings from the root base to the selected leaf.
The leaf's `id` remains the selected profile identity. Nonblank string fields
(`backend`, `endpoint`, `model`, `service_tier`, `reasoning_effort`, and
`api_key_env`) and nonzero numeric fields replace inherited values. A nonempty
`extra_params` map replaces the complete inherited map rather than merging
keys, and `APIKeyRequired: true` remains true through the chain. Backend and
endpoint are independent: replacing one does not clear the other.
There is no profile-level clearing syntax. Blank strings, zero numbers, false,
and empty maps remain unspecified and inherit from a base. Use existing
presence-aware request overrides where an execution needs an explicit zero or
empty reasoning setting.
An absent directly selected profile reports the ordinary not-found error. Once
the selected profile exists, a missing base, cycle, overlong chain, or
incomplete resolved target is a profile-load failure. Ordinary operations
resolve chains afresh; prepared execution retains the fully resolved target.
## Built-In Profile Catalog
Every built-in profile selects one maintained built-in backend and inherits
that backend's connection and credential metadata. Profile files do not repeat
those values. A configured, application fallback, or in-memory profile with
the same profile ID takes precedence.
Promptkit owns two opt-in profiles. The following definitions are current
selections, not automatic defaults:
| ID | Current definition |
| --- | --- |
| `promptkit-openrouter-standard` | Inherits `gemini-flash-latest`. |
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
Select either name through an existing request `profile_id` or prompt
`default_profile`. Consumers can replace either definition or its inherited
base through the ordinary profile-source precedence above. The
[release policy](policy/release.md#catalog-profile-compatibility) defines the
cross-version guarantee for these names.
## External Profile Inventories
The external profile inventory is defined by the selected catalog modules, not
duplicated here. Browse the exact data selected by this Promptkit version:
- [OpenRouter catalog v2.20260926.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v2.20260926.0/catalog/profiles)
- [Rakestrawhome catalog v2.20260926.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v2.20260926.0/catalog/profiles)
External profile entries are volatile across catalog releases. The
[release policy](policy/release.md#catalog-profile-compatibility) owns the precise
compatibility boundary; a selected module version identifies tested local data,
not live provider availability.
## Schemas ## Schemas
Schemas are JSON documents selected by a prompt or request Schemas are JSON documents selected by a prompt or request `schema_path`.
`schema_path`. For a directory or `fs.FS` source, paths resolve within the Directory and `fs.FS` schema paths resolve within the configured root;
configured source root. Referenced nested schemas resolve relative to the referenced nested schemas resolve relative to the owning schema document.
owning schema document. `WithSchemaFile` exposes one schema, addressed by its `WithSchemaFile` exposes one schema under its base name. An unreadable,
base name. invalid, or unresolvable schema causes an operational validation error.
Valid JSON that fails the schema instead yields a failed validation result.
An unreadable, invalid, or unresolvable schema produces an operational
validation error. Generated content that is valid JSON but does not satisfy the
schema produces a failed validation result.
## Credentials ## Credentials
Credential values belong at the request or environment boundary, never in Only environment-variable names belong in backend/profile definitions, never
prompt, profile, schema, or example files: credential values. A request may supply a direct `APIKey` or an `APIKeyEnv`
override. A direct key takes precedence, followed by a request environment
source, an explicit profile source, and an inherited backend source. A true
profile `api_key_required` suppresses the backend-derived environment source,
but an explicit profile or request environment source remains usable. False
restores ordinary optional-credential behavior. Missing or blank optional
environment values leave the built-in client's Authorization header absent;
required availability is checked at preparation and again during prepared
execution. Injected clients own their credential-resolution behavior.
- a backend or file profile can name an optional environment lookup source Credential values are excluded from JSON results and redacted by public
with `APIKeyEnv` or `api_key_env`; string formatters. ConfigKit inspection redacts credential-shaped dynamic
- an in-memory profile may set `APIKeyRequired` as an explicit local provider keys, but does not make arbitrary prompt/provider content safe to
requirement; disclose. Applications own report disclosure and secret handling.
- a request can provide a direct `APIKey` or override the optional `APIKeyEnv`
source; and
- a direct request key takes precedence over environment lookup.
After a direct request key, the credential-source precedence is request
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
profile with `APIKeyRequired` clears an inherited backend environment name and
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
Named environment sources are optional: when the selected source is absent,
empty, or whitespace-only, the built-in client omits the `Authorization`
header and handles the provider response normally. `APIKeyRequired` is the
only explicit local availability requirement. Promptkit validates required
credential availability during preparation and rechecks it when a prepared
execution runs. Injected clients receive resolved source metadata but define
their own credential-resolution behavior.
Direct keys are excluded from JSON results and redacted by public string
formatters. Environment-variable names may appear in prepared metadata, but
their values do not.

View File

@@ -45,6 +45,45 @@ precedence over the backend default. Only the name reaches prepared metadata;
the environment value is read just before the provider call and is never added the environment value is read just before the provider call and is never added
to the JSON body. to the JSON body.
## Explicit Smoke Diagnostics
Promptkit's explicit smoke diagnostics are separate consumer operations; they
are not part of ordinary `Run`, the offline examples, or release validation.
With the built-in client, inference diagnostics use the same configured client
and `/chat/completions` request as generation. Their content cases are private
to Promptkit and are not application prompt, schema, or provider protocol
extensions. The public
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns request selection,
report facts, and injected-client behavior.
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
one unauthenticated `GET` to the selected diagnostic URL; an observed response
counts as reachable even when its status is non-2xx. Authentication behavior
is selected from backend metadata, not inferred from the host. The
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
backend base, unless configuration supplies a full diagnostic endpoint (see
the [backend format reference](../formats.md#backend-definitions)). It may send
the available bearer credential, but can dispatch without one. A 401 is
recognized as rejection. A 200 is recognized as acceptance only when its JSON
object has a `data` object containing string `label`, Boolean `is_free_tier`,
and numeric `usage`; unrelated fields and account limits are ignored. This is
only a minimal identifying subset of OpenRouter's documented
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
not validation of the full account document. Other statuses and unrecognized
200 responses do not establish acceptance.
Diagnostic HTTP evidence describes the last request observed at Promptkit's
RoundTripper boundary and any response associated with it. It includes the
selected destination without URL user information, response status, and
credential presence when observed. Redirects can change the last observed
destination or strip credentials. This evidence does not prove that bytes
reached a peer or reveal dispatch hidden inside a custom transport. Diagnostic
reports do not retain headers, credentials, provider
bodies, or generated text. A consumer-injected `LLMClient` can support inference
diagnostics, but does not establish HTTP status, destination, reachability, or
upstream credential-supply facts; Promptkit does not fall back to a built-in
transport for HTTP-only diagnostics.
## Request Body ## Request Body
The request body always contains `model` and `messages`. The execution The request body always contains `model` and `messages`. The execution
@@ -78,9 +117,8 @@ The client conditionally includes:
The engine resolves backend, profile, and request extra-parameter maps by The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is whole-map replacement rather than key merging. The resulting effective map is
then merged directly into the top-level body after JSON serialization is merged into the top-level body. Empty keys, values that cannot be encoded, and
verified. Empty keys and collisions with these reserved fields are rejected collisions with these reserved fields are rejected before any provider call:
before any provider call:
- `model` - `model`
- `session_id` - `session_id`

View File

@@ -88,6 +88,17 @@ release, including canceled acquisition without dispatch. A retry wait holds
the logical admission lease but no active permit. Observer callbacks must not the logical admission lease but no active permit. Observer callbacks must not
block on work that requires the same run's admission slot. block on work that requires the same run's admission slot.
## Smoke-Test Capacity
The engine's smoke tester shares this manager and generation coordinator with
the ordinary runner. Inference diagnostics reserve the same logical admission
slot and acquire the same backend-scoped active permits for actual model
attempts, including retries. They therefore participate in the same configured
backend limits as `Run`. Reachability and authentication HTTP probes do not
enter generation admission or consume active permits. The public
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns the caller-visible
operation and injected-client limits.
## Cancellation And Release ## Cancellation And Release
Admission checks the caller context before reserving a slot. A call canceled Admission checks the caller context before reserving a slot. A call canceled
@@ -127,4 +138,7 @@ cover shared and independent backend pools during retry and repair. The
own the public prepared-capacity boundary. The own the public prepared-capacity boundary. The
[root error-boundary tests](../../errors_internal_test.go) own preservation of [root error-boundary tests](../../errors_internal_test.go) own preservation of
the public generation category and context identity when generation is the public generation category and context identity when generation is
canceled. canceled. The assembled
[smoke diagnostic contract tests](../../smoke_integration_contract_test.go)
verify that inference diagnostics share the engine's Run admission and recover
capacity after cancellation or failure.

View File

@@ -41,9 +41,22 @@ original error chain. This keeps injected attempt classifications consistent
with terminal observations without parsing diagnostic text or retrying injected with terminal observations without parsing diagnostic text or retrying injected
clients. clients.
The diagnostic path uses the same private attempt and retry implementation as
ordinary generation. It retains invocation-local transport, response,
generation, usage, capacity-wait, and retry-delay evidence before normal
result-discard and cancellation checks. Built-in preparation also supplies its
frozen request URL to the evidence path before attempts begin; injected
generation has no equivalent HTTP destination. This evidence is returned to
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
observer accounting. The [model-client document](llm.md) describes the
transport-observation boundary, and the root
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.
Focused coordinator tests use controlled transport and wait functions. They Focused coordinator tests use controlled transport and wait functions. They
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
release, and the injected-client path. The capacity manager tests own permit release, and the injected-client path. The
[evidence tests](../../internal/generation/evidence_test.go) own diagnostic
attempt retention and retry evidence. The capacity manager tests own permit
acquisition and idempotent release. Public acquisition and idempotent release. Public
[integration tests](../../reliable_execution_contract_test.go) cover retry, [integration tests](../../reliable_execution_contract_test.go) cover retry,
repair, observation, frozen prepared execution, and shared-backend behavior repair, observation, frozen prepared execution, and shared-backend behavior

View File

@@ -34,9 +34,13 @@ resolved request target may supply the endpoint. Generation preparation then:
1. validates shared execution-setting invariants and the final selected base 1. validates shared execution-setting invariants and the final selected base
endpoint; endpoint;
2. maps the internal request into the OpenAI-compatible chat payload; 2. maps the internal request into the OpenAI-compatible chat payload;
3. validates and merges extra parameters; 3. checks extra-parameter keys against empty and reserved names, then merges
their values into the payload;
4. composes `/chat/completions` through parsed URL path operations; 4. composes `/chat/completions` through parsed URL path operations;
5. resolves authentication and freezes the URL, body, and optional credential. 5. encodes the complete body once on success. If encoding fails, extra values
are checked individually to retain invalid-request classification; otherwise
the complete-request failure remains a request failure;
6. resolves authentication and freezes the URL, body, and optional credential.
Each execution of that prepared request creates a fresh HTTP request, body Each execution of that prepared request creates a fresh HTTP request, body
reader, and attempt timeout context. It closes the response body and returns reader, and attempt timeout context. It closes the response body and returns
@@ -55,6 +59,33 @@ This client does not schedule retries; the [generation coordinator](generation.m
does so around its prepared attempts. The client has no tool-call support, does so around its prepared attempts. The client has no tool-call support,
provider catalog, inbound HTTP behavior, or durable session store. provider catalog, inbound HTTP behavior, or durable session store.
## Diagnostic HTTP Evidence
Built-in diagnostics wrap the selected HTTP client's `RoundTripper` for one
invocation using a shallow client copy, preserving its timeout, redirect policy,
cookie jar, and underlying transport without mutating the configured client.
Evidence distinguishes a diagnostic dispatch invocation from an observed
request and response. A newly observed request replaces the previous
destination-specific response evidence, so redirects retain only the last
observed hop. A response is recorded before envelope decoding or later context
checks can discard a generation result.
The collector keeps only the last observed destination (without URL user
information), status, credential-presence, response-availability, decoded
generation, and presence-aware usage facts needed by the diagnostic. It does
not retain headers, credentials, raw request or response bodies, or a redirect
trace. These are observations at Promptkit's RoundTripper boundary, not proof
that bytes reached a peer or that a custom transport did no hidden dispatch.
Reachability and named authentication probes use the same configured built-in
client but discard their response bodies after local recognition. The
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
owns their observable HTTP semantics.
An injected `LLMClient` does not pass through this HTTP boundary. Its returned
response can establish response and inference facts, but not an HTTP
destination, status, reachability, or upstream credential-supply fact. The
facade does not silently install a built-in transport for those diagnostics.
## Prepared Generation ## Prepared Generation
For [`RunPrepared`](../../engine.go), the runner supplies the model client with For [`RunPrepared`](../../engine.go), the runner supplies the model client with
@@ -133,7 +164,10 @@ own configuration, client cloning, deterministic deadline precedence,
authentication, request and response mapping, malformed data, error identity, authentication, request and response mapping, malformed data, error identity,
cancellation, endpoint selection and composition, pre-transport rejection, and cancellation, endpoint selection and composition, pre-transport rejection, and
bounded single-document successful-response framing, closure, and bounded single-document successful-response framing, closure, and
response-body suppression. The focused response-body suppression. The [HTTP evidence tests](../../internal/llm/http_evidence_test.go)
own invocation-local last-hop request/response facts, while the
[prepared-generation evidence tests](../../internal/llm/prepared_generation_evidence_test.go)
own retention across decoding and cancellation. The focused
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go) [provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
own envelope parsing, normalization, and bounded-reader cases; their own envelope parsing, normalization, and bounded-reader cases; their
[transport tests](../../internal/llm/provider_http_error_transport_test.go) [transport tests](../../internal/llm/provider_http_error_transport_test.go)

View File

@@ -23,6 +23,9 @@ callback. The caller supplies normalized failure metadata; the recorder never
parses an error string or receives provider bodies, credentials, headers, or parses an error string or receives provider bodies, credentials, headers, or
other content. Events are values with no mutable containers. Callback panics other content. Events are values with no mutable containers. Callback panics
propagate to the caller. propagate to the caller.
Resolved prompt and profile IDs are the exact runtime identities selected for
the invocation, including namespace qualification when supplied by an engine
view.
Per-operation callbacks are ordered and synchronous; different operations may Per-operation callbacks are ordered and synchronous; different operations may
invoke the same callback concurrently. The recorder marks each measured phase's invoke the same callback concurrently. The recorder marks each measured phase's
end before delivery, so callback work does not inflate provider duration. end before delivery, so callback work does not inflate provider duration.
@@ -41,6 +44,11 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its one dispatch. A terminal event is an aggregate and must not be added to its
attempt events. attempt events.
Smoke diagnostics mask inherited execution-recorder state while invoking
generation or HTTP collaborators. Their private attempt evidence is assembled
into a `SmokeTestReport`; it does not create ordinary run events or enter an
enclosing run's observation totals.
The [recorder tests](../../internal/observation/recorder_test.go) own usage The [recorder tests](../../internal/observation/recorder_test.go) own usage
presence, phase sums, event ordering, empty-run accounting, identity snapshots, presence, phase sums, event ordering, empty-run accounting, identity snapshots,
and callback panic behavior. The [generation coordinator](generation.md) owns and callback panic behavior. The [generation coordinator](generation.md) owns

View File

@@ -11,27 +11,27 @@ contributor workflow and validation.
| Component | Implemented responsibility | References | | Component | Implemented responsibility | References |
| --- | --- | --- | | --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, profile-inspection, and ordinary-run observation values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) | | Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) | | `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) | | `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) | | `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) | | `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
| `internal/catalog` | Strictly validates imported immutable maintained backend and profile catalog assets before engine assembly uses them. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) | | `internal/configuration` | Loads root and namespaced collection resources through ConfigKit; records effective field presence, validates final values including exact references and optional diagnostic metadata, captures indexed prompt definitions and source-relative message content, compiled templates, input metadata snapshots, sorted backend/profile/prompt identities, resource namespace sets, effective collection declarations, backend and presence-aware profile snapshots, and the unchanged read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) | | `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) | | `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles external catalogs, Promptkit-owned profiles, repositories, registry, capacity, collaborators, repairer, and the one use-case runner for each engine instance. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) | | `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) | | `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) | | `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
| `internal/profile` | Owns immutable in-memory and filesystem profile repositories, local profile validation, raw-source overlays with error-preserving fallback, and inherited-profile resolution. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) | | `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | | `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) | | `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary and single-claim prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | | `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The internal engine assembles these internal components without exposing their The internal engine assembles these internal components without exposing their
representations. Consumers depend only on the root facade. representations. Consumers depend only on the root facade.

View File

@@ -4,7 +4,7 @@
This document describes Promptkit's implemented internal orchestration. The This document describes Promptkit's implemented internal orchestration. The
[architecture policy](../policy/architecture.md) owns dependency and consumer [architecture policy](../policy/architecture.md) owns dependency and consumer
boundaries. The [source and validation document](sources.md) owns repository, boundaries. The [source and validation document](sources.md) owns configuration,
artifact, rendering, and validation behavior, while the artifact, rendering, and validation behavior, while the
[model-client document](llm.md) owns generation behavior and failure [model-client document](llm.md) owns generation behavior and failure
categories. categories.
@@ -16,11 +16,10 @@ and override semantics consumed by the runner.
## Collaborators ## Collaborators
`Runner` coordinates narrow internal interfaces for prompt definitions, `Runner` coordinates narrow internal interfaces for captured prompt definitions
profiles, backend resolution, artifacts, rendering, model generation, and and resolved profiles, artifacts, rendering, model generation, and validation.
validation. The internal engine supplies one immutable registry containing the The internal engine supplies one engine-local run admitter and a generation
built-in backend and validated consumer additions, one engine-local run coordinator using the same capacity manager. Validation
admitter, and a generation coordinator using the same capacity manager. Validation
plans and provider-facing schema metadata come from the validator's preparation plans and provider-facing schema metadata come from the validator's preparation
interface. interface.
The internal engine supplies one default output repairer through the explicit The internal engine supplies one default output repairer through the explicit
@@ -33,32 +32,81 @@ The runner has no durable run or session store.
## Shared Prompt Selection ## Shared Prompt Selection
The runner uses one prompt-selection and hashing boundary for ordinary The internal engine applies each namespace view to slashless caller selections
preparation and exact prompt inspection. Preparation retains its early before they reach the runner. The runner therefore selects captured prompts and
request-ID check before direct-session normalization; both operations then use profiles by the resulting exact runtime IDs. Already-qualified selections pass
the configured prompt repository to select one definition, load referenced through, while configured defaults and stored references are used as authored
message content, and calculate the same prompt hash. and are not requalified.
Inspection stops after that structural lookup. It does not parse templates or Ordinary preparation selects the full prompt definition and construction-time
touch profile, artifact, schema, renderer, validator, admission, or model hash. It retains its early request-ID check before direct-session
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns normalization. Referenced message content and static template syntax were
the public operation's exact contract. captured when the engine was built.
`InspectPrompt` selects captured metadata through an optional repository
capability, falling back to the full definition and existing hash resolution
for internal repositories without it. It returns owned metadata without
rendering templates or touching profile, artifact, schema, renderer, validator,
admission, or model collaborators. The root
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
exact contract.
`InspectRequirements` uses the same metadata selection to resolve the
request-profile/default identity, then obtains the captured requirement
snapshot by the selected prompt ID. The analyzer receives only
those identity fields rather than an execution definition. It analyzes the
snapshot with the supplied caller variables and normalized direct-session
bypass; it does not resolve target readiness or enter the preparation
pipeline. Missing snapshot capability is a prompt-load failure rather than a
source fallback. The root
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
operation's exact contract.
## Shared Profile Selection ## Shared Profile Selection
The runner uses one profile-selection and target-resolution boundary for The runner uses one profile-selection and target-resolution boundary for
ordinary preparation and exact profile inspection. Preparation first selects a ordinary preparation and exact profile inspection. Preparation first selects a
request profile or a prompt default; inspection begins with its required request profile or a prompt default; inspection begins with its required
explicit profile ID. Both then apply the ordinary source precedence, resolve a explicit profile ID. Both use the captured, presence-aware target constructed
named backend, and construct the effective target from framework, backend, and from the final effective profile and backend configuration. Request overrides
profile values. apply afterward; preparation checks endpoint and model completeness only after
those overrides. Inspection can report a partial profile without requiring a
Inspection stops after the resulting endpoint and model are structurally runnable target. Neither path checks credential availability or performs prompt,
validated. It does not check credential availability or perform prompt,
artifact, schema, rendering, admission, or model-client work. The root artifact, schema, rendering, admission, or model-client work. The root
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's [`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
exact contract. exact contract.
## Smoke-Test Diagnostics
The separate `SmokeTester` in `internal/usecase` selects from the same captured
configuration snapshot and backend/profile collaborators as the runner. The
internal engine applies namespace-view qualification to slashless caller IDs
before diagnostic selection; captured defaults and references remain exact.
Explicit profile selection takes precedence over prompt-default selection and a
backend diagnostic default; HTTP-only selection does not resolve a default
inference profile. Exact public selection rules belong to the root
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go).
Diagnostics do not call `Run`, `Prepare`, or the prompt preparation pipeline.
They do not render application messages, load application artifacts, use
application schemas, or inspect session templates. Private text and structured
cases and the structured case's schema remain internal to the diagnostic
operation; they are not entries in prompt/profile listings or inspection.
`InspectConfiguration` remains a separate, explicit operator-oriented view of
effective settings and source history. A diagnostic report may include selected
effective settings and identifiers, so consumers still apply their disclosure
policy.
The engine assembles the smoke tester with the runner's generation coordinator
and capacity manager. Inference holds the ordinary logical admission lease and
uses the same active permits, retries, timeout, and request evidence path as
generation. Reachability and authentication probes bypass generation capacity
and do not require a model. An engine with an injected `LLMClient` can run
inference diagnostics but has no built-in HTTP probe collaborator; HTTP-only
diagnostics report that limitation instead of falling back to another client.
The public GoDoc owns the caller-visible timeout, cancellation, report, and
transport semantics.
## Shared Preparation Pipeline ## Shared Preparation Pipeline
`Prepare` and `Run` share one private preparation pipeline split at the point `Prepare` and `Run` share one private preparation pipeline split at the point
@@ -66,11 +114,10 @@ where a run can be assigned to its selected backend pool. The resolution phase
performs only the work needed to validate routing and admission: performs only the work needed to validate routing and admission:
1. validate the required prompt selection and normalize any direct session ID; 1. validate the required prompt selection and normalize any direct session ID;
2. load the prompt definition and hash the original definition; 2. select the captured prompt definition and hash;
3. select the request profile or the prompt's default profile; 3. select the request profile or the prompt's default profile;
4. resolve the profile's backend ID, when present; 4. retain the profile's selected backend identity, when present;
5. resolve application-neutral defaults, backend defaults, profile values, 5. apply explicit request overrides to its captured backend-completed target;
and explicit request overrides in that order;
6. validate endpoint, model, numeric overrides, and credential requirements; 6. validate endpoint, model, numeric overrides, and credential requirements;
7. resolve and validate the effective output contract without loading its 7. resolve and validate the effective output contract without loading its
schema; and schema; and
@@ -95,33 +142,42 @@ one execution-precedence and error-ordering implementation while allowing a
full backend pool to reject work before expensive schema, artifact, and full backend pool to reject work before expensive schema, artifact, and
rendering operations. `Prepare` discards the plan after returning its public rendering operations. `Prepare` discards the plan after returning its public
metadata. `Run` retains the plan through initial and repaired-output validation metadata. `Run` retains the plan through initial and repaired-output validation
and discards it when the operation ends. Prepared execution stores the same and discards it when the operation ends. Prepared execution transfers the
kind of plan only in its private payload. completed preparation's owned message, input-hash, and copied-schema values
into its private payload, while deep-copying mutable provider extras and
creating an independent details snapshot. Claim completion or discard clears
the execution payload without removing the details snapshot.
The root facade's `Engine.ValidateRequest` uses the same complete preparation
path as `Prepare` and discards the resulting internal prepared value without
converting it into a public snapshot. It does not add a runner operation or
validator component; request conversion and public error mapping remain at the
facade boundary.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. Endpoint overrides do not change out-of-range values fail as invalid requests. Endpoint overrides do not change
the selected backend identity. Non-empty extra-parameter maps replace whole the selected backend identity. Non-empty extra-parameter maps replace whole
lower-precedence maps. A direct API key takes precedence over environment lower-precedence maps. A direct API key takes precedence over environment
lookup; otherwise request, profile, and backend environment-variable names lookup; otherwise request, explicit profile, and backend environment-variable
apply in that order. A profile requiring a direct key clears an inherited names apply in that order. A profile requiring a direct key suppresses an inherited
backend environment name unless the request supplies its own name. Secret backend environment name unless the request supplies its own name. Secret
values remain excluded from serialized metadata. values remain excluded from serialized metadata.
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
nonblank string trims and replaces it, and a pointer to a blank string clears nonblank string trims and replaces it, and a pointer to a blank string clears
it. A nonblank direct session is normalized before source loading, bypasses it. A nonblank direct session is normalized before prompt selection, bypasses
the prompt session template, and is applied after ordinary message rendering. the prompt session template, and is applied after ordinary message rendering.
A blank direct value retains prompt-template behavior. The runner clears the A blank direct value retains prompt-template behavior. The runner clears the
template only on a value copy of the definition, so the definition hash always template only on a value copy of the definition, so the definition hash always
describes the original source while the rendered-prompt hash includes the describes captured effective content while the rendered-prompt hash includes the
effective direct or rendered session and the complete effective message effective direct or rendered session and the complete effective message
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256 sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
encoding of that session, every role and content value, and cache-control encoding of that session, every role and content value, and cache-control
presence and values; its hexadecimal value is opaque. presence and values; its hexadecimal value is opaque.
The registry is read-only after engine construction. Concurrent `Prepare` and The configuration snapshot is read-only after engine construction. Concurrent
`Run` calls resolve independent defensive backend values and keep all `Prepare` and `Run` calls copy their resolved targets and keep all invocation
invocation state local. state local.
## Run Flow ## Run Flow
@@ -215,7 +271,7 @@ generation, validation, repair, and cancellation failures.
Other context cancellation propagates through the invoked collaborator and is Other context cancellation propagates through the invoked collaborator and is
classified by the owning operation. In particular, cancellation observed by classified by the owning operation. In particular, cancellation observed by
validation retains the context identity through the validation error category. validation retains the context identity through the validation error category.
An overlong direct session is an invalid request before source loading, while An overlong direct session is an invalid request before prompt selection, while
an invalid or overlong prompt session template remains a prompt-render failure. an invalid or overlong prompt session template remains a prompt-render failure.
An unknown selected backend, or a selected backend with no configured resolver, An unknown selected backend, or a selected backend with no configured resolver,
is classified as a profile-load failure. is classified as a profile-load failure.
@@ -237,4 +293,8 @@ Changes to orchestration should continue to use the existing package
interfaces, keep request state local to an invocation, and preserve the shared interfaces, keep request state local to an invocation, and preserve the shared
resolution and completion pipeline. Source, renderer, validator, or resolution and completion pipeline. Source, renderer, validator, or
model-client contract changes belong first in their owning package and model-client contract changes belong first in their owning package and
document. document. The
[smoke selection tests](../../internal/usecase/smoke_selection_test.go),
[diagnostic orchestration tests](../../internal/usecase/smoke_tester_test.go),
and [private inference-case tests](../../internal/usecase/smoke_inference_test.go)
own diagnostic selection, report assembly, and inference findings.

View File

@@ -2,117 +2,163 @@
## Purpose ## Purpose
This document describes Promptkit's implemented internal source, artifact, This document describes Promptkit's implemented configuration, artifact,
rendering, and output-validation behavior. The rendering, and output-validation behavior. The
[architecture policy](../policy/architecture.md) owns the library boundary and [architecture policy](../policy/architecture.md) owns dependency rules, while
dependency rules. None of these internal packages is a supported consumer API, the [framework format reference](../formats.md) owns fields, defaults, and
and the internal engine assembles them behind the public facade's source options and values. source semantics. Internal packages are not consumer APIs.
The [framework format reference](../formats.md) owns the exact file fields,
validation modes, built-in catalog, and source precedence. ## Typed Configuration Loading
`internal/configuration` is the single adapter from ConfigKit's final
effective configuration to execution-owned definitions. It performs one
layered load, captures effective presence (including explicit null), builds
detached normalized interpretations during validation, and validates all effective
root and collection backend, profile, and prompt values. Replaced lower-layer
domain values are not separately validated. It indexes root resources under
their unqualified IDs and collection resources under their full
`namespace/local-key` IDs while retaining ConfigKit's effective tree in its
authored shape. It builds an immutable backend index and presence-aware profile
targets, applying final backend defaults only after profile inheritance. Runtime
lookups use those final indexes and exact stored references; they do not rewrite
or resolve resources from ConfigKit source history. The runtime/domain packages
do not depend on ConfigKit.
ConfigKit owns source headers, imports, composition, and source evidence.
Promptkit does not preparse contributions: it validates the final effective
values once, captures the indexed identities and their original resource
contexts, and retains ConfigKit's composed result for explicit operator inspection.
Accepted non-main source headers are consumed during composition; the main
source declaration remains available in that inspection view. Collection
release labels are descriptive metadata, not selection inputs. The
[format reference](../formats.md#source-headers-collections-and-resource-ids)
owns the exact source-header and collection contracts.
The `ConfigurationInspection.Diff` facade method compares two retained results
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
once to ConfigKit v1.0.0 and returns its public difference records. Comparison
uses only frozen effective values and does not reopen sources, resolve runtime
credentials, or enter the execution path. ConfigKit applies sensitivity from
both results; reports may still contain ordinary application data. See the
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and ConfigKit's [v1.0.0 comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md#effective-configuration-diffs).
After resource materialization and effective-profile snapshot construction, the
adapter captures lexically ordered identity inventories for backends, profiles,
and prompts, per-kind resource namespace sets, and metadata for effective
collection declarations. Listing methods return caller-owned copies from those
captured values without consulting ConfigKit or copying resource definitions.
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
files under an explicit root, provide resource context, and leave imports
disabled. Each returned slice belongs to one strict-composition group; file
order does not confer priority. Caller underlays are composed in their given
order, followed by the optional main input. The
[configuration tests](../../internal/configuration/load_test.go) exercise the
real loader and final-value handoff.
The public `CollectionInput` helper mounts one caller-selected source at
`collections/<namespace>` and enables ConfigKit imports for that input. The
helper does not read the source; ConfigKit retains responsibility for loading,
composition, imports, and source-local resource context. Imported or inherited
`content_file` values continue resolving relative to the source that contributed
the effective field, not to a flattened runtime identity. Explicit
`InspectConfiguration` views expose the retained effective configuration and
provenance for operator review; runtime selection and inventories do not derive
from source history.
## Prompt Definitions ## Prompt Definitions
`internal/promptdef` uses one source-neutral flow for prompt selection and The configuration adapter indexes each prompt by exact ID,
normalization. That flow scans normalized YAML ID and version metadata, materializes file-backed message content using the exact effective field's
requires one strictly decoded document per file, classifies errors for the origin and resource context, and checks static message/session template syntax
selected definition, detects duplicates, and normalizes the exact match. at construction. It stores captured content, compiled templates, declared
Small operating-system and `fs.FS` adapters own discovery, byte reads, display input metadata, and a definition hash. Alongside the unchanged hashed
paths, content opening, and root containment. Each lookup remains a definition, it retains content-type and description presence for each declared
point-in-time scan: definitions and catalogs are not cached, and file-backed input and provides an internal requirement snapshot with those declarations
message content is opened only for the exact selected candidate. and the captured parses. Metadata values come from the final normalized
declaration; absent and null fields remain distinguishable from explicit
strings, including empty strings. It closes content-file handles before
returning and retains no source capability for public inspection.
Message roles are normalized through the shared domain owner by trimming Prompt materialization shares a lazy metadata snapshot across root and collection
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and resources during construction, avoiding a full metadata copy per collection.
`assistant` are published; invalid roles remain selected prompt-definition
failures rather than becoming request errors. Cache-control metadata uses the
same shared domain normalization and defensive-copy rule.
Operating-system sources enforce containment against canonical roots and Normalization and defaults are applied to execution-owned interpretations,
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment including message roles and cache controls. ConfigKit retains the composed
in their clean relative path namespace. A single-file source uses the selected values and origins, rather than synthetic Promptkit normalization events. List
prompt file's containing directory as its root. Every content path must be indexes identify fields for presence, validation, sensitivity, and resource
relative and is opened from its exact parsed text after a separate blank check; context; full IDs select captured runtime indexes. Recording a sequence-wide
contained parent components and whitespace-bearing names remain valid. normalization would replace origins of unchanged content-file fields.
Exact prompt inspection performs one point-in-time lookup through that same The runner selects full definitions from this index for preparation and uses
repository and validates referenced message content before returning declared captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
metadata. It does not parse templates or read profile, input, or schema internal requirement snapshot lookup uses the selected prompt ID alone.
sources, and it does not retain the definition for a later execution. These paths do not rescan configuration or reopen message files. `InspectPrompt`
returns declared metadata without rendering or resolving profile, input, or
schema resources, and its public shape and prompt hashes do not include the
captured presence state. Runtime rendering still checks request variables and
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
cover resource origins, context containment, capture, and selection.
Its package tests own prompt selection, strict decoding, definition validation, The configuration adapter shares ID-only selection across its internal prompt
duplicate detection, and source containment: lookups. The execution snapshot lookup returns an owned prompt
[prompt-definition repository tests](../../internal/promptdef/repository_test.go). definition and cached hash; metadata inspection returns owned input metadata
without copying execution templates. Profile selection likewise has an
identity-only lookup for consumers that do not need the execution target,
while the full profile lookup continues to return an owned target and deep copy
of provider extras. These lookups use captured indexes and do not query
ConfigKit again.
## Profiles And Built-Ins The `internal/prompt` requirement walker uses the captured parse trees and
declaration metadata without executing templates or reading artifacts. It
collects literal `input` names, direct root caller keys, and literal-key
`index` lookups. For `if` and `with`, it classifies branches when their
conditions are simple supported values: direct values, literals, negation, or
two-string equality/inequality. It uses Go template truthiness, so a supplied
nonempty string such as `false` is true. Missing decision values and
content-dependent conditions leave dependency-bearing branches conditional
and report incomplete discovery; unresolved conditions with no branch
dependencies do not by themselves make discovery incomplete. Recognizable
references on known skipped paths remain optional inventory entries with an
inactive-use explanation, unless another applicable use establishes a
stronger classification. Skipped expressions are scanned only for
syntactically recognizable names and are not evaluated. A direct caller value
or literal-key lookup can also supply an input name, including through simple
parentheses or a direct pipeline to `input`; names are used exactly as
supplied. Unresolved names and unsupported dependency-bearing expressions
produce structured limitations while retaining recognizable references.
Where an unresolved name or scope could affect existing optional entries, those
entries are reported as conditional rather than inactive-only or definitively
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
values remain separate from unknown scopes; accesses through untracked local
maps report incomplete discovery rather than silently losing caller keys.
Unsupported name expressions reuse value analysis's conservative syntax scan
instead of recursively rescanning the same subtree.
Invoked named templates are scanned conservatively, and
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
and bypassed session templates are not traversed. This internal walker does not
change rendering behavior.
`internal/profile` owns immutable in-memory and filesystem repositories, local ## Profiles
validation, overlays, and resolution for execution profiles. In-memory
definitions are normalized and validated once at that repository boundary and
are defensively copied on storage and lookup. A file contains
exactly one YAML document and its trimmed YAML `id` is its only selection
identity; filenames do not confer authority. Each point lookup reads discovered
files once for their metadata and reuses the selected file's bytes for strict
decoding; unrelated profiles are not fully decoded. Strict selected decoding
recognizes `base_profile` and the optional `backend` field, trims their values,
and permits inherited target fields only when a base is named. File-backed
`extra_params` values are validated and defensively copied through the shared
bounded JSON-value owner before a profile is published. OpenAI-compatible
reserved-field policy remains with the model-client and backend-registry owners.
`LoadFSRepository` is the eager immutable loading boundary for internal ConfigKit resolves profile inheritance before the handoff. Within a collection,
catalog consumers. It discovers and strictly validates every raw profile once, `extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
preserves safe source metadata including explicitly present YAML fields, and The configuration adapter validates final effective resource references and
publishes independently copied values from memory. It does not resolve profile field values, then builds target
inheritance. Configured consumer sources continue to use the lazy point lookup snapshots with backend defaults and presence/suppression state. A profile can
repositories described above. remain partial until runtime overrides complete it. `InspectProfile` selects
that captured target without prompt, artifact, schema, credential lookup, or
provider work. Preparation applies runtime overrides and then checks target
completeness. Identity-only profile selection can identify a partial captured
profile without copying its target or requiring readiness. The
[target tests](../../internal/configuration/targets_test.go) cover effective
target interpretation.
`internal/catalog` validates the imported immutable OpenRouter and The internal engine delegates listing through a narrow identity-inventory
Rakestrawhome asset modules as one private adapter boundary. It enforces their interface directly to the configuration snapshot. The execution runner has no
manifest, layout, profile ownership, inheritance, and secret-safety rules listing responsibility.
before returning raw catalog sources. Internal engine assembly uses those validated
catalogs as the lowest-precedence profile source.
Normalized external catalog profile IDs with the exact `promptkit-` prefix are
rejected at this adapter boundary. That reservation applies only to imported
catalogs; configured consumer sources retain the ordinary profile-ID rules.
The overlay repository consults the next repository only when the
higher-precedence repository reports that a profile is absent. A reliably
selected malformed profile stops fallback, while an unrelated malformed file
does not become authoritative through its filename. Loading does not check
backend registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation and exact
profile inspection.
The internal engine assembles one raw composite catalog in precedence order:
in-memory profiles, one ordinary configured source, an application fallback
source, Promptkit-owned profiles, then the maintained external catalog. It
constructs the owned definitions through the in-memory repository and validates
their maintained-only resolution and backend membership before consumer sources
are overlaid. An explicit file or `fs.FS` profile source replaces
`Config.ProfileDir` within the ordinary configured-source category. One outer
resolving repository wraps that complete raw catalog, so each base lookup
observes the same precedence and shadowing rules.
The resolving repository traverses every selected chain afresh, retains no
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
new caller-owned value, and validates the final target before publishing it. It
does not check backend registry membership. Exact `base_profile` syntax, merge
rules, and consumer-visible failure behavior belong to the [framework format
reference](../formats.md#profile-inheritance).
Exact profile inspection performs one point-in-time resolved lookup through
those profile sources and checks the final target without reading prompt, input,
or schema sources. It does not retain that lookup for a later execution.
Prepared execution instead freezes the fully resolved target; a later ordinary
operation performs a fresh traversal.
The imported external catalog provides external profiles and matching backend
definitions. The internal engine owns the Promptkit profiles documented in the
[framework format reference](../formats.md#built-in-profile-catalog). Profile
loading and overlay behavior are owned by the [profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, backend selection, and duplicate IDs are owned by the
[catalog adapter tests](../../internal/catalog/catalog_test.go).
## Ordinary Artifacts ## Ordinary Artifacts
@@ -123,10 +169,15 @@ and after opening the file. It copies content into an artifact, records
metadata and an opaque content-equality value, and applies a content-type metadata and an opaque content-equality value, and applies a content-type
fallback. fallback.
Regular files are read synchronously in bounded chunks. Cancellation is Regular files are read synchronously in bounded chunks. The opened file's size
checked before opening, before and after every read, and before publishing the is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
artifact, so a canceled read never publishes partial content. The ordinary the returned body determines artifact size and hash. If an overreported hint
reader does not detach file reads into background goroutines. leaves unused capacity greater than the actual body length, the completed body
is copied to release the excess backing storage, including when the body is
empty. Cancellation is checked before opening, before and after every read,
and before publishing the artifact, so a
canceled read never publishes partial content. The ordinary reader does not
detach file reads into background goroutines.
This ordinary reader does not implement an inbound HTTP security boundary. In This ordinary reader does not implement an inbound HTTP security boundary. In
particular, it does not constrain files to an application root or impose an particular, it does not constrain files to an application root or impose an
@@ -137,19 +188,22 @@ implemented reader behavior and failures.
## Rendering ## Rendering
`internal/prompt` renders definition messages as Go templates using named `internal/prompt` renders captured, pre-parsed definition templates using named
artifacts and variables. Within one render, each referenced artifact body is artifacts and variables. It clones templates to bind request-specific input
converted to text lazily and cached by input name for reuse across the session helpers without mutating shared state. Within one render, each referenced
and every message; the cache is not shared across renders. Conversion uses artifact body is converted to text lazily and cached by input name for reuse
bounded chunks and preserves the artifact bytes exactly. across the session and every message; the cache is not shared across renders.
Conversion uses bounded chunks and preserves the artifact bytes exactly.
Session and message parsing and execution remain synchronous. The renderer Template parsing occurs during engine construction. Runtime execution remains
checks cancellation before and after each parse and execution boundary, synchronous. The renderer checks cancellation around execution boundaries,
between artifact conversion chunks, around each message, and before publishing between artifact conversion chunks, around each message, and before publishing
the complete prompt. It cannot interrupt template work already in progress and the complete prompt. It cannot interrupt template work already in progress and
never publishes a partial prompt after observing cancellation. It validates and never publishes a partial prompt after observing cancellation. It validates and
canonicalizes message roles before carrying roles, session IDs, and cache canonicalizes message roles before carrying roles, session IDs, and cache
control into the rendered prompt. The control into the rendered prompt. Message and session templates execute into
per-call string builders, so their rendered strings do not require a final
buffer-to-string copy. The
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior. [renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
## Schemas And Output Validation ## Schemas And Output Validation
@@ -157,7 +211,8 @@ control into the rendered prompt. The
`internal/validate` provides validators backed by an operating-system `internal/validate` provides validators backed by an operating-system
filesystem or an `fs.FS`. Invalid generated content is returned as a validation filesystem or an `fs.FS`. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational result; inability to load, register, or compile a schema is an operational
error. error. Basic validation checks body emptiness on the byte slice directly
instead of first making a response-sized string copy.
Every preparation operation creates one operation-local validation plan. None, Every preparation operation creates one operation-local validation plan. None,
basic, and JSON modes retain the effective output contract without source basic, and JSON modes retain the effective output contract without source
@@ -181,9 +236,10 @@ not detach dependency work into background goroutines.
`Prepare` discards its validation plan after returning metadata. `Run` retains `Prepare` discards its validation plan after returning metadata. `Run` retains
its plan for initial and repaired-output validation, then discards it with the its plan for initial and repaired-output validation, then discards it with the
operation. `PrepareExecution` retains the plan in its private frozen payload; operation. `PrepareExecution` retains the plan in its private frozen payload;
`RunPrepared` uses that plan without reopening prompt, profile, input, or `RunPrepared` uses that plan without reopening configuration, message, input,
schema sources or rerendering the request. A later ordinary `Run` always or schema resources or rerendering the request. A later ordinary `Run` uses
performs fresh source resolution and preparation. the same captured configuration but prepares request artifacts and schemas
afresh.
The [validator tests](../../internal/validate/standard_validator_test.go) own The [validator tests](../../internal/validate/standard_validator_test.go) own
basic, JSON, JSON Schema, source resolution, schema loading, compilation, basic, JSON, JSON Schema, source resolution, schema loading, compilation,

View File

@@ -6,45 +6,37 @@ Material: policy documents under `docs/policy/` and the Policy Reading Guide in
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied `docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
without modification from upstream `agents/AGENTS.md`. without modification from upstream `agents/AGENTS.md`.
The release policy uses upstream commit
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
`d84a38ebab26f048cafbe59a2246700c35d29843`.
Copyright (C) 2026 Eric Rakestraw Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: CC-BY-SA-4.0 SPDX-License-Identifier: CC-BY-SA-4.0
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) | License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) | [COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE). [LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843). [Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
## Promptkit OpenRouter Catalog [Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2` ## ConfigKit
at `v2.20260926.0`.
Selected module: `go.maximumdirect.net/configkit` at `v1.0.0`.
Copyright (C) 2026 Eric Rakestraw Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [MIT](licenses/MIT.txt) | License and notices: [LGPL 3.0](../../LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) | [COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE). [LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3). [Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53).
## Promptkit Rakestrawhome Catalog
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2`
at `v2.20260926.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
## jsonschema ## jsonschema
@@ -73,7 +65,7 @@ License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
## Go YAML v3 ## Go YAML v3
Selected module: `gopkg.in/yaml.v3` at `v3.0.1`. Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency).
Copyright (c) 2006-2010 Kirill Simonov Copyright (c) 2006-2010 Kirill Simonov
@@ -91,3 +83,22 @@ License and notices: [MIT](licenses/MIT.txt) |
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE). [NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
[Source](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51). [Source](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
## Go YAML v4
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0-rc.6` (transitive dependency).
Copyright 2006-2010 Kirill Simonov
Copyright 2011-2019 Canonical Ltd
Copyright 2025 The go-yaml Project Contributors
SPDX-License-Identifier: MIT AND Apache-2.0
License and notices: [MIT](licenses/MIT.txt) |
[Apache 2.0](licenses/Apache-2.0.txt) |
[LICENSE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/LICENSE) |
[NOTICE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/NOTICE).
[Source](https://github.com/yaml/go-yaml/tree/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125).

View File

@@ -51,23 +51,22 @@ The [internal component overview](../internal/overview.md) owns the inventory
of implemented packages and their concrete responsibilities. This policy owns of implemented packages and their concrete responsibilities. This policy owns
the boundaries and dependency rules those components must preserve. the boundaries and dependency rules those components must preserve.
The internal engine assembles one immutable backend registry, one capacity manager, The internal engine composes caller-selected ConfigKit inputs once, then
the internal repositories, renderer, validator, outbound client, and use-case assembles the captured definitions, capacity manager, renderer, validator,
runner. The registry contains validated maintained definitions plus engine-scoped outbound client, and use-case runner. The configuration adapter interprets and
consumer additions. The root facade translates public values and errors at the validates only the final effective values, including presence, null, and
library boundary. resource context. It retains source history for explicit operator inspection,
The internal engine constructs the capacity manager from the registry's immutable not for validation or a second resolution pass. The root facade translates
policy snapshot, wraps the selected built-in or injected model client, and public values and errors at the library boundary.
supplies bounded admission to the runner. The defaults and renderer depend on
the domain model. Prompt-definition and profile repositories use the domain The engine constructs capacity policy from final backend definitions, wraps
model, file catalog, and YAML decoder. The catalog adapter validates imported the selected built-in or injected model client, and supplies bounded admission
immutable maintained data, including its reserved namespace, before internal to the runner. Runtime/domain components consume execution-owned values, not
engine assembly supplies it to the profile package. Artifact reading uses the ConfigKit types. Artifact reading and validation use the domain model and
domain model and application-neutral defaults. Validation uses the domain model, application-neutral defaults; validation also uses file-catalog helpers and
file catalog, and JSON Schema implementation. The model client uses the domain JSON Schema. The model client uses the domain model, defaults, and an injected
model and application-neutral defaults, and an injected or standard-library HTTP or standard-library HTTP client. The runner coordinates narrow lookup and
client. The use-case runner depends on the narrow interfaces owned by each collaborator interfaces without rereading configuration.
internal component, including backend lookup and run admission.
The current implementation follows this dependency direction: The current implementation follows this dependency direction:
@@ -84,9 +83,10 @@ downstream consumers, including Scriptorium
narrow injected abstractions narrow injected abstractions
``` ```
The backend registry depends on the domain model and shared JSON-value The backend index depends on the domain model and shared JSON-value ownership
validation, has no mutation API after construction, and consumes the and has no mutation API after construction. The configuration adapter applies
OpenAI-compatible reserved request-field rule owned by the model client. The the OpenAI-compatible reserved request-field rule owned by the model client
when it validates final provider maps. The
capacity component depends on the domain model and the narrow internal capacity component depends on the domain model and the narrow internal
model-client boundary, not on provider transport implementation. The model model-client boundary, not on provider transport implementation. The model
client does not depend on registry or capacity configuration. The facade client does not depend on registry or capacity configuration. The facade
@@ -97,7 +97,8 @@ on consumers or on Scriptorium.
`internal/domain` owns source-neutral invariants for values shared across `internal/domain` owns source-neutral invariants for values shared across
multiple input and execution boundaries, including execution-setting bounds, multiple input and execution boundaries, including execution-setting bounds,
OpenAI-compatible base endpoints, session identifiers, and output-contract OpenAI-compatible base endpoints, session identifiers, and output-contract
legality. Callers retain source parsing, required-field rules, other legality, plus shared resource-identity syntax and qualification. Callers retain
source parsing, required-field rules, other
source-specific normalization, defaulting, error classification, and policy source-specific normalization, defaulting, error classification, and policy
specific to their own boundary. specific to their own boundary.

View File

@@ -88,7 +88,7 @@ secret values.
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. | | Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. | | Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. | | Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. | | Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
Create supporting documents for existing interfaces and responsibilities when Create supporting documents for existing interfaces and responsibilities when
substantive content is available. substantive content is available.

View File

@@ -2,7 +2,7 @@
**Base Policy Version:** 1.8 **Base Policy Version:** 1.8
**Last Updated**: 2026-09-27 **Last Updated**: 2026-11-02
**Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md) **Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
@@ -221,7 +221,8 @@ Promptkit retains its LGPL-3.0-or-later library grant in
[COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate [COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate
scope identified there and in [attribution](../oss/attribution.md). scope identified there and in [attribution](../oss/attribution.md).
Promptkit publishes source-only Go module tags on Gitea, with no binaries or Promptkit publishes source-only Go module tags and Gitea release pages, with no
separate hosted download page. The [release procedure](../release.md) owns binaries. Each new hosted page reuses the complete checked-in release note,
publication and concrete footer URLs. The hosted-page guidance applies if such including its licensing footer. The [release procedure](../release.md) owns
a channel is deliberately introduced; it does not require creating one. source publication, hosted delivery, and concrete footer URLs. Previously
published notes and tags retain their original records.

View File

@@ -1,10 +1,10 @@
# Release Policy # Release Policy
**Base Policy Version:** 1.4 **Base Policy Version:** 1.6
**Last Updated**: 2026-09-27 **Last Updated**: 2026-11-02
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md) **Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
## Purpose ## Purpose
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
intended source revision under a fresh version. Releases should require little intended source revision under a fresh version. Releases should require little
routine intervention, avoid repeated work, and preserve published contents. routine intervention, avoid repeated work, and preserve published contents.
Adopt stronger verification or reproducibility requirements through an explicit
project-specific decision justified by the intended audience and support needs.
The repository-specific commands belong in `docs/release.md`. Historical release The repository-specific commands belong in `docs/release.md`. Historical release
summaries belong in `docs/releases/`. The procedure must agree with the adopted summaries belong in `docs/releases/`. The procedure must agree with the adopted
policy and link to it rather than introduce additional release requirements. policy and link to it rather than introduce additional release requirements.
@@ -59,7 +62,8 @@ requires a deliberate update to the policy and procedure where needed.
For releases at or after `v1.0.0`: For releases at or after `v1.0.0`:
- a major release may make intentional incompatible changes; - a major release may make intentional incompatible changes;
- a minor release may add backward-compatible behavior; and - a minor release may add backward-compatible behavior or raise the Go consumer
minimum as described below; and
- a patch release may make backward-compatible fixes. - a patch release may make backward-compatible fixes.
Before `v1.0.0`, a minor release may change a documented public, operational, Before `v1.0.0`, a minor release may change a documented public, operational,
@@ -71,14 +75,44 @@ Additional compatibility promises and deliberate exceptions belong in the
adopted policy or canonical contract documentation. Release notes explain adopted policy or canonical contract documentation. Release notes explain
changes; they do not redefine current compatibility contracts. changes; they do not redefine current compatibility contracts.
## Release Documentation ## Consumer Minimum And Release Toolchain
Distinguish the minimum toolchain supported for consumers from the toolchain
maintainers use to build and validate a release. Record the consumer minimum in
the language's standard metadata and explain its support meaning in canonical
documentation. Define the selected release toolchain and its setup commands in
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
choices.
For Go repositories, normally declare the initial stable release of the selected
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
when a dependency or an identified correctness fix requires it, and document
the reason.
For release-policy purposes, successful build and release validation on the
selected Go patch release are sufficient to presume support across that Go
series, subject to the declared consumer minimum and concrete evidence of
incompatibility. Investigate specific compatibility concerns as they arise.
Use the selected release toolchain for candidate validation and release builds,
including CI jobs performing those tasks. A newer validation patch version alone
leaves the consumer minimum unchanged.
A minor release may raise the Go consumer minimum, including from one Go series
to the next, both before and after `v1.0.0`. This is an explicit exception to the
backward-compatibility requirement for minor releases. Patch releases preserve
the consumer minimum. Explain an increase's impact and required consumer action
in the release note. Other incompatible changes remain subject to the versioning
rules above.
## Release Documentation And Preparation
Every new release must include `docs/releases/<version>.md` in the tagged Every new release must include `docs/releases/<version>.md` in the tagged
source tree. Use the complete version tag as the filename and begin with a source tree. Use the complete version tag as the filename and begin with a
title containing the product name and that exact version. title containing the product name and that exact version.
Include a short summary and any relevant compatibility impact or upgrade Include a short summary and any relevant compatibility impact or upgrade
instructions. Use the Promptkit section structure below and keep the note instructions. Choose sections suited to the release and keep the note
proportionate to the change. proportionate to the change.
Keep exact current contracts in their canonical documents and update affected Keep exact current contracts in their canonical documents and update affected
@@ -186,7 +220,8 @@ release without replacing its contents or reusing its version.
Maintain `docs/release.md` as a short, executable maintainer runbook. It must Maintain `docs/release.md` as a short, executable maintainer runbook. It must
identify: identify:
1. prerequisites, primary branch, and authoritative remote; 1. prerequisites, selected release toolchain, primary branch, and authoritative
remote;
2. version and release-note preparation; 2. version and release-note preparation;
3. one publication command with its selected local checks and automated guards; 3. one publication command with its selected local checks and automated guards;
4. declared deliverables, required build targets, and automation responsibilities; 4. declared deliverables, required build targets, and automation responsibilities;
@@ -206,47 +241,52 @@ navigation on 2026-09-26. Apply them prospectively; previously published notes
and tags retain their original records. and tags retain their original records.
Promptkit publishes source-only Go modules from main to origin on Gitea, using Promptkit publishes source-only Go modules from main to origin on Gitea, using
annotated stable SemVer tags. It has no binaries, hosted download pages, or annotated stable SemVer tags. The source commit is pushed through main before
asynchronous deliverables. The source commit is pushed through main before the the tag. After source publication, maintainers use ReleaseKit locally to deliver
tag. The [release procedure](../release.md) owns commands and recovery. a Gitea release page containing the complete checked-in note, including its
licensing footer. There are no binaries or other deliverables. Source publication
and hosted delivery have separate outcomes; delivery credentials and service
availability do not gate source-release completion. The
[release procedure](../release.md) owns commands and recovery.
The existing publication script requires Summary, Compatibility, Upgrade, and Release preparation uses the latest Go installation available in the development
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State environment to build the maintainer release executable and validate the
“No action required” or “None” where appropriate. This retained local structure candidate, including the consumer examples. Keep that same local toolchain
matches the automated release-note guard; changing it is separate work. active throughout those operations.
The [publication script](../../scripts/release.sh) runs the For a minor release, advance Promptkit's consumer minimum to the initial stable
[local checks](../../scripts/release-check.sh) once for the final committed release of that Go series, retaining a higher patch minimum when required by a
candidate: race-enabled tests, vet, three offline consumer examples, tracked dependency or an identified correctness fix. Patch releases preserve the
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a existing consumer minimum. Commit any module update before validating the final
credential-pattern scan. These protect concurrency, assembled library use, candidate. The [release procedure](../release.md#prepare-the-candidate) owns
source distribution, and documentation integrity. A separate ordinary test run setup commands, and the consumer guide explains
or standalone build is not an additional release gate. Candidate, tag, and [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
synchronized-main checks remain automated. No CI or module-availability wait
is required. Licensing footer URLs are preparation work, not a network gate.
## Catalog Profile Compatibility Use a title containing `Promptkit` and the exact version, followed by a short
summary and any relevant compatibility impact or upgrade instructions. The
note is committed with the candidate and supplies the annotated tag message.
No fixed set of section headings is required.
External catalog profile data may be added, removed, renamed, or modified The [publication command](../../scripts/release.sh) delegates to the released
without deprecation or advance notice. This exception is limited to profile ReleaseKit tool selected in `go.mod`. The committed
data imported from the maintained catalog modules. It does not weaken ordinary [profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
compatibility for Promptkit's public Go API, configuration and format semantics, committed Go formatting, local Markdown links, workspace and filesystem
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or replacement inspection. The Go suite includes the offline semantic checks for
consumer-owned definitions. the three maintained consumer examples. It also rejects the configured local
delivery token path if committed; this narrowly scoped check is not a general credential
scan. These are the selected checks for the final candidate; a separate
ordinary test run or standalone build is not an
additional release gate. The upstream tool owns candidate, remote, and tag
publication checks. It returns native status 0 for confirmed publication, 1
for a known validation or publication failure, 2 for invalid input or profile,
and 3 when the push outcome is unknown. For status 3, resolve the exact
captured publication endpoint before retrying; the repository-specific
[procedure](../release.md#failure-recovery) links to the upstream recovery
instructions. Do not wait for CI or Go module availability. Licensing footer
URLs are preparation work, not a network gate.
Within the catalog versions selected and tested by a Promptkit release, and Hosted delivery is a separate post-publication step for new releases. It reuses
unless a consumer overrides the name or its base, Promptkit keeps the literal the immutable tag and note without rerunning candidate validation or waiting
profile IDs `promptkit-openrouter-standard` and for CI. Retry failed delivery against that same version after resolving any
`promptkit-openrouter-auto` available. The names do not promise a permanent uncertain hosted write; preserve published tags and hosted content. Report
model, backend, routing behavior, settings, cost, output, or upstream delivery status separately from the completed source release.
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
owns their current definitions.
`v0.10.0` is the initial minor release for this compatibility transition.
Afterward, catalog-only updates and adjustments to the current selections may
ship in a patch release when all other compatibility obligations remain intact.
External profile removals can require consumer action; they are not automatic
migrations.
Catalog adoption and release validation remain local and offline. They do not
probe provider endpoints, request model lists, or make inference calls.

View File

@@ -58,17 +58,24 @@ and repository hygiene.
Introducing hosted CI later would supplement, not silently redefine, this Introducing hosted CI later would supplement, not silently redefine, this
documented validation model. documented validation model.
Full release-candidate validation includes race-enabled package tests, explicit The selected ReleaseKit profile runs the race-enabled package suite and full
static analysis, and execution with automated semantic checks of the three maintained vet, checks committed Go formatting, local Markdown links, and module
offline consumer examples. The test run builds the packages; the examples distribution hygiene. The Go suite includes semantic assertions for the three
also exercise ordinary compilation. Separate ordinary tests and a standalone maintained offline consumer examples. ReleaseKit owns this shared validation
build are not additional release gates. Ordinary tests remain useful during machinery and publication workflow. The test run builds the packages and
development. The preparation example protects assembled preparation and exercises the examples. Separate ordinary tests and a
standalone build are not additional release gates. Ordinary tests remain useful
during development. The preparation example protects assembled preparation and
inspection behavior. The execution example separately protects assembled inspection behavior. The execution example separately protects assembled
`Run`, injected-client, validation, usage, and result behavior. The observation `Run`, injected-client, validation, usage, and result behavior. The observation
example protects assembled built-in retries, structured proposal and assessment example protects assembled built-in retries, structured proposal and assessment
runs, and application-owned interpretation of metadata. runs, and application-owned interpretation of metadata.
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
ReleaseKit wiring against disposable local repositories. It is focused
development validation when release tooling changes, not part of the default
Go test suite or a recurring release check.
Tests in the default suite must be deterministic, offline, and independent of Tests in the default suite must be deterministic, offline, and independent of
real credentials. They must not invoke paid APIs, use live network real credentials. They must not invoke paid APIs, use live network
dependencies, or depend on mutable external services. Tests that require live dependencies, or depend on mutable external services. Tests that require live

View File

@@ -1,97 +1,147 @@
# Release Procedure # Release Procedure
This runbook implements the [release policy](policy/release.md). A successful This runbook implements the [release policy](policy/release.md). Promptkit
push of the selected annotated tag to origin completes the release. Never wait publishes source-only Go module tags from `main` to `origin`, followed by a
for CI to finish or succeed. page-only Gitea release. Successful publication of the intended tag, with its
source commit available on `main`, completes the source release. Report hosted
delivery separately. Do not wait for CI or Go module availability.
## Prepare The Candidate ## Prepare The Candidate
Required tools are Git, Go with race-detector support, POSIX shell, and Python Required tools are Git, Go with race-detector support, and POSIX shell. Work
3. Publication requires push access to origin. Work from the repository root. from the repository root with push access to `origin`.
Select an unused version and create `docs/releases/<version>.md` using the Use the latest Go installation available locally for the release tool and its
[required structure](policy/release.md#release-documentation). Describe child commands. Put it first on `PATH` and keep it selected throughout:
compatibility impact and consumer actions and update affected canonical docs.
For catalog updates, record selected catalog versions and any consumer action,
linking to the catalog policy rather than repeating its contracts.
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution ```sh
changes. Each new note must end with the following navigation, replacing export GOTOOLCHAIN=local
`vMAJOR.MINOR.PATCH` with the selected release tag: go version
```markdown
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
``` ```
Check these file paths against the candidate; a future tag need not resolve For a minor release, normally advance the consumer minimum to the initial
remotely during preparation. The annotated tag derives from this same note, stable patch of that Go series:
including its footer. Promptkit publishes no separate hosted release body or
download page. If such a channel is deliberately introduced, reuse the note
and preserve its footer. No extra network checks, artifacts, or approval gates
are introduced, and published notes are not retrofitted.
Retire completed roadmaps, preserving planned or deferred work and repairing ```sh
links. This is housekeeping, not a separate approval gate. Review changes release_go_version=$(go env GOVERSION)
through the ordinary development workflow, commit them, and push main to release_go_series=${release_go_version#go}
origin. No repeated release-specific diff review is required. release_go_series=${release_go_series%.*}
go mod edit "-go=${release_go_series}.0"
```
Retain a higher patch minimum when required by a dependency or identified fix.
For a patch release, preserve the existing minimum. Explain minimum increases
under the [versioning policy](policy/release.md#versioning-and-compatibility).
Prepare dependency caches before committing the candidate:
```sh
go mod download
```
Choose an unused stable version and create `docs/releases/<version>.md` with a
title containing `Promptkit` and that exact version, a short summary, and any
compatibility impact or upgrade instructions. End the note with:
```markdown
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
```
Substitute the selected version and check the paths against the candidate;
future tag URLs need not resolve during preparation. Follow the
[OSS policy](policy/oss.md). Historical notes and tags remain unchanged.
Supply delivery credentials through `RELEASEKIT_GITEA_TOKEN` or the ignored
local file `.releasekit/gitea-token`, with owner-only file permissions. A
nonblank environment value takes precedence. Source publication neither reads
nor requires this token. The [profile](../releasekit.json) selects the token
path for committed-file protection; keep credentials out of commits and logs.
Retire completed roadmaps, preserve unfinished work, and repair affected links.
Review and commit the candidate on `main`, including any module updates, then
push `main` to `origin` before publishing its tag.
## Validate And Publish ## Validate And Publish
Run the following with the chosen unused version (replace the placeholder):
```sh ```sh
./scripts/release.sh vMAJOR.MINOR.PATCH ./scripts/release.sh vMAJOR.MINOR.PATCH
``` ```
The [publication script](../scripts/release.sh) checks a clean, synchronized The launcher uses `go tool` to build or reuse the module-selected ReleaseKit
main candidate, the version and unused tag, and the release note's title and executable in Go's cache, using the selected local toolchain. ReleaseKit runs
required headings. It runs the [local checks](../scripts/release-check.sh) the [profile's checks](../releasekit.json) once, verifies publication state, and
once, rechecks candidate identity and publication state, generates the publishes only the intended tag. Preserve its native exit status and diagnostic.
annotation from the committed note plus the validated commit, and pushes only Successful publication needs no separate remote read-back or CI wait.
the selected annotated tag.
There is no separate tag-message authoring, example-output inspection, For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`.
metadata inspection, or human sign-off. Compatibility and release-note content It is not a required step before publication. The upstream
remain ordinary editorial responsibilities. Development validation is useful [candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/check.md)
but does not require a second full run before invoking the publication script. and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/cli.md)
own shared checks and exit-status definitions.
The script stops successfully after the tag push. CI completion, CI success, ## Deliver The Hosted Release
remote read-back, and Go module availability are not release completion gates.
Do not wait or poll for them. Provider probes and inference calls are never
part of release validation.
## Catalog Profile Compatibility After source publication, keep a clean checkout at the published tag's commit:
The [release policy](policy/release.md#catalog-profile-compatibility) owns the ```sh
catalog compatibility boundary. This heading preserves older incoming links. ./scripts/releasekit.sh deliver vMAJOR.MINOR.PATCH
```
The profile selects page-only Gitea delivery for this repository. The hosted
body is the complete checked-in note, including its licensing footer. There are
no binary assets, checksum manifests, candidate revalidation, or CI waits.
ReleaseKit checks delivery credentials during delivery, not as a publication
gate. Report the hosted URL on success; delivery failure leaves the source
release complete.
For a later retry, use a clean checkout of the original tag with its recorded
module files and supply the token there. In a separate checkout, for example:
```sh
git switch --detach vMAJOR.MINOR.PATCH
./scripts/releasekit.sh deliver vMAJOR.MINOR.PATCH
```
Resolve uncertain hosted writes before retrying the same immutable version.
Matching content is accepted; conflicts are not overwritten. Do not rerun
`publish` or move tags to repair delivery. Follow the upstream
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/delivery.md)
for result interpretation and recovery. These commands apply to tags containing
this delivery configuration; older releases retain their original workflows.
## Failure Recovery ## Failure Recovery
If local checks fail, fix the candidate, commit and push changes, then rerun For source-publication failures, use ReleaseKit's status and diagnostic to
the publication command. It never automatically removes a tag. distinguish a candidate defect from a publication conflict. Correct and commit
candidate defects, push `main`, and rerun publication as appropriate. A failure
after local tag creation may leave a ref even when its object ID is unavailable.
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the Status 3 means the push outcome remains unknown. Do not retry or remove the local
attempted version and inspect origin before retrying: tag until the authoritative outcome is resolved. ReleaseKit attempts to save
private publication evidence and reports a record path or a separate recording
warning. A recording failure does not change the publication outcome.
Use the recovery command reported for that attempt, or discover local records
and explicitly select the matching attempt:
```sh ```sh
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}" ./scripts/releasekit.sh publication-attempts
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}" ./scripts/releasekit.sh reconcile --record /absolute/path/to/selected-record.json
``` ```
Matching remote tag and peeled commit values confirm publication, even if the Discovery is local; reconciliation observes the original destination without
push command reported an error. If the remote query fails, publication status rerunning validation, changing tags, or delivering hosted content. Neither is
is still unknown. Do not delete or recreate the local tag until absence on a routine gate after successful publication. Follow the upstream
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's [recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/release.md#resolve-publication-failures)
local tag can be removed and the publication command rerun. Never overwrite for outcomes, incomplete evidence, and older attempts without records.
a published tag.
CI or module-resolution problems may be investigated separately; temporary Records are stored under Git's common directory by default and can contain
service, network, or authentication failures do not require another release. credential-bearing endpoints. Keep them private, out of source control and
Only an actual defect requiring changes to published contents warrants a new shared logs; retain them while needed for recovery and remove them when that
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery). value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
for storage and retention details.
## Policy Changes Current absence does not prove a version was never published. Remove a retained
local tag only after establishing absence and confirming it was not already
Update the policy and this procedure together when changing release governance. published or used by another release channel. Never reuse a known published
Consumers own their dependency-availability and release checks; their readiness version or replace a published tag or hosted page. Corrections require a new
does not delay completion of a Promptkit release. appropriate version.

View File

@@ -15,7 +15,7 @@ the existing Go API. This refactor requires no consumer code changes.
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
`v2.20260922.0`; their exact inventories are linked from the `v2.20260922.0`; their exact inventories are linked from the
[format reference](../formats.md#external-profile-inventories). [migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
The maintained names provide continuity only within the catalog versions The maintained names provide continuity only within the catalog versions
selected and tested by a Promptkit release and when consumers do not override selected and tested by a Promptkit release and when consumers do not override
@@ -37,8 +37,8 @@ an available profile or supply their own complete definition; the new
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic `deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
aliases and need not preserve the previous settings. aliases and need not preserve the previous settings.
See the [release policy](../release.md#catalog-profile-compatibility) for the See the [release policy](../policy/release.md#versioning-and-compatibility) for the
compatibility boundary, the [format reference](../formats.md#built-in-profile-catalog) compatibility boundary, the [migration identity mapping](../consumers/configuration-migration.md#identity-mapping)
for current Promptkit-owned definitions and selected external inventories, and for current resource mappings, and
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model) the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
for configuration choices. for configuration choices.

View File

@@ -72,5 +72,5 @@ for a structured proposal-and-assessment workflow.
Backend catalog selections are unchanged from v0.10.0: OpenRouter Backend catalog selections are unchanged from v0.10.0: OpenRouter
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the `v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility) [release compatibility policy](../policy/release.md#versioning-and-compatibility)
and [profile inventories](../formats.md#external-profile-inventories). and [migration identity mapping](../consumers/configuration-migration.md#identity-mapping).

View File

@@ -15,7 +15,7 @@ No public API, runtime behavior, or profile changes are introduced.
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed. Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
See the [project grant](../../COPYRIGHT), See the [project grant](../../COPYRIGHT),
[component attribution](../oss/attribution.md), and [component attribution](../oss/attribution.md), and
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility). [release compatibility policy](../policy/release.md#versioning-and-compatibility).
## Upgrade ## Upgrade

74
docs/releases/v0.12.0.md Normal file
View File

@@ -0,0 +1,74 @@
# Promptkit v0.12.0
This release replaces Promptkit's configuration loader with ConfigKit and
migrates maintained catalogs to keyed `/v3` assets. It intentionally changes
the public construction API and configuration format.
## Summary
Backends, profiles, and versioned prompts now share one layered configuration
snapshot built when `NewEngine` is called. ConfigKit supplies composition,
inheritance, presence, and source history; Promptkit validates the final
effective definitions and captures prompt content and templates. Consumers can
inspect the captured configuration explicitly without triggering a reload.
## Compatibility
This is an incompatible minor release under Promptkit's pre-1.0
[versioning policy](../policy/release.md#versioning-and-compatibility). The
former source options, `Config.PromptDir`, `Config.ProfileDir`, and the old
profile/backend convenience values have been removed. Configuration documents
now use literal keys beneath `backends`, `profiles`, and `prompts`; embedded
`id`/`version` fields and `base_profile` are no longer accepted. Source groups
compose strictly within a group and overlay in the supplied order. Explicit
zero, false, empty, and null values can now differ from omission; maps overlay
recursively, and an empty map does not clear an inherited map.
All final definitions, including unused ones, are validated during engine
construction. Prompt text and static templates are captured then; an existing
engine does not automatically reload changed configuration or message files.
Request artifacts, schemas, credentials, and provider operations retain their
runtime boundaries. See the [format reference](../formats.md) for the current
configuration contract and the [consumer migration guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
for API and source-selection changes.
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
assets directly must adopt the keyed format. The maintained Promptkit profile
IDs remain available under the [release compatibility policy](../policy/release.md#versioning-and-compatibility),
which does not promise fixed profile data.
## Upgrade
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.12.0
```
Replace removed source and backend registration options with
`Config.Sources`, explicitly ordered ConfigKit underlays, and optionally a
highest-priority main input. Use `DirectoryInputs` or `FilesystemInputs` for
selected directories, `ProfileSource` and `BackendSource` for typed
programmatic values, or `configkit.Mapping` for explicit null and combined
documents. Move profile `base_profile` to `extends` and embedded identities to
mapping keys. Review zero/false/null values and any assumptions that replacing
a model or backend discards unrelated explicit settings. Rebuild the engine
when definitions or referenced message files change. The
[consumer guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
provides the migration details and maintained examples.
## Changes
- Add ConfigKit v0.5.0 as the configuration boundary and consume released
OpenRouter and Rakestrawhome `/v3` catalog modules.
- Add keyed backend, profile, and prompt definitions with final-value domain
validation, explicit presence/null behavior, and source-relative content
capture.
- Add `ConfigurationSources`, directory discovery helpers, programmatic
profile/backend source builders, and read-only configuration inspection.
- Preserve execution workflows while rendering captured templates, applying
backend defaults after inheritance, and honoring explicit suppression of
provider settings.
- Remove superseded loader, catalog preflight, and inheritance machinery;
update examples, format guidance, and internal documentation.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/oss/attribution.md)

69
docs/releases/v0.13.0.md Normal file
View File

@@ -0,0 +1,69 @@
# Promptkit v0.13.0
This release adds prompt requirement discovery and profile/prompt listing,
reduces avoidable configuration and execution allocations, and standardizes
maintainer publication through ReleaseKit. The minimum supported Go version
increases to 1.26.0.
## Discover Requirements And Available Definitions
- `Engine.InspectRequirements` reports recognizable input and caller-variable
dependencies for a selected prompt and supplied caller context before input
artifacts exist. It classifies dependencies as required, conditional, or
optional and reports structured limitations when discovery is incomplete.
Discovery coverage is not proof that requirements are satisfied or execution
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
and [GoDoc](../../requirements.go).
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
engine's captured configuration, are lexically ordered and caller-owned, and
include consumer definitions and maintained profiles that remain effective.
Listing performs no provider or execution work. See the
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections)
and [GoDoc](../../listing.go).
## Efficiency And Tooling
- Reduce response-sized copies in basic validation and rendered-message output,
and encode valid provider extra parameters only as part of the complete
request rather than serializing them twice.
- Use bounded initial file-size reservations without limiting accepted content,
releasing excess backing storage for small or empty completed artifacts.
- Reuse owned preparation values when creating prepared-execution snapshots,
while preserving independent inspection details and caller-owned data.
- Use lightweight captured profile identities and prompt metadata for inspection
instead of copying execution targets and definitions unnecessarily.
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
and inspection processing. Configuration semantics remain unchanged; these
improvements primarily benefit construction and explicit operator inspection,
not upstream inference latency.
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
publication. The [release procedure](../release.md) owns the current commands
and recovery workflow; successful tag publication remains the completion
point, without waiting for CI.
These changes preserve existing execution and ownership contracts. Benefits
depend on workload; this release introduces no fixed performance guarantee.
## Compatibility And Upgrade
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
previous 1.25.5 minimum. Update application build environments and any CI images
still using Go 1.25 before upgrading:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
```
No application-code or configuration migration is required from v0.12.0 beyond
the Go toolchain update. The new discovery methods are optional additions;
existing construction, inspection, preparation, and execution workflows remain
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
for the distinction between the consumer minimum and maintainer toolchain.
Maintainers should follow the current release runbook rather than the retired
`scripts/release-check.sh`. The publication entry point remains
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
mechanics.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)

48
docs/releases/v0.14.0.md Normal file
View File

@@ -0,0 +1,48 @@
# Promptkit v0.14.0
This release adds comparison of captured configuration inspections and updates
ConfigKit and the maintainer release tool. The minimum supported Go version
remains 1.26.0.
## Compare Captured Configurations
`ConfigurationInspection.Diff` compares the effective configuration captured by
two engines. Consumers can use it to review a candidate configuration, inspect
an overlay's effect, or present configuration changes to operators.
Comparison delegates to ConfigKit and returns its difference records, including
presence information, paired redaction, and protected-change indicators. It
does not reload sources or contact providers. Source-history differences alone
do not produce value differences.
An empty diff is not proof of equivalent execution: referenced file contents,
resolved credentials, later execution preparation, and request-specific
overrides are outside the comparison. Records can contain ordinary application
data; callers retain responsibility for their disclosure.
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
## Dependency And Tooling Updates
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
behavior from v0.7.0.
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
retains its existing root-module profile, publication entry point, and release
gates. See the [release procedure](../release.md) for maintainer commands.
## Compatibility And Upgrade
No application-code, configuration, or Go-toolchain migration is required from
v0.13.0. Configuration comparison is an optional API addition; existing
construction, inspection, preparation, and execution workflows remain available.
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
```
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
distinction between the consumer minimum and maintainer toolchain.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)

34
docs/releases/v0.15.0.md Normal file
View File

@@ -0,0 +1,34 @@
# Promptkit v0.15.0
This release adds error-only validation of a concrete prompt request without
calling a model. The minimum supported Go version remains 1.26.0.
## Validate A Concrete Request
`Engine.ValidateRequest` accepts an ordinary `RunRequest` and returns nil when
it passes preparation, or the existing preparation error. Consumers can use it
for fail-fast checks, including downstream CLI validation commands, without
receiving a prepared snapshot.
The method shares `Prepare`'s request conversion and preparation path. Unlike
requirements inspection, it checks the actual supplied values and bindings.
Artifact and schema resource I/O still occurs; success does not certify provider
acceptance or later execution success. A subsequent run prepares independently.
Validation is optional, not a required step before preparation or execution.
See the [request validation guide](../consumers/pkg-promptkit.md#validate-a-concrete-request)
and [GoDoc](../../engine.go) for usage and the exact contract. Consumers that
need execution tied to a checked snapshot can use the existing
[prepared execution workflow](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
## Compatibility And Upgrade
No application-code, configuration, or Go-toolchain migration is required from
v0.14.0. This is an optional API addition; existing inspection, preparation, and
execution workflows remain available.
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.15.0
```
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/docs/oss/attribution.md)

86
docs/releases/v0.16.0.md Normal file
View File

@@ -0,0 +1,86 @@
# Promptkit v0.16.0
This release adds explicit smoke diagnostics, configuration collections,
namespace-scoped engine views, and expanded resource discovery. It also makes
intentional breaking changes to configuration ownership and prompt selection.
The minimum supported Go version remains 1.26.0.
## Configuration Collections And Discovery
Applications can compose caller-selected collections of backends, profiles,
and prompts. Collection resources have qualified identities such as
`support/answer`; root resources retain unqualified identities. `CollectionInput`
provides a convenience for mounting a selected ConfigKit input into a collection.
Promptkit now uses ConfigKit v0.8.0, including its optional source specification
header support.
`Engine.WithNamespaces` creates an immutable view with separate prompt, profile,
and backend scopes for short caller-supplied IDs. Views share the underlying
configuration, clients, capacity pools, and prepared-execution owner. Explicitly
qualified IDs and configured references keep their exact identities.
`ListProfiles` and `ListPrompts` support namespace filters. New `ListBackends`,
`ListNamespaces`, and `ListCollections` methods expose captured resource and
collection inventories. See the [collection format](../formats.md#source-headers-collections-and-resource-ids),
[scoped-view guide](../consumers/pkg-promptkit.md#use-scoped-engine-views), and
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections).
## Explicit Smoke Diagnostics
`Engine.SmokeTest` provides backend reachability, authentication, text inference,
and structured-output inference diagnostics. Reports provide factual
observations, timing, and available usage rather than an overall readiness
verdict. Optional Boolean facts distinguish negative observations from unknown
or inapplicable checks; matching a JSON schema does not prove that a provider
enforced it.
A concluded diagnostic can report failure with a nil Go error. Invalid requests
return ordinary errors, and caller cancellation can return partial observations
with a recognizable context error. Diagnostics can contact providers and incur
inference costs only when explicitly invoked; they are not automatic execution
or release checks. See the [smoke diagnostic guide](../consumers/pkg-promptkit.md#run-explicit-smoke-diagnostics)
and [API GoDoc](../../smoke_diagnostics.go).
## Breaking Changes And Upgrade
Consumers upgrading from v0.15.0 should follow the
[configuration and API migration guide](../consumers/configuration-migration.md):
- Supply backend and profile definitions explicitly. Promptkit no longer bundles
provider catalogs or the `promptkit-openrouter-standard` and
`promptkit-openrouter-auto` profiles. Conventional backend names do not install
definitions automatically. Applications choose and maintain their sources.
- Convert version-keyed prompt mappings into one definition per prompt ID.
Prompt `version` is now an optional descriptive label, not a selector.
Choose deliberately which former definition to install.
- Remove `PromptVersion` selectors from `RunRequest`,
`RequirementInspectionRequest`, and `SmokeTestRequest`, where applicable.
Change prompt inspection calls to `InspectPrompt(ctx, promptID)`. Update code
consuming version-bearing inspection, identity, result, or observation fields
to handle optional labels, including nil and explicitly empty labels.
- When moving resources into collections, update stored backend, profile, and
diagnostic-default references to their intended qualified IDs. Mounting a
source does not rewrite references; profile inheritance remains within its
sibling mapping. Root configuration `version`, when supplied, is now a source
specification header, distinct from descriptive resource labels.
The migration guide includes identity mappings and before/after prompt examples.
The [maintained offline example](../../examples/go-library/prepare/main.go)
demonstrates explicit composition, source-relative content, scoped preparation,
and inventory without contacting a provider.
After adapting configuration and API calls:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.16.0
```
## Correctness And Efficiency
Collection prompt materialization shares one lazy origin-metadata snapshot,
avoiding repeated full metadata copies across collections. Internal prepared
snapshots detach optional prompt labels. Smoke diagnostics preserve caller
cancellation and report actual inference destinations and known-zero usage when
no generation request was dispatched.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/docs/oss/attribution.md)

68
docs/releases/v0.17.0.md Normal file
View File

@@ -0,0 +1,68 @@
# Promptkit v0.17.0
This release adopts the vanity module path `go.maximumdirect.net/promptkit`
and ConfigKit v1.0.0, standardizes configuration resources as identified lists,
and removes descriptive prompt-version metadata. These are intentional breaking
changes before v1. The minimum supported Go version increases from 1.26.0 to
1.27.0 for the ConfigKit v1 migration.
## Breaking Changes And Upgrade
Consumers upgrading from v0.16.0 should follow the
[configuration and API migration guide](../consumers/configuration-migration.md):
- Change Promptkit imports from `gitea.maximumdirect.net/eric/promptkit` to
`go.maximumdirect.net/promptkit`. Update packages that exchange Promptkit
types together; old-path and vanity-path types are distinct.
- Change ConfigKit imports to `go.maximumdirect.net/configkit` at v1.0.0 and
adopt its renamed public inspection types and fields where used.
- Convert root and collection `prompts`, `profiles`, and `backends` from keyed
mappings to lists whose records contain an explicit `id`. Collections remain
keyed by namespace. Sibling inheritance now uses `extends: {id: parent}`;
later same-ID records replace lower records in full.
- Change `ProfileSource` and `BackendSource` calls to pass slices of
`ProfileConfig` and `BackendConfig`, setting each record's `ID`.
- Change source-root `configkit:` controls to `config:`. Imports are optional
unless marked `required: true`. Optional Promptkit specification headers now
use `version: 2` or `version: "2"`; headerless sources remain valid.
- Remove descriptive prompt `version` values and code consuming `PromptVersion`,
`PromptIdentity.Version`, or corresponding `prompt_version` output members.
Collection `release_version` labels remain available. Prompt hashes may change
across this migration.
After adapting Go imports, configuration, and API calls, use Go 1.27.0 or newer:
```sh
go get go.maximumdirect.net/promptkit@v0.17.0
go mod tidy
```
Earlier tags retain their original module declarations. No permanent local
replacement is needed for this cutover. See the
[framework format reference](../formats.md) for current configuration semantics
and the [maintained preparation example](../../examples/go-library/prepare/main.go)
for explicit composition, source-relative imports, and scoped discovery.
## Configuration Inspection And Execution
Configuration inspection and diffs expose ConfigKit's composed values and
provenance, including list-index paths. Promptkit applies domain normalization
and execution defaults to owned interpretations without replacing the source
origins needed for relative resources. Profile inspection and preparation expose
their interpreted or effective settings. See the
[migration guide](../consumers/configuration-migration.md#select-and-compose-the-configuration-you-need)
for the distinction and inspection-path updates.
## Maintainer Tooling
Release tooling now uses the module-selected ReleaseKit v0.8.0 through `go tool`,
with page-only Gitea delivery and saved-publication-attempt recovery. The
[release procedure](../release.md) describes publication and recovery.
The three maintained offline examples now have ordinary Go workflow tests,
included in the standard suite. Their separate Python runner and custom release
hook have been removed; Python is no longer a normal release prerequisite.
The optional release-wiring diagnostic remains Python. See the
[development guide](../development.md#maintainer-validation).
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/docs/oss/attribution.md)

View File

@@ -63,8 +63,8 @@ Every engine now includes the reserved `rakestrawhome` backend, identified by
selects that backend. Consumers can use the maintained endpoint, credential, selects that backend. Consumers can use the maintained endpoint, credential,
capacity, and model defaults without registering either definition themselves. capacity, and model defaults without registering either definition themselves.
See the [built-in backend and profile catalogs](../formats.md#built-in-backends) See the [backend format reference](../formats.md#backend-definitions)
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile) and the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping)
for the current contracts. for the current contracts.
## Structured Generation Errors ## Structured Generation Errors
@@ -126,8 +126,8 @@ cycle-safe, and resolved afresh for ordinary operations. Prepared execution
freezes the fully resolved target. The selected leaf ID remains public while freezes the fully resolved target. The selected leaf ID remains public while
effective execution settings reflect the resolved chain. effective execution settings reflect the resolved chain.
See the [profile inheritance format reference](../formats.md#profile-inheritance), See the [profile inheritance format reference](../formats.md#resource-inheritance),
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile), the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping),
and the [`Profile` GoDoc](../../types.go) for exact merge and validation and the [`Profile` GoDoc](../../types.go) for exact merge and validation
behavior. behavior.

View File

@@ -96,9 +96,8 @@ precedence rules. Promptkit no longer keeps duplicate embedded profile assets
or hard-coded definitions for those maintained backends. or hard-coded definitions for those maintained backends.
The module versions in Promptkit's `go.mod` identify the catalog releases The module versions in Promptkit's `go.mod` identify the catalog releases
tested with this release. The [built-in backend and profile format tested with this release. The [backend format reference](../formats.md#backend-definitions)
reference](../formats.md#built-in-backends) remains the canonical consumer remains the canonical consumer contract, while the [internal source documentation](../internal/sources.md#profiles)
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
describes the dependency boundary. describes the dependency boundary.
## Public API Changes ## Public API Changes

View File

@@ -42,20 +42,21 @@ checkpoint equality or when a broader semantic identity design is selected.
### Eager Source Validation ### Eager Source Validation
**Reason for deferral:** Exact prompt and profile inspection may already **Reason for deferral:** Exact prompt and profile inspection may already
provide a sufficiently small validation surface. Experience from downstream provide a sufficiently small validation surface alongside construction-time
adoption should establish whether an engine-wide operation would add enough definition validation. Experience from downstream adoption should establish
value to justify its broader contract. whether a separate operation would add enough value to justify its broader
contract.
Promptkit could provide an explicit offline operation that discovers and Promptkit could provide an explicit offline operation that also checks schema
structurally validates configured prompt, profile, and schema sources without resources without model generation. `NewEngine` already validates effective
model generation. The normal `NewEngine` path would remain lazy. definitions and captures prompt content; schema resources remain
request-dependent.
An eager operation would need coherent handling for duplicate prompt IDs and Such an operation would need coherent handling for schema syntax and transitive
versions, strict YAML decoding, referenced content files, profile/backend references, context cancellation, and source-specific public errors. Credential
membership, schema syntax and transitive references, context cancellation, declarations must remain separate from credential values; checking current
and source-specific public errors. Credential declarations must remain environment availability, if supported at all, should be an explicit option
separate from credential values; checking current environment availability, and must not expose secrets.
if supported at all, should be an explicit option and must not expose secrets.
Reconsider this idea after downstream use of `InspectPrompt`, Reconsider this idea after downstream use of `InspectPrompt`,
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap. `InspectProfile`, and fixture-based preparation demonstrates a concrete gap.

View File

@@ -38,8 +38,30 @@ consumers.
## Ideas ## Ideas
No ideas are currently awaiting selection. Active feature work belongs in its ### Field-Level Normalization With Preserved List Provenance
focused roadmap rather than this catalog.
Reevaluate normalizing resource fields directly in ConfigKit's effective
configuration after upstream ConfigKit supports recording changes within a
list without replacing the source attribution of unchanged fields.
This could let configuration inspection show PromptKit-normalized values and
field-level change history while preserving the origins needed to resolve
relative resources such as `content_file`. Assess whether that inspection
benefit justifies changing the execution-owned normalization approach.
Include normalization of roles and cache-control values in configured messages
in this reevaluation. This could make inspection reflect the values used for
execution and reduce separate normalization paths. Upstream support would need
to preserve untouched field origins within nested message lists as well as outer
resource lists. PromptKit would continue owning the normalization rules;
ConfigKit would record their effects and provenance. Caller-appended messages
would still need request-time normalization because they do not pass through
ConfigKit.
Consider moving these transformations only if it improves inspection consistency
and simplifies implementation without changing execution behavior.
Upstream support is the reconsideration trigger, not a dependency of the
current ConfigKit migration or a commitment to adopt the enhancement.
## Entry Format ## Entry Format

289
engine.go
View File

@@ -8,14 +8,15 @@ import (
"net/http" "net/http"
"time" "time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/configkit"
internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine" "go.maximumdirect.net/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/observation" internalengine "go.maximumdirect.net/promptkit/internal/engine"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase" "go.maximumdirect.net/promptkit/internal/observation"
"go.maximumdirect.net/promptkit/internal/usecase"
) )
// ErrInvalidConfig identifies invalid engine construction, including missing // ErrInvalidConfig identifies invalid engine construction, including missing
// required configuration, invalid options or backend registrations, and a nil // effective configuration, invalid options, and a nil
// Engine receiver. // Engine receiver.
var ErrInvalidConfig = errors.New("invalid engine configuration") var ErrInvalidConfig = errors.New("invalid engine configuration")
@@ -23,9 +24,8 @@ var (
// ErrInvalidRequest identifies a request whose required values, overrides, // ErrInvalidRequest identifies a request whose required values, overrides,
// credentials, or effective settings are invalid. // credentials, or effective settings are invalid.
ErrInvalidRequest = errors.New("invalid run request") ErrInvalidRequest = errors.New("invalid run request")
// ErrPromptNotFound identifies a requested prompt ID or version that is not // ErrPromptNotFound identifies a requested prompt ID that is not present in
// present in the selected prompt source. It does not also match // the captured configuration. It does not also match ErrPromptLoad.
// ErrPromptLoad.
ErrPromptNotFound = errors.New("prompt not found") ErrPromptNotFound = errors.New("prompt not found")
// ErrProfileNotFound identifies a selected profile ID that is absent from // ErrProfileNotFound identifies a selected profile ID that is absent from
// every configured profile source. It does not also match ErrProfileLoad. // every configured profile source. It does not also match ErrProfileLoad.
@@ -34,13 +34,13 @@ var (
// nor the selected prompt's default profile is present. Such an error also // nor the selected prompt's default profile is present. Such an error also
// matches ErrInvalidRequest. // matches ErrInvalidRequest.
ErrProfileRequired = errors.New("profile selection is required") ErrProfileRequired = errors.New("profile selection is required")
// ErrPromptLoad identifies a failure to read, decode, validate, select, or // ErrPromptLoad identifies a prompt selection failure other than the
// hash a prompt definition, except for the not-found case represented by // not-found case represented by ErrPromptNotFound. Invalid definitions
// ErrPromptNotFound. // instead fail construction with ErrInvalidConfig.
ErrPromptLoad = errors.New("failed to load prompt definition") ErrPromptLoad = errors.New("failed to load prompt definition")
// ErrProfileLoad identifies a failure to read, decode, validate, or select // ErrProfileLoad identifies a profile selection or resolution failure other
// an execution profile or resolve its backend, except for the profile // than the not-found case represented by ErrProfileNotFound. Invalid
// not-found case represented by ErrProfileNotFound. // definitions instead fail construction with ErrInvalidConfig.
ErrProfileLoad = errors.New("failed to load execution profile") ErrProfileLoad = errors.New("failed to load execution profile")
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose // ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
// environment variable is unset or empty after direct RunRequest.APIKey // environment variable is unset or empty after direct RunRequest.APIKey
@@ -81,31 +81,40 @@ var (
ErrValidation = errors.New("failed to validate output") ErrValidation = errors.New("failed to validate output")
) )
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt // Engine inspects prompts and profiles, enumerates prompt requirements, and
// requests. // prepares and runs Promptkit prompt requests.
// //
// An Engine is safe for concurrent calls to [Engine.InspectPrompt], // An Engine is safe for concurrent calls to [Engine.WithNamespaces],
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution], // [Engine.InspectConfiguration],
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent // [Engine.InspectPrompt],
// backend-capacity pools that coordinate Run and RunPrepared admission and // [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// model generation. Injected collaborators may still be invoked concurrently // [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
// across different backend pools or for unlimited backends. // [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
// [Engine.ListCollections]. Engines constructed separately by [NewEngine] own
// independent backend-capacity pools that coordinate Run, RunPrepared, and
// inference SmokeTest admission and model generation. Namespace views returned
// by [Engine.WithNamespaces] share their underlying engine's pools, clients,
// and prepared-execution owner. Injected collaborators may still be invoked
// concurrently across different backend pools or for unlimited backends.
type Engine struct { type Engine struct {
internal *internalengine.Engine internal *internalengine.Engine
} }
// Config selects the directory-backed sources and built-in model-client // ConfigurationSources selects ordered ConfigKit underlays and an optional
// transport used by [NewEngine]. Config has no stable JSON representation. // highest-priority main input. Each underlay's inputs strictly compose within
// their group; later underlays overlay earlier ones. A nil Main uses
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
// may fail construction if invalid.
type ConfigurationSources struct {
Underlays []configkit.Underlay
Main *configkit.Input
}
// Config selects configuration sources and built-in model-client transport
// used by [NewEngine]. Config has no stable JSON representation.
type Config struct { type Config struct {
// PromptDir is the directory searched recursively for prompt definitions. // Sources combines maintained definitions with caller-selected inputs.
// It is required unless a WithPromptFS or WithPromptFile option supplies the Sources ConfigurationSources
// prompt source.
PromptDir string
// ProfileDir is an optional ordinary configured source whose profiles take
// precedence over application fallback and maintained catalog profiles. An
// empty value selects the lower-precedence sources unless a profile-source
// option supplies the ordinary source.
ProfileDir string
// SchemaDir is the root for JSON Schema files. An empty value uses the // SchemaDir is the root for JSON Schema files. An empty value uses the
// current directory. WithSchemaFS or WithSchemaFile replaces this source. // current directory. WithSchemaFS or WithSchemaFile replaces this source.
SchemaDir string SchemaDir string
@@ -128,12 +137,9 @@ type Config struct {
// Option customizes engine construction. // Option customizes engine construction.
// //
// NewEngine applies options in argument order and ignores nil options. Within // NewEngine applies options in argument order and ignores nil options. Within
// each prompt-source, ordinary-profile-source, fallback-profile-source, // each schema-source, model-client, and artifact-reader category, the last
// in-memory-profile, schema-source, model-client, and artifact-reader // non-nil valid option replaces earlier options. An invalid option fails
// category, the last non-nil valid option replaces earlier options in that // construction even if a later option would replace it.
// category. WithBackend is the additive exception: unique registrations
// accumulate, and a repeated backend ID is an error rather than a replacement.
// An invalid option fails construction even if a later option would replace it.
type Option interface { type Option interface {
apply(*engineOptions) error apply(*engineOptions) error
} }
@@ -181,102 +187,6 @@ func WithArtifactReader(reader ArtifactReader) Option {
}) })
} }
// WithPromptFS loads prompt definitions from fsys under root.
//
// The source uses the same strict prompt YAML rules as configured prompt
// directories, and prompt content_file paths resolve within this source.
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
// prompt-source options.
func WithPromptFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetPromptFS(fsys, root)
})
}
// WithPromptFile loads prompt definitions from the single prompt file at path.
//
// Relative prompt content_file paths resolve from the file's directory. path
// must name an existing non-directory file when NewEngine applies the option.
// This option replaces Config.PromptDir and earlier prompt-source options.
func WithPromptFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetPromptFile(path)
})
}
// WithProfileFS loads execution profiles from fsys under root.
//
// Profiles from this ordinary configured source take precedence over
// application fallback and built-in profiles. Profile YAML must use api_key_env
// for environment-based credentials; raw API keys are rejected. fsys must be
// non-nil and root must be non-empty; otherwise NewEngine fails with
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
// FS profile-source options, but remains below WithProfiles in precedence.
func WithProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetProfileFS(fsys, root)
})
}
// WithProfileFile loads execution profiles from the single profile file at path.
//
// The profile takes precedence over application fallback and built-in profiles.
// Profile YAML must use api_key_env for environment-based credentials; raw API
// keys are rejected. path must name an existing non-directory file when
// NewEngine applies the option. This option replaces Config.ProfileDir and
// earlier file or FS profile-source options, but remains below WithProfiles in
// precedence.
func WithProfileFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetProfileFile(path)
})
}
// WithFallbackProfileFS supplies application-owned fallback profile
// definitions from fsys under root.
//
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
// Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
// profiles. Each source supplies a complete profile definition; profile fields
// are not merged between sources. Only an absent profile ID proceeds to the
// next source. A matching read, parse, duplicate, validation, or credential
// format failure stops resolution.
//
// Files use the ordinary strict profile YAML and api_key_env credential rules.
// Loading and validation are lazy: NewEngine validates this option's arguments
// but does not read profile files. fsys must be non-nil and root must be
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
// Repeating this option replaces the earlier valid fallback source.
//
// This option controls profile-definition lookup, not provider or generation
// failover.
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetFallbackProfileFS(fsys, root)
})
}
// WithProfiles configures in-memory profiles that take precedence over
// ordinary configured, application fallback, and built-in profiles.
//
// NewEngine locally validates and copies every profile. IDs must be unique
// within one call. An invalid local definition, duplicate ID, or unsupported
// ExtraParams value makes construction fail with ErrInvalidConfig. A derived
// profile's base reference and resolved target completeness are checked when it
// is selected or inspected. Repeating WithProfiles replaces the complete
// earlier in-memory set rather than merging it.
func WithProfiles(profiles ...Profile) Option {
return optionFunc(func(options *engineOptions) error {
definitions := make([]domain.ExecutionProfile, len(profiles))
for index, publicProfile := range profiles {
definitions[index] = toDomainProfile(publicProfile)
}
return options.internal.SetMemoryProfiles(definitions)
})
}
// WithSchemaFS loads JSON Schema documents from fsys under root. // WithSchemaFS loads JSON Schema documents from fsys under root.
// //
// Prompt schema_path values resolve within this source when schema validation // Prompt schema_path values resolve within this source when schema validation
@@ -302,14 +212,15 @@ func WithSchemaFile(path string) Option {
// NewEngine constructs an Engine from configuration and options. // NewEngine constructs an Engine from configuration and options.
// //
// Options are applied in order according to [Option]. PromptDir is required // ConfigKit composes caller-selected underlays in their given order, followed
// unless a prompt-source option is present. Construction validates option // by the optional main input. Construction validates the final effective
// arguments, in-memory profiles, and backend registrations but defers reading // backend, profile, and prompt definitions and captures referenced prompt
// and validating prompt, file-backed profile, and schema contents until Prepare // content. A missing prompt collection is allowed. Schema documents and
// or Run needs them. // request artifacts remain runtime resources.
// //
// NewEngine returns an error matching ErrInvalidConfig for invalid // NewEngine returns an error matching ErrInvalidConfig for invalid
// configuration, options, or backend-capacity policies. Each constructed // configuration or options, preserving a ConfigKit LoadError when loading
// fails. Each constructed
// Engine has independent backend-capacity pools. Construction does not perform // Engine has independent backend-capacity pools. Construction does not perform
// model requests or require credentials. A negative Config.TransientRetries // model requests or require credentials. A negative Config.TransientRetries
// is invalid. // is invalid.
@@ -320,28 +231,27 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
continue continue
} }
if err := opt.apply(&options); err != nil { if err := opt.apply(&options); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
} }
} }
assembled, err := internalengine.New( assembled, err := internalengine.New(
internalengine.Config{PromptDir: cfg.PromptDir, ProfileDir: cfg.ProfileDir, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)}, internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
options.internal, options.internal,
) )
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err) return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
} }
return &Engine{internal: assembled}, nil return &Engine{internal: assembled}, nil
} }
// InspectPrompt resolves one explicit prompt definition without selecting a // InspectPrompt resolves one prompt ID without selecting a profile or starting
// profile or starting execution work. // execution work.
// //
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and // InspectPrompt requires a nonblank promptID and performs an exact,
// promptVersion values unchanged to the engine's ordinary, case-sensitive // case-sensitive lookup in the captured configuration snapshot. A
// prompt selection. An empty version succeeds only when that source has one // namespace-scoped view qualifies slashless caller IDs before lookup; already
// selected ID; a nonempty version selects one exact ID/version pair. The // qualified IDs pass through.
// configured prompt source is used without merging, fallback, or enumeration.
// //
// A successful result proves that the selected definition and any referenced // A successful result proves that the selected definition and any referenced
// message content files were structurally loaded. Inputs are returned in // message content files were structurally loaded. Inputs are returned in
@@ -349,32 +259,30 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
// resolved. OutputContract is the normalized declared contract, with a JSON // resolved. OutputContract is the normalized declared contract, with a JSON
// Schema path when declared but without loading or compiling that schema. // Schema path when declared but without loading or compiling that schema.
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for // PromptHash is the same opaque equality value as PreparedRun.PromptHash for
// the selected definition and observed source state; its spelling, length, // the selected captured definition; its spelling, length,
// encoding, algorithm, and security properties are not contracts. // encoding, algorithm, and security properties are not contracts.
// //
// This method does not return prompt bodies, templates, source paths, schemas, // This method does not return prompt bodies, templates, source paths, schemas,
// rendered messages, or execution settings. It does not resolve a profile or // rendered messages, or execution settings. It does not resolve a profile or
// credential, read artifacts or schemas, render, validate, admit capacity, // credential, read artifacts or schemas, render, validate, admit capacity,
// contact a provider, or generate model output. The returned PromptInspection // contact a provider, or generate model output. The returned PromptInspection
// and its input slice are caller-owned. Filesystem-backed inspection is a // and its input slice are caller-owned. Inspection does not reload configuration
// point-in-time lookup and does not freeze a definition for later execution. // or content files; construct another Engine to observe source changes.
// //
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID // A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
// matches ErrInvalidRequest. An absent exact ID or version matches // matches ErrInvalidRequest. An absent exact ID matches ErrPromptNotFound and
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate, // not ErrPromptLoad. Invalid definitions and resource failures reject
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad. // construction instead. Cancellation during lookup preserves the context
// Cancellation during lookup matches ErrPromptLoad while preserving the // error. InspectPrompt returns no partial result on error.
// context error. InspectPrompt returns no partial result on error.
func (e *Engine) InspectPrompt( func (e *Engine) InspectPrompt(
ctx context.Context, ctx context.Context,
promptID string, promptID string,
promptVersion string,
) (*PromptInspection, error) { ) (*PromptInspection, error) {
if e == nil || e.internal == nil { if e == nil || e.internal == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
} }
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion) inspection, err := e.internal.InspectPrompt(ctx, promptID)
if err != nil { if err != nil {
return nil, mapPublicError(err) return nil, mapPublicError(err)
} }
@@ -384,10 +292,9 @@ func (e *Engine) InspectPrompt(
// InspectProfile resolves one explicit profile without selecting a prompt or // InspectProfile resolves one explicit profile without selecting a prompt or
// starting execution work. // starting execution work.
// //
// InspectProfile trims surrounding whitespace from profileID and looks up the // InspectProfile looks up a nonblank profileID exactly and case-sensitively
// resulting nonblank ID exactly and case-sensitively through the engine's // in the captured effective configuration. It applies the framework timeout
// in-memory, ordinary configured-source, application fallback, and built-in // baseline, selected
// profile precedence. It applies the framework timeout baseline, selected
// backend, and then selected profile to EffectiveModelParams without a request // backend, and then selected profile to EffectiveModelParams without a request
// override. BackendID is empty for an endpoint-only profile. // override. BackendID is empty for an endpoint-only profile.
// //
@@ -398,16 +305,15 @@ func (e *Engine) InspectPrompt(
// absent or blank named environment variable is not an error. // absent or blank named environment variable is not an error.
// //
// The returned ProfileInspection and all nested mutable values are // The returned ProfileInspection and all nested mutable values are
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and // caller-owned. Inspection does not reload configuration; construct another
// does not freeze the profile for a later execution. This method does not load // Engine to observe source changes. This method does not load
// a prompt, render, read artifacts or schemas, admit backend capacity, contact // a prompt, render, read artifacts or schemas, admit backend capacity, contact
// a provider, or generate model output. // a provider, or generate model output.
// //
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID // A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and // matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend, // not ErrProfileLoad. Invalid definitions or unknown backends reject
// or an invalid resolved target matches ErrProfileLoad. Cancellation during // construction instead. Cancellation preserves the context error.
// profile loading matches ErrProfileLoad while preserving the context error.
// InspectProfile returns no partial result on error. // InspectProfile returns no partial result on error.
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) { func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
if e == nil || e.internal == nil { if e == nil || e.internal == nil {
@@ -441,6 +347,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
// relationship to ctx.Err is promised. Prepare returns no partial result on // relationship to ctx.Err is promised. Prepare returns no partial result on
// error. It validates RunRequest.TransientRetries but makes no model attempts. // error. It validates RunRequest.TransientRetries but makes no model attempts.
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) { func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
prepared, err := e.prepareRequest(ctx, req)
if err != nil {
return nil, err
}
return fromDomainPreparedRun(prepared), nil
}
// ValidateRequest reports whether req passes the same preparation path used by
// [Engine.Prepare] at the time of this call. A nil error means this concrete
// request passed preparation. It checks only this invocation and returns errors
// as encountered rather than aggregating them. All supplied inputs follow the
// same loading behavior as [Engine.Prepare], including inputs unused by the
// selected template branch. The method does not call an LLM or retain a
// prepared snapshot or executable handle, but preparation may still perform
// resource I/O and incur its associated time and memory costs.
//
// Its errors and context handling follow [Engine.Prepare], including an error
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
// when execution must use the snapshot produced by preparation.
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
_, err := e.prepareRequest(ctx, req)
return err
}
func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.PreparedRun, error) {
ctx = observation.WithoutExecutionState(ctx) ctx = observation.WithoutExecutionState(ctx)
if e == nil || e.internal == nil { if e == nil || e.internal == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig) return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
@@ -455,15 +387,16 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
if err != nil { if err != nil {
return nil, mapPublicError(err) return nil, mapPublicError(err)
} }
return fromDomainPreparedRun(prepared), nil return prepared, nil
} }
// PrepareExecution completely prepares a prompt request without calling the // PrepareExecution completely prepares a prompt request without calling the
// configured LLMClient or reserving backend admission capacity. Validated // configured LLMClient or reserving backend admission capacity. Validated
// RunRequest.AppendedMessages are included in the frozen effective messages. // RunRequest.AppendedMessages are included in the frozen effective messages.
// //
// The returned opaque handle is bound to this Engine and permits one // The returned opaque handle is bound to this Engine's underlying execution
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources, // owner, shared with its namespace views, and permits one [Engine.RunPrepared]
// invocation in total. Preparation freezes the selected sources,
// rendered messages, effective settings, inputs, provider structured-output // rendered messages, effective settings, inputs, provider structured-output
// metadata, and validation resources needed by that invocation. The handle // metadata, and validation resources needed by that invocation. The handle
// retains a direct RunRequest.APIKey only in private execution state; // retains a direct RunRequest.APIKey only in private execution state;
@@ -495,8 +428,9 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
} }
// Run prepares a request, invokes the configured LLMClient, and validates the // Run prepares a request, invokes the configured LLMClient, and validates the
// generated output. Each call resolves current sources and composes a fresh, // generated output. Each call uses the Engine's captured configuration and
// stateless effective prompt with any validated RunRequest.AppendedMessages. // composes a fresh effective prompt with any validated
// RunRequest.AppendedMessages.
// //
// A content-validation failure is a successful run whose // A content-validation failure is a successful run whose
// RunResult.Validation has Status ValidationFailed. When its output contract // RunResult.Validation has Status ValidationFailed. When its output contract
@@ -574,6 +508,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
// RunPrepared atomically claims and executes a handle created by // RunPrepared atomically claims and executes a handle created by
// [Engine.PrepareExecution]. // [Engine.PrepareExecution].
// //
// The originating Engine and namespace views sharing its underlying owner may
// claim the handle. Execution uses its captured selections without applying the
// invoking view's namespaces. An Engine constructed separately by [NewEngine]
// is a foreign owner.
//
// A valid owning-Engine invocation consumes the handle's one attempt before // A valid owning-Engine invocation consumes the handle's one attempt before
// credential revalidation, backend admission, generation, or validation. // credential revalidation, backend admission, generation, or validation.
// Cancellation, capacity rejection, generation failure, operational // Cancellation, capacity rejection, generation failure, operational

File diff suppressed because it is too large Load Diff

View File

@@ -5,12 +5,12 @@ import (
"fmt" "fmt"
"strings" "strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity" "go.maximumdirect.net/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/generation" "go.maximumdirect.net/promptkit/internal/generation"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile" "go.maximumdirect.net/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef" "go.maximumdirect.net/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase" "go.maximumdirect.net/promptkit/internal/usecase"
) )
func mapPublicError(err error) error { func mapPublicError(err error) error {
@@ -72,10 +72,6 @@ func publicErrorFor(err error) error {
return ErrPromptLoad return ErrPromptLoad
case errors.Is(err, usecase.ErrProfileLoad): case errors.Is(err, usecase.ErrProfileLoad):
return ErrProfileLoad return ErrProfileLoad
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
return ErrPromptLoad
case isProfileLoadCause(err):
return ErrProfileLoad
case errors.Is(err, usecase.ErrAPIKeyEnvMissing): case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing) return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
case errors.Is(err, capacity.ErrCapacityExceeded): case errors.Is(err, capacity.ErrCapacityExceeded):
@@ -94,9 +90,3 @@ func publicErrorFor(err error) error {
return nil return nil
} }
} }
func isProfileLoadCause(err error) bool {
return errors.Is(err, profile.ErrInvalidYAML) ||
errors.Is(err, profile.ErrInvalidProfile) ||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
}

View File

@@ -6,8 +6,8 @@ import (
"fmt" "fmt"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase" "go.maximumdirect.net/promptkit/internal/usecase"
) )
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) { func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {

View File

@@ -10,7 +10,8 @@ import (
"os" "os"
"strings" "strings"
"gitea.maximumdirect.net/eric/promptkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
) )
//go:embed prompts/*.yaml schemas/*.json //go:embed prompts/*.yaml schemas/*.json
@@ -78,14 +79,26 @@ type summary struct {
func run() (summary, error) { func run() (summary, error) {
transport := &scriptedTransport{} transport := &scriptedTransport{}
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}}, inputs, err := promptkit.FilesystemInputs("example prompts", assets, "prompts")
promptkit.WithPromptFS(assets, "prompts"), if err != nil {
promptkit.WithSchemaFS(assets, "schemas"), return summary{}, err
promptkit.WithBackend(promptkit.Backend{ID: "offline-backend", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1}), }
promptkit.WithProfiles( backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
promptkit.Profile{ID: "proposal-profile", BackendID: "offline-backend", Model: "proposal-model"}, backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
promptkit.Profile{ID: "assessment-profile", BackendID: "offline-backend", Model: "assessment-model"}, backend.ID = backendID
)) inputs = append(inputs,
configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{
backend,
})},
configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
})},
)
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
HTTPClient: &http.Client{Transport: transport},
}, promptkit.WithSchemaFS(assets, "schemas"))
if err != nil { if err != nil {
return summary{}, err return summary{}, err
} }
@@ -145,13 +158,16 @@ func run() (summary, error) {
} }
func main() { func main() {
value, err := run() if err := execute(os.Stdout); err != nil {
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err := json.NewEncoder(os.Stdout).Encode(value); err != nil {
fmt.Fprintln(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
os.Exit(1) os.Exit(1)
} }
} }
func execute(out io.Writer) error {
value, err := run()
if err != nil {
return err
}
return json.NewEncoder(out).Encode(value)
}

View File

@@ -0,0 +1,30 @@
package main
import (
"bytes"
"encoding/json"
"testing"
)
func TestObserveExample(t *testing.T) {
var output bytes.Buffer
if err := execute(&output); err != nil {
t.Fatal(err)
}
var got struct {
ProposalAttempts *int `json:"proposal_attempts"`
ProposalRetries *int `json:"proposal_transient_retries"`
UsageComplete *bool `json:"proposal_usage_complete"`
AssessmentAttempts *int `json:"assessment_attempts"`
AssessmentRetries *int `json:"assessment_transient_retries"`
Outcome string `json:"assessment_outcome"`
}
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.ProposalAttempts == nil || *got.ProposalAttempts != 2 || got.ProposalRetries == nil || *got.ProposalRetries != 1 ||
got.UsageComplete == nil || *got.UsageComplete || got.AssessmentAttempts == nil || *got.AssessmentAttempts != 1 ||
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" {
t.Fatalf("unexpected observation summary: %+v", got)
}
}

View File

@@ -1,17 +1,17 @@
id: example.assessment prompts:
version: "1.0.0" - id: example.assessment
default_profile: assessment-profile default_profile: assessment-profile
description: Assess a structured proposal independently of schema validity. description: Assess a structured proposal independently of schema validity.
inputs: inputs:
- name: proposal - name: proposal
required: true required: true
content_type: application/json content_type: application/json
messages: messages:
- role: system - role: system
content: Return a concise assessment as JSON. content: Return a concise assessment as JSON.
- role: user - role: user
content: '{{input "proposal"}}' content: '{{input "proposal"}}'
output: output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: assessment.schema.json schema_path: assessment.schema.json

View File

@@ -1,17 +1,17 @@
id: example.proposal prompts:
version: "1.0.0" - id: example.proposal
default_profile: proposal-profile default_profile: proposal-profile
description: Propose a bounded improvement from a synthetic idea. description: Propose a bounded improvement from a synthetic idea.
inputs: inputs:
- name: idea - name: idea
required: true required: true
content_type: text/plain content_type: text/plain
messages: messages:
- role: system - role: system
content: Return a concise proposal as JSON. content: Return a concise proposal as JSON.
- role: user - role: user
content: '{{input "idea"}}' content: '{{input "idea"}}'
output: output:
format: json format: json
validation_mode: json_schema validation_mode: json_schema
schema_path: proposal.schema.json schema_path: proposal.schema.json

View File

@@ -0,0 +1,17 @@
backends:
- id: offline
endpoint: https://review.example.invalid/v1
profiles:
- id: audit
backend: review/offline
model: review-model
prompts:
- id: check
default_profile: review/audit
description: Inspect a review item without contacting a model provider.
messages:
- role: user
content: Review the supplied synthetic example.
output:
format: text
validation_mode: none

View File

@@ -0,0 +1,10 @@
version: "2"
config:
imports:
- path: fragments/resources.yaml
required: true
- path: fragments/prepare.yaml
destination: [prompts]
placement: sequence-item
required: true
release_version: "1.4.0"

View File

@@ -1,6 +1,5 @@
id: example.prepare id: prepare
version: "1.0.0" default_profile: support/fast
default_profile: offline-example
description: Prepare a prompt without contacting a model provider. description: Prepare a prompt without contacting a model provider.
inputs: inputs:
- name: note - name: note
@@ -10,7 +9,7 @@ messages:
- role: system - role: system
content: Summarize the note in one sentence. content: Summarize the note in one sentence.
- role: user - role: user
content: '{{input "note"}}' content_file: ../messages/note.md
output: output:
format: text format: text
validation_mode: basic validation_mode: basic

View File

@@ -0,0 +1,11 @@
backends:
- id: offline
endpoint: https://example.invalid/v1
profiles:
- id: base
backend: support/offline
model: offline-model
timeout_seconds: 90
- id: fast
extends: {id: base}
timeout_seconds: 60

View File

@@ -0,0 +1 @@
{{input "note"}}

View File

@@ -0,0 +1,5 @@
# A later input replaces the profile with an inherited variant.
profiles:
- id: fast
extends: {id: base}
timeout_seconds: 30

View File

@@ -4,57 +4,162 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"os" "os"
"path/filepath"
"gitea.maximumdirect.net/eric/promptkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
) )
type summary struct { type summary struct {
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version"` SelectedProfile string `json:"selected_profile"`
SelectedProfile string `json:"selected_profile"` Model string `json:"model"`
Model string `json:"model"` MessageCount int `json:"message_count"`
MessageCount int `json:"message_count"` ProfileTimeout int `json:"profile_timeout_seconds"`
Namespaces []string `json:"namespaces"`
Collections []collectionSummary `json:"collections"`
ScopedPrompts []string `json:"scoped_prompts"`
ReviewPrompts []string `json:"review_prompts"`
}
type collectionSummary struct {
Namespace string `json:"namespace"`
ReleaseVersion *string `json:"release_version"`
} }
func main() { func main() {
engine, err := promptkit.NewEngine( if err := run(context.Background(), "examples/go-library/prepare", os.Stdout); err != nil {
promptkit.Config{}, fmt.Fprintln(os.Stderr, err)
promptkit.WithPromptFile("examples/go-library/prepare/prompt.yaml"), os.Exit(1)
promptkit.WithProfiles(promptkit.Profile{ }
ID: "offline-example", }
Endpoint: "https://example.invalid/v1",
Model: "offline-model", func run(ctx context.Context, exampleDir string, out io.Writer) error {
}), support, err := promptkit.CollectionInput(
"support",
configkit.File(filepath.Join(exampleDir, "collections", "support", "collection.yaml")),
) )
if err != nil { if err != nil {
exit(err) return err
}
review, err := promptkit.CollectionInput(
"review",
configkit.File(filepath.Join(exampleDir, "collections", "review", "collection.yaml")),
)
if err != nil {
return err
}
override, err := promptkit.CollectionInput(
"support",
configkit.File(filepath.Join(exampleDir, "collections", "support", "operator-overrides.yaml")),
)
if err != nil {
return err
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{support, review}},
{Inputs: []configkit.Input{override}},
},
}})
if err != nil {
return err
}
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
Prompt: "support", Profile: "support", Backend: "support",
})
if err != nil {
return err
}
view, err := engine.InspectConfiguration()
if err != nil {
return err
}
if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
return fmt.Errorf("collection profile override was not found")
}
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
PromptID: "prepare",
})
if err != nil {
return err
}
if !requirements.Complete || len(requirements.Inputs) != 1 ||
requirements.Inputs[0].Name != "note" ||
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
return fmt.Errorf("required note input was not discovered before preparation")
} }
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ request := promptkit.RunRequest{
PromptID: "example.prepare", PromptID: "prepare",
ProfileID: "fast",
Inputs: map[string]promptkit.ArtifactRef{ Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."), "note": promptkit.Inline("Ada finished the migration review."),
}, },
}
if err := scoped.ValidateRequest(ctx, request); err != nil {
return err
}
prepared, err := scoped.Prepare(ctx, request)
if err != nil {
return err
}
profile, err := scoped.InspectProfile(ctx, "fast")
if err != nil {
return err
}
if profile.EffectiveModelParams.TimeoutSeconds != 30 {
return fmt.Errorf(
"later profile override was not applied: timeout is %d seconds",
profile.EffectiveModelParams.TimeoutSeconds,
)
}
namespaces, err := scoped.ListNamespaces(ctx)
if err != nil {
return err
}
collections, err := scoped.ListCollections(ctx)
if err != nil {
return err
}
scopedPrompts, err := scoped.ListPrompts(ctx)
if err != nil {
return err
}
reviewPrompts, err := scoped.ListPrompts(ctx, promptkit.NamespaceFilter{
Mode: promptkit.NamespaceNamed, Name: "review",
}) })
if err != nil { if err != nil {
exit(err) return err
}
scopedPromptIDs := make([]string, len(scopedPrompts))
for index := range scopedPrompts {
scopedPromptIDs[index] = scopedPrompts[index].ID
}
reviewPromptIDs := make([]string, len(reviewPrompts))
for index := range reviewPrompts {
reviewPromptIDs[index] = reviewPrompts[index].ID
}
collectionSummaries := make([]collectionSummary, len(collections))
for index := range collections {
collectionSummaries[index] = collectionSummary{
Namespace: collections[index].Namespace, ReleaseVersion: collections[index].ReleaseVersion,
}
} }
encoder := json.NewEncoder(os.Stdout) encoder := json.NewEncoder(out)
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
if err := encoder.Encode(summary{ return encoder.Encode(summary{
PromptID: prepared.PromptID, PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
SelectedProfile: prepared.SelectedProfileID, SelectedProfile: prepared.SelectedProfileID,
Model: prepared.EffectiveModelParams.Model, Model: prepared.EffectiveModelParams.Model,
MessageCount: len(prepared.Messages), MessageCount: len(prepared.Messages),
}); err != nil { ProfileTimeout: profile.EffectiveModelParams.TimeoutSeconds,
exit(err) Namespaces: namespaces,
} Collections: collectionSummaries,
} ScopedPrompts: scopedPromptIDs,
ReviewPrompts: reviewPromptIDs,
func exit(err error) { })
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
} }

View File

@@ -0,0 +1,45 @@
package main
import (
"bytes"
"encoding/json"
"reflect"
"testing"
)
func TestPrepareExample(t *testing.T) {
var output bytes.Buffer
if err := run(t.Context(), ".", &output); err != nil {
t.Fatal(err)
}
var got struct {
PromptID string `json:"prompt_id"`
Profile string `json:"selected_profile"`
Model string `json:"model"`
MessageCount int `json:"message_count"`
Timeout int `json:"profile_timeout_seconds"`
Namespaces []string `json:"namespaces"`
Collections []map[string]json.RawMessage `json:"collections"`
ScopedPrompts []string `json:"scoped_prompts"`
ReviewPrompts []string `json:"review_prompts"`
}
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.PromptID != "support/prepare" || got.Profile != "support/fast" || got.Model != "offline-model" ||
got.MessageCount <= 0 || got.Timeout != 30 || !reflect.DeepEqual(got.Namespaces, []string{"review", "support"}) ||
!reflect.DeepEqual(got.ScopedPrompts, []string{"support/prepare"}) || !reflect.DeepEqual(got.ReviewPrompts, []string{"review/check"}) || len(got.Collections) != 2 {
t.Fatalf("unexpected preparation summary: %+v", got)
}
for index, want := range []struct{ namespace, release string }{{"review", ""}, {"support", "1.4.0"}} {
var namespace string
if err := json.Unmarshal(got.Collections[index]["namespace"], &namespace); err != nil || namespace != want.namespace {
t.Fatalf("collection namespace = %q, %v", namespace, err)
}
value, present := got.Collections[index]["release_version"]
var release *string
if !present || json.Unmarshal(value, &release) != nil || (index == 0 && release != nil) || (index == 1 && (release == nil || *release != want.release)) {
t.Fatalf("collection release = %s, present %t; want %q", value, present, want.release)
}
}
}

View File

@@ -4,9 +4,12 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"os" "os"
"path/filepath"
"gitea.maximumdirect.net/eric/promptkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
) )
type deterministicClient struct{} type deterministicClient struct{}
@@ -38,44 +41,41 @@ type summary struct {
} }
func main() { func main() {
engine, err := promptkit.NewEngine( if err := run(context.Background(), "examples/go-library/run", os.Stdout); err != nil {
promptkit.Config{}, fmt.Fprintln(os.Stderr, err)
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"), os.Exit(1)
promptkit.WithProfiles(promptkit.Profile{ }
ID: "offline-example", }
Endpoint: "https://example.invalid/v1",
Model: "offline-model", func run(ctx context.Context, exampleDir string, out io.Writer) error {
}), endpoint, model := "https://example.invalid/v1", "offline-model"
promptkit.WithLLMClient(deterministicClient{}), engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
) Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml"))},
}}, promptkit.WithLLMClient(deterministicClient{}))
if err != nil { if err != nil {
exit(err) return err
} }
result, err := engine.Run(context.Background(), promptkit.RunRequest{ result, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "example.run", PromptID: "example.run",
Inputs: map[string]promptkit.ArtifactRef{ Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."), "note": promptkit.Inline("Ada finished the migration review."),
}, },
}) })
if err != nil { if err != nil {
exit(err) return err
} }
encoder := json.NewEncoder(os.Stdout) encoder := json.NewEncoder(out)
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
if err := encoder.Encode(summary{ return encoder.Encode(summary{
Output: result.RawOutput, Output: result.RawOutput,
ValidationStatus: result.Validation.Status, ValidationStatus: result.Validation.Status,
IsValid: result.Validation.IsValid, IsValid: result.Validation.IsValid,
Model: result.ModelName, Model: result.ModelName,
TotalTokens: result.Usage.TotalTokens, TotalTokens: result.Usage.TotalTokens,
}); err != nil { })
exit(err)
}
}
func exit(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
} }

View File

@@ -0,0 +1,28 @@
package main
import (
"bytes"
"encoding/json"
"testing"
)
func TestRunExample(t *testing.T) {
var output bytes.Buffer
if err := run(t.Context(), ".", &output); err != nil {
t.Fatal(err)
}
var got struct {
Output string `json:"output"`
Valid *bool `json:"is_valid"`
Status string `json:"validation_status"`
Model string `json:"model"`
Tokens *int `json:"total_tokens"`
}
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.Output != "Ada finished the migration review." || got.Valid == nil || !*got.Valid ||
got.Status != "passed" || got.Model != "offline-model" || got.Tokens == nil || *got.Tokens != 18 {
t.Fatalf("unexpected execution summary: %+v", got)
}
}

View File

@@ -1,16 +1,16 @@
id: example.run prompts:
version: "1.0.0" - id: example.run
default_profile: offline-example default_profile: offline-example
description: Run a prompt with a deterministic injected model client. description: Run a prompt with a deterministic injected model client.
inputs: inputs:
- name: note - name: note
required: true required: true
content_type: text/plain content_type: text/plain
messages: messages:
- role: system - role: system
content: Summarize the note in one sentence. content: Summarize the note in one sentence.
- role: user - role: user
content: '{{input "note"}}' content: '{{input "note"}}'
output: output:
format: text format: text
validation_mode: basic validation_mode: basic

View File

@@ -18,9 +18,8 @@ func (r RunRequest) GoString() string {
func (r RunRequest) redactedString() string { func (r RunRequest) redactedString() string {
return fmt.Sprintf( return fmt.Sprintf(
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}", "promptkit.RunRequest{PromptID:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
r.PromptID, r.PromptID,
r.PromptVersion,
r.ProfileID, r.ProfileID,
r.APIKey != "", r.APIKey != "",
len(r.Inputs), len(r.Inputs),

View File

@@ -0,0 +1,176 @@
package promptkit_test
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
const capturedPromptDocument = `
prompts:
- id: captured
default_profile: offline
session_id: '{{.session}}'
messages:
- role: user
content_file: body.tmpl
output:
format: text
validation_mode: none
`
func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
t.Helper()
inputs, err := promptkit.FilesystemInputs("application prompts", source, "config")
if err != nil {
t.Fatalf("select prompt inputs: %v", err)
}
endpoint, model := "https://example.invalid/v1", "offline-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{
{Inputs: inputs},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
{ID: "offline", Endpoint: &endpoint, Model: &model},
})}}},
},
}})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
return engine
}
func TestEngineCapturesPromptContentUntilReconstruction(t *testing.T) {
files := fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte("before {{.name}}")},
}
first := capturedPromptEngine(t, files)
files["config/body.tmpl"] = &fstest.MapFile{Data: []byte("after {{.name}}")}
request := promptkit.RunRequest{PromptID: "captured", Vars: map[string]string{"name": "Ada", "session": "session-1"}}
oldPrepared, err := first.Prepare(context.Background(), request)
if err != nil {
t.Fatalf("prepare with existing engine: %v", err)
}
second := capturedPromptEngine(t, files)
newPrepared, err := second.Prepare(context.Background(), request)
if err != nil {
t.Fatalf("prepare with rebuilt engine: %v", err)
}
if oldPrepared.Messages[0].Content != "before Ada" || newPrepared.Messages[0].Content != "after Ada" || oldPrepared.PromptHash == newPrepared.PromptHash {
t.Fatalf("captured content or hash did not follow engine lifetime: old=%+v new=%+v", oldPrepared, newPrepared)
}
}
type unreadableAfterConstructionFS struct {
fs.FS
closed atomic.Bool
reads atomic.Int64
}
func (source *unreadableAfterConstructionFS) Open(name string) (fs.File, error) {
if source.closed.Load() {
source.reads.Add(1)
return nil, fs.ErrPermission
}
return source.FS.Open(name)
}
func TestCapturedTemplatesBindInputsIndependentlyAcrossConcurrentPreparations(t *testing.T) {
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{.name}}: {{input "note"}}`)},
}}
engine := capturedPromptEngine(t, source)
source.closed.Store(true)
const calls = 24
var wait sync.WaitGroup
errs := make(chan error, calls)
for index := 0; index < calls; index++ {
wait.Add(1)
go func(index int) {
defer wait.Done()
name := fmt.Sprintf("caller-%d", index)
body := fmt.Sprintf("body-%d", index)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "captured", SessionID: "direct-session",
Vars: map[string]string{"name": name},
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline(body)},
})
if err == nil && (prepared.Messages[0].Content != name+": "+body || prepared.SessionID != "direct-session") {
err = fmt.Errorf("unexpected prepared content: %+v", prepared)
}
errs <- err
}(index)
}
wait.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
if source.reads.Load() != 0 {
t.Fatalf("preparation reopened captured source %d times", source.reads.Load())
}
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "captured", Vars: map[string]string{"name": "Ada"},
Inputs: map[string]promptkit.ArtifactRef{"note": promptkit.Inline("body")},
})
if !errors.Is(err, promptkit.ErrPromptRender) {
t.Fatalf("missing session variable = %v, want runtime render error", err)
}
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "captured", SessionID: "direct-session", Vars: map[string]string{"name": "Ada"},
})
if !errors.Is(err, promptkit.ErrPromptRender) {
t.Fatalf("missing input helper artifact = %v, want runtime render error", err)
}
}
func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *testing.T) {
endpoint, model := "https://example.invalid/v1", "offline-model"
schemas := fstest.MapFS{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
{ID: "offline", Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
"prompts": []any{map[string]any{"id": "structured",
"default_profile": "offline",
"inputs": []any{map[string]any{"name": "note", "required": true}},
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
}},
})},
}}, promptkit.WithSchemaFS(schemas, "."))
if err != nil {
t.Fatalf("construct engine before request resources exist: %v", err)
}
schemas["schema.json"] = &fstest.MapFile{Data: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object"}`)}
artifactPath := filepath.Join(t.TempDir(), "note.txt")
if err := os.WriteFile(artifactPath, []byte("request artifact"), 0o600); err != nil {
t.Fatalf("write request artifact: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "structured", Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.File(artifactPath),
}})
if err != nil {
t.Fatalf("prepare with resources supplied after construction: %v", err)
}
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "request artifact" || prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil {
t.Fatalf("runtime artifact or schema was not prepared: %+v", prepared)
}
}

View File

@@ -9,7 +9,7 @@ import (
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestBuiltInGenerationError(t *testing.T) { func TestBuiltInGenerationError(t *testing.T) {
@@ -21,14 +21,14 @@ func TestBuiltInGenerationError(t *testing.T) {
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`) `{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
result, err := engine.Run(context.Background(), generationErrorRunRequest()) result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil { if result != nil {
t.Fatalf("Run result = %#v, want nil", result) t.Fatalf("Run result = %#v, want nil", result)
} }
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker) assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`) preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest()) prepared, err := preparedEngine.PrepareExecution(context.Background(), simpleGenerationRequest())
if err != nil { if err != nil {
t.Fatalf("PrepareExecution: %v", err) t.Fatalf("PrepareExecution: %v", err)
} }
@@ -43,7 +43,7 @@ func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
const messageMarker = "combined-request-provider-marker" const messageMarker = "combined-request-provider-marker"
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
`{"error":{"message":"`+messageMarker+`"}}`) `{"error":{"message":"`+messageMarker+`"}}`)
request := generationErrorRunRequest() request := simpleGenerationRequest()
request.AppendedMessages = []promptkit.RenderedMessage{ request.AppendedMessages = []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "consumer correction"}, {Role: promptkit.RoleUser, Content: "consumer correction"},
@@ -63,7 +63,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
messageMarker = "repair-message-marker" messageMarker = "repair-message-marker"
) )
calls := 0 calls := 0
config := contractConfig(frameworkSchemaDir) config := simpleConfig("message")
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++ calls++
if calls == 1 { if calls == 1 {
@@ -77,7 +77,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewEngine: %v", err) t.Fatalf("NewEngine: %v", err)
} }
req := generationErrorRunRequest() req := simpleGenerationRequest()
req.Validation = &promptkit.OutputContract{ req.Validation = &promptkit.OutputContract{
Format: promptkit.FormatJSON, Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSON, ValidationMode: promptkit.ValidationJSON,
@@ -96,7 +96,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
func TestBuiltInGenerationFailureCategories(t *testing.T) { func TestBuiltInGenerationFailureCategories(t *testing.T) {
transportCause := errors.New("private transport diagnostic") transportCause := errors.New("private transport diagnostic")
config := contractConfig(frameworkSchemaDir) config := simpleConfig("message")
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportCause return nil, transportCause
})} })}
@@ -104,7 +104,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
result, err := engine.Run(context.Background(), generationErrorRunRequest()) result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) { if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) {
t.Fatalf("transport result=%#v, error=%v", result, err) t.Fatalf("transport result=%#v, error=%v", result, err)
} }
@@ -122,7 +122,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) {
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body) engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body)
result, err := engine.Run(context.Background(), generationErrorRunRequest()) result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) { if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) {
t.Fatalf("malformed result=%#v, error=%v", result, err) t.Fatalf("malformed result=%#v, error=%v", result, err)
} }
@@ -139,7 +139,7 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
{name: "deadline", cause: context.DeadlineExceeded}, {name: "deadline", cause: context.DeadlineExceeded},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
config := contractConfig(frameworkSchemaDir) config := simpleConfig("message")
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, tc.cause return nil, tc.cause
})} })}
@@ -147,7 +147,7 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
result, err := engine.Run(context.Background(), generationErrorRunRequest()) result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) { if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) {
t.Fatalf("context result=%#v, error=%v", result, err) t.Fatalf("context result=%#v, error=%v", result, err)
} }
@@ -165,8 +165,8 @@ func TestInjectedGenerationFailureCategories(t *testing.T) {
{name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}}, {name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(tc.client)) engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(tc.client))
result, err := engine.Run(context.Background(), generationErrorRunRequest()) result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed { if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed {
t.Fatalf("injected result=%#v, error=%v", result, err) t.Fatalf("injected result=%#v, error=%v", result, err)
} }
@@ -207,7 +207,7 @@ func assertGenerationError(t *testing.T, err error, statusCode int, code, provid
func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine { func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
t.Helper() t.Helper()
config := contractConfig(frameworkSchemaDir) config := simpleConfig("message")
noRetries := 0 noRetries := 0
config.TransientRetries = &noRetries config.TransientRetries = &noRetries
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
@@ -224,12 +224,6 @@ func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string)
return engine return engine
} }
func generationErrorRunRequest() promptkit.RunRequest { func simpleGenerationRequest() promptkit.RunRequest {
return promptkit.RunRequest{ return promptkit.RunRequest{PromptID: "prompt"}
PromptID: frameworkMarkdownSummaryPromptID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
}
} }

18
go.mod
View File

@@ -1,12 +1,18 @@
module gitea.maximumdirect.net/eric/promptkit module go.maximumdirect.net/promptkit
go 1.25.5 go 1.27.0
require ( require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1 go.maximumdirect.net/configkit v1.0.0
) )
require golang.org/x/text v0.14.0 // indirect require (
github.com/yuin/goldmark/v2 v2.1.6 // indirect
go.maximumdirect.net/releasekit v0.8.0 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
golang.org/x/mod v0.40.0 // indirect
golang.org/x/text v0.14.0 // indirect
)
tool go.maximumdirect.net/releasekit/cmd/releasekit

18
go.sum
View File

@@ -1,14 +1,16 @@
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
go.maximumdirect.net/configkit v1.0.0 h1:cguQnxkaRk3ClQOZNzzhTdc4bfzbraGKv8CkoCp+HLc=
go.maximumdirect.net/configkit v1.0.0/go.mod h1:B6fJ5xBLBF1nuywqfLeEI4sLm29UkIcDLu8i6I6gGXE=
go.maximumdirect.net/releasekit v0.8.0 h1:hSaHPbUI7tAhnolfbu7UMj2fvmm9FGBY+jhkygdbxR0=
go.maximumdirect.net/releasekit v0.8.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -10,8 +10,8 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
var ( var (
@@ -21,6 +21,7 @@ var (
) )
const fileReadChunkSize = 64 * 1024 const fileReadChunkSize = 64 * 1024
const fileReadInitialCapacityLimit = 1 << 20
// Reader resolves artifact references into actual artifacts. // Reader resolves artifact references into actual artifacts.
type Reader interface { type Reader interface {
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path) return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
} }
data := make([]byte, 0) capacity := 0
if size := openedInfo.Size(); size > 0 {
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
}
data := make([]byte, 0, capacity)
chunk := make([]byte, fileReadChunkSize) chunk := make([]byte, fileReadChunkSize)
for { for {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr) return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
} }
} }
if cap(data)-len(data) > len(data) {
// A stale size hint should not make a small completed artifact retain a
// disproportionately large backing array.
compact := make([]byte, len(data))
copy(compact, data)
data = compact
}
contentType := mime.TypeByExtension(filepath.Ext(path)) contentType := mime.TypeByExtension(filepath.Ext(path))
if contentType == "" { if contentType == "" {

View File

@@ -0,0 +1,180 @@
package artifact
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"mime"
"os"
"path/filepath"
"runtime"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
var artifactFileReadBenchmarkSink *domain.Artifact
func BenchmarkArtifactFileRead(b *testing.B) {
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
b.Run(fixture.name, func(b *testing.B) {
reader := &fileReader{open: openArtifactFile}
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
checkArtifactFileRead(b, reader, ref, fixture.body)
b.ReportAllocs()
b.SetBytes(int64(len(fixture.body)))
b.ResetTimer()
for range b.N {
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("read artifact: %v", err)
}
artifactFileReadBenchmarkSink = artifact
}
b.StopTimer()
runtime.KeepAlive(artifactFileReadBenchmarkSink)
})
}
}
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
// opened file reports a size larger than the content that reaches EOF.
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
for _, fixture := range []struct {
name string
body []byte
}{
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
{name: "empty", body: []byte{}},
} {
b.Run(fixture.name, func(b *testing.B) {
benchmarkRetainedArtifactFileRead(b, fixture.body)
})
}
}
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
path := filepath.Join(b.TempDir(), "overreported.txt")
if err := os.WriteFile(path, body, 0o600); err != nil {
b.Fatal(err)
}
mime.TypeByExtension(filepath.Ext(path))
reader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
}}
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
checkArtifactFileRead(b, reader, ref, body)
const retainedBatchSize = 4
retained := make([]*domain.Artifact, retainedBatchSize)
artifactFileReadBenchmarkSink = nil
runtime.GC()
var setupMemory runtime.MemStats
runtime.ReadMemStats(&setupMemory)
runtime.KeepAlive(reader)
runtime.KeepAlive(ref)
b.ReportAllocs()
b.SetBytes(int64(len(body)))
b.ResetTimer()
for iteration := 0; iteration < b.N; iteration++ {
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("read retained artifact: %v", err)
}
retained[iteration%len(retained)] = artifact
}
b.StopTimer()
for i, artifact := range retained {
if artifact != nil {
continue
}
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("fill retained artifact batch: %v", err)
}
retained[i] = artifact
}
runtime.GC()
var retainedMemory runtime.MemStats
runtime.ReadMemStats(&retainedMemory)
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
runtime.KeepAlive(retained)
runtime.KeepAlive(reader)
runtime.KeepAlive(ref)
}
type artifactReadBenchmarkFile struct {
name string
path string
body []byte
}
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
b.Helper()
fixtures := []struct {
name string
size int
}{
{name: "small_4KiB", size: 4 << 10},
{name: "1MiB", size: 1 << 20},
{name: "large_4MiB", size: 4 << 20},
}
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
for _, fixture := range fixtures {
body := bytes.Repeat([]byte("x"), fixture.size)
path := filepath.Join(b.TempDir(), fixture.name+".txt")
if err := os.WriteFile(path, body, 0o600); err != nil {
b.Fatal(err)
}
mime.TypeByExtension(filepath.Ext(path))
results = append(results, artifactReadBenchmarkFile{
name: fixture.name,
path: path,
body: body,
})
}
return results
}
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
b.Helper()
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("check artifact read: %v", err)
}
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
}
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
if artifact.Hash != wantHash {
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
}
}
type sizeReportingArtifactFile struct {
artifactFile
reportedSize int64
}
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
info, err := file.artifactFile.Stat()
if err != nil {
return nil, err
}
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
}
type reportedArtifactFileInfo struct {
os.FileInfo
size int64
}
func (info reportedArtifactFileInfo) Size() int64 { return info.size }

View File

@@ -10,7 +10,7 @@ import (
"testing" "testing"
"time" "time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) { func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) {

View File

@@ -3,13 +3,15 @@ package artifact
import ( import (
"bytes" "bytes"
"context" "context"
"crypto/sha256"
"errors" "errors"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) { func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) {
@@ -232,6 +234,64 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
}) })
} }
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
tests := []struct {
name string
body []byte
reportedSize int64
}{
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
t.Fatal(err)
}
reader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
file.Close()
return nil, err
}
return &reportedInfoFile{
artifactFile: file,
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
}, nil
}}
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: filePath,
})
if err != nil {
t.Fatalf("read artifact: %v", err)
}
if !bytes.Equal(artifact.Body, tc.body) {
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
}
if artifact.Size != int64(len(tc.body)) {
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
}
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
}
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
if artifact.Hash != wantHash {
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
}
})
}
}
func TestFileReaderCancelsAfterReadProgress(t *testing.T) { func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.bin") filePath := filepath.Join(t.TempDir(), "artifact.bin")
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2) content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
@@ -268,6 +328,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
return f.info, nil return f.info, nil
} }
type reportedFileSizeInfo struct {
os.FileInfo
size int64
}
func (info reportedFileSizeInfo) Size() int64 { return info.size }
type cancelAfterProgressFile struct { type cancelAfterProgressFile struct {
artifactFile artifactFile
cancel context.CancelFunc cancel context.CancelFunc

View File

@@ -5,63 +5,38 @@ package backend
import ( import (
"errors" "errors"
"fmt" "fmt"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
) )
const ( const (
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter // OpenRouterID is the conventional OpenRouter backend identifier.
// backend.
OpenRouterID = "openrouter" OpenRouterID = "openrouter"
// RakestrawHomeID is the reserved ID of Promptkit's built-in Rakestrawhome // RakestrawHomeID is the conventional Rakestrawhome backend identifier.
// backend.
RakestrawHomeID = "rakestrawhome" RakestrawHomeID = "rakestrawhome"
defaultQueueCapacity = 1024
) )
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID. // ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
var ErrBackendNotFound = errors.New("backend not found") var ErrBackendNotFound = errors.New("backend not found")
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// Registry is an immutable collection of validated backend definitions. // Registry is an immutable collection of validated backend definitions.
type Registry struct { type Registry struct {
backends map[string]domain.Backend backends map[string]domain.Backend
} }
// NewRegistry constructs a registry containing maintained definitions followed // NewEffectiveRegistry stores already-validated effective backend definitions
// by consumer additions. Every ID must be unique across both groups. // under their exact configuration keys. Configuration validation and
func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) { // normalization precede this ownership boundary.
registry := &Registry{ func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
backends: make(map[string]domain.Backend, len(maintained)+len(additions)), registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
} for id, definition := range definitions {
definition.ID = id
definitions := make([]domain.Backend, 0, len(maintained)+len(additions)) cloned, err := domain.CloneBackend(definition)
definitions = append(definitions, maintained...)
definitions = append(definitions, additions...)
for _, definition := range definitions {
definition.ID = strings.TrimSpace(definition.ID)
if definition.ID == "" {
return nil, errors.New("backend ID must not be blank")
}
if _, exists := registry.backends[definition.ID]; exists {
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
}
normalized, err := NormalizeDefinition(definition)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("backend %q: %w", id, err)
} }
registry.backends[normalized.ID] = normalized registry.backends[id] = cloned
} }
return registry, nil return registry, nil
} }
@@ -74,12 +49,11 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if !ok { if !ok {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id) return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
} }
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams) cloned, err := domain.CloneBackend(definition)
if err != nil { if err != nil {
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err) return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
} }
definition.ExtraParams = extraParams return cloned, nil
return definition, nil
} }
// CapacityPolicies returns a copy of the normalized policies for limited // CapacityPolicies returns a copy of the normalized policies for limited
@@ -100,80 +74,3 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
} }
return policies return policies
} }
// NormalizeDefinition validates and defensively copies one backend definition.
func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
}
definition.Endpoint = endpoint
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
return domain.Backend{}, fmt.Errorf(
"backend %q api key environment variable %q is invalid",
definition.ID,
definition.APIKeyEnv,
)
}
if definition.ConcurrencyLimit < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q concurrency limit must not be negative",
definition.ID,
)
}
if definition.QueueCapacity < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity must not be negative",
definition.ID,
)
}
if definition.ConcurrencyLimit == 0 {
if definition.QueueCapacitySet {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity requires a positive concurrency limit",
definition.ID,
)
}
definition.QueueCapacity = 0
} else {
if !definition.QueueCapacitySet {
definition.QueueCapacity = defaultQueueCapacity
definition.QueueCapacitySet = true
}
maxInt := int(^uint(0) >> 1)
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
return domain.Backend{}, fmt.Errorf(
"backend %q total capacity overflows int",
definition.ID,
)
}
}
keys := make([]string, 0, len(definition.ExtraParams))
for key := range definition.ExtraParams {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if key == "" {
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
}
if llm.IsReservedOpenAIChatRequestField(key) {
return domain.Backend{}, fmt.Errorf(
"backend %q extra parameter %q collides with a reserved request field",
definition.ID,
key,
)
}
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return definition, nil
}

View File

@@ -3,374 +3,90 @@ package backend_test
import ( import (
"errors" "errors"
"reflect" "reflect"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend" "go.maximumdirect.net/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
const validEndpoint = "https://backend.example/v1" func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
params := map[string]any{"nested": map[string]any{"enabled": true}}
func TestRegistryIncludesMaintainedDefinitions(t *testing.T) { defaultProfile := "catalog-profile"
maintained := []domain.Backend{ reachabilityEndpoint := "https://probe.example/ready"
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true}, authenticationKind := "openrouter_key"
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true}, authenticationEndpoint := "https://probe.example/key"
} smokeTests := &domain.BackendSmokeTestConfig{
registry, err := backend.NewRegistry(maintained, nil) DefaultProfile: &defaultProfile,
if err != nil { Reachability: &domain.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
t.Fatalf("construct registry: %v", err) Authentication: &domain.AuthenticationProbeConfig{
} Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
for _, expected := range maintained {
t.Run(expected.ID, func(t *testing.T) {
definition, err := registry.GetBackend(expected.ID)
if err != nil {
t.Fatalf("look up maintained definition: %v", err)
}
if !reflect.DeepEqual(definition, expected) {
t.Fatalf("unexpected maintained definition: %#v", definition)
}
})
}
policies := registry.CapacityPolicies()
if len(policies) != 2 ||
policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
QueueCapacity: 3,
}) ||
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 4,
QueueCapacity: 5,
}) {
t.Fatalf("unexpected built-in capacity policies: %#v", policies)
}
}
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
extraParams := map[string]any{
"count": int64(7),
"nested": nested,
}
registry, err := backend.NewRegistry(nil, []domain.Backend{
{
ID: " custom ",
Endpoint: " https://custom.example/openai/v1 ",
APIKeyEnv: " CUSTOM_API_KEY ",
ExtraParams: extraParams,
ConcurrencyLimit: 3,
QueueCapacity: 2,
QueueCapacitySet: true,
}, },
{ }
ID: "Custom", registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
Endpoint: validEndpoint, " local.v1 ": {
Endpoint: "https://backend.example/v1", ExtraParams: params,
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
SmokeTests: smokeTests,
}, },
}) })
if err != nil { if err != nil {
t.Fatalf("construct registry: %v", err) t.Fatal(err)
} }
params["nested"].(map[string]any)["enabled"] = false
nested["limit"] = 99 defaultProfile = "mutated-profile"
extraParams["added"] = true reachabilityEndpoint = "https://mutated.example/ready"
authenticationKind = "none"
got, err := registry.GetBackend("custom") authenticationEndpoint = "https://mutated.example/key"
if err != nil { selected, err := registry.GetBackend(" local.v1 ")
t.Fatalf("look up custom backend: %v", err) if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
} }
if got.ID != "custom" || if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
got.Endpoint != "https://custom.example/openai/v1" || *selected.SmokeTests.DefaultProfile != "catalog-profile" || selected.SmokeTests.Reachability == nil ||
got.APIKeyEnv != "CUSTOM_API_KEY" || selected.SmokeTests.Reachability.Endpoint == nil || *selected.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
got.ConcurrencyLimit != 3 || selected.SmokeTests.Authentication == nil || selected.SmokeTests.Authentication.Kind == nil ||
got.QueueCapacity != 2 || *selected.SmokeTests.Authentication.Kind != "openrouter_key" || selected.SmokeTests.Authentication.Endpoint == nil ||
!got.QueueCapacitySet { *selected.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
t.Fatalf("unexpected normalized definition: %#v", got) t.Fatalf("registry did not capture nested diagnostic metadata: %+v", selected.SmokeTests)
} }
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 { selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"]) *selected.SmokeTests.DefaultProfile = "mutated-return"
*selected.SmokeTests.Reachability.Endpoint = "https://mutated-return.example/ready"
*selected.SmokeTests.Authentication.Kind = "none"
*selected.SmokeTests.Authentication.Endpoint = "https://mutated-return.example/key"
again, err := registry.GetBackend(" local.v1 ")
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
} }
gotNested, ok := got.ExtraParams["nested"].(map[string]int) if again.SmokeTests == nil || again.SmokeTests.DefaultProfile == nil ||
if !ok || gotNested["limit"] != 2 { *again.SmokeTests.DefaultProfile != "catalog-profile" || again.SmokeTests.Reachability == nil ||
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"]) again.SmokeTests.Reachability.Endpoint == nil || *again.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
again.SmokeTests.Authentication == nil || again.SmokeTests.Authentication.Kind == nil ||
*again.SmokeTests.Authentication.Kind != "openrouter_key" || again.SmokeTests.Authentication.Endpoint == nil ||
*again.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
t.Fatalf("lookup exposed nested diagnostic metadata: %+v", again.SmokeTests)
} }
if _, exists := got.ExtraParams["added"]; exists {
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
}
gotNested["limit"] = 100
got.ExtraParams["added"] = true
again, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend again: %v", err)
}
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
}
if _, exists := again.ExtraParams["added"]; exists {
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
}
if _, err := registry.GetBackend("Custom"); err != nil {
t.Fatalf("backend IDs should be case-sensitive: %v", err)
}
policies := registry.CapacityPolicies() policies := registry.CapacityPolicies()
if len(policies) != 1 { if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
t.Fatalf("unexpected capacity policy count: %#v", policies) " local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
}
policies["custom"] = domain.BackendCapacityPolicy{}
delete(policies, "custom")
againPolicies := registry.CapacityPolicies()
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 3,
QueueCapacity: 2,
}) { }) {
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies) t.Fatalf("capacity policy = %+v", policies)
}
delete(policies, " local.v1 ")
if len(registry.CapacityPolicies()) != 1 {
t.Fatal("capacity policy map exposed registry state")
} }
} }
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) { func TestEffectiveRegistryMissingBackend(t *testing.T) {
maxInt := int(^uint(0) >> 1) registry, err := backend.NewEffectiveRegistry(nil)
tests := []struct {
name string
definition domain.Backend
want domain.BackendCapacityPolicy
wantSet bool
wantError bool
}{
{
name: "unlimited when omitted",
definition: domain.Backend{},
},
{
name: "default queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
QueueCapacity: 1024,
},
wantSet: true,
},
{
name: "explicit zero queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
QueueCapacitySet: true,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
},
wantSet: true,
},
{
name: "negative concurrency limit",
definition: domain.Backend{
ConcurrencyLimit: -1,
},
wantError: true,
},
{
name: "negative queue capacity",
definition: domain.Backend{
ConcurrencyLimit: 1,
QueueCapacity: -1,
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "queue without limit",
definition: domain.Backend{
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "total overflow",
definition: domain.Backend{
ConcurrencyLimit: maxInt,
QueueCapacity: 1,
QueueCapacitySet: true,
},
wantError: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
tc.definition.ID = "custom"
tc.definition.Endpoint = validEndpoint
registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
if tc.wantError {
if err == nil {
t.Fatal("expected invalid capacity policy error")
}
return
}
if err != nil {
t.Fatalf("construct registry: %v", err)
}
definition, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
definition.QueueCapacity != tc.want.QueueCapacity ||
definition.QueueCapacitySet != tc.wantSet {
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
definition.ConcurrencyLimit,
definition.QueueCapacity,
definition.QueueCapacitySet,
tc.want.ConcurrencyLimit,
tc.want.QueueCapacity,
tc.wantSet,
)
}
policies := registry.CapacityPolicies()
got, ok := policies["custom"]
if ok != tc.wantSet || got != tc.want {
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
}
})
}
}
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
tests := []struct {
name string
additions []domain.Backend
wantID string
}{
{
name: "OpenRouter collision after normalization",
additions: []domain.Backend{{ID: " openrouter "}},
wantID: backend.OpenRouterID,
},
{
name: "Rakestrawhome collision after normalization",
additions: []domain.Backend{{ID: " rakestrawhome "}},
wantID: backend.RakestrawHomeID,
},
{
name: "consumer collision after normalization",
additions: []domain.Backend{
{ID: "custom", Endpoint: validEndpoint},
{ID: " custom ", Endpoint: "https://other.example/v1"},
},
wantID: "custom",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, tc.additions)
if err == nil {
t.Fatal("expected duplicate ID error")
}
if !strings.Contains(err.Error(), tc.wantID) {
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
}
})
}
}
func TestNewRegistryValidatesIDs(t *testing.T) {
for _, id := range []string{"", " \t\n "} {
t.Run(id, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: id,
Endpoint: validEndpoint,
}})
if err == nil {
t.Fatal("expected blank ID error")
}
})
}
}
func TestNewRegistryValidatesEndpoints(t *testing.T) {
tests := []struct {
name string
endpoint string
}{
{name: "blank", endpoint: ""},
{name: "relative", endpoint: "/v1"},
{name: "missing host", endpoint: "https:///v1"},
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
{name: "user information", endpoint: "https://user@backend.example/v1"},
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
{name: "empty query", endpoint: "https://backend.example/v1?"},
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom",
Endpoint: tc.endpoint,
}})
if err == nil {
t.Fatal("expected invalid endpoint error")
}
})
}
}
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
t.Run(name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
APIKeyEnv: name,
}})
if err == nil {
t.Fatal("expected invalid environment-variable name error")
}
})
}
}
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
}{
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
ExtraParams: tc.extraParams,
}})
if err == nil {
t.Fatal("expected invalid extra parameters error")
}
})
}
}
func TestRegistryLookupReportsNotFound(t *testing.T) {
registry, err := backend.NewRegistry(nil, nil)
if err != nil { if err != nil {
t.Fatalf("construct registry: %v", err) t.Fatal(err)
} }
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
_, err = registry.GetBackend("missing") t.Fatalf("missing backend error = %v", err)
if !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("expected ErrBackendNotFound, got %v", err)
} }
if !strings.Contains(err.Error(), "missing") { if _, err := registry.GetBackend("local.v1"); !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("expected error to identify backend, got %v", err) t.Fatalf("trimmed lookup unexpectedly matched: %v", err)
} }
} }

View File

@@ -3,8 +3,8 @@ package capacity
import ( import (
"context" "context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
) )
type client struct { type client struct {

View File

@@ -10,8 +10,8 @@ import (
"testing" "testing"
"time" "time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
) )
type generateResult struct { type generateResult struct {

View File

@@ -10,7 +10,7 @@ import (
"strings" "strings"
"sync" "sync"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
// ErrCapacityExceeded identifies an admission rejected because a backend's // ErrCapacityExceeded identifies an admission rejected because a backend's

View File

@@ -5,7 +5,7 @@ import (
"errors" "errors"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
func TestNewManagerRejectsInvalidPolicies(t *testing.T) { func TestNewManagerRejectsInvalidPolicies(t *testing.T) {

View File

@@ -1,249 +0,0 @@
// Package catalog validates immutable maintained backend catalog assets.
package catalog
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"path"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
// Source identifies one immutable backend catalog asset tree.
type Source struct {
Name string
ExpectedBackendID string
FS fs.FS
Root string
}
// Set is the validated maintained backend and raw profile catalog.
type Set struct {
Backends []domain.Backend
Profiles profile.Repository
profileIDs []string
}
// Load validates and combines immutable catalog sources in source order.
func Load(sources ...Source) (Set, error) {
if len(sources) == 0 {
return Set{}, errors.New("at least one catalog source is required")
}
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
names := map[string]bool{}
backendIDs := map[string]bool{}
profileIDs := map[string]bool{}
for _, source := range sources {
if err := validateSource(source, names); err != nil {
return Set{}, err
}
names[source.Name] = true
definition, err := loadBackend(source)
if err != nil {
return Set{}, err
}
if backendIDs[definition.ID] {
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
}
backendIDs[definition.ID] = true
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
}
if len(metadata) == 0 {
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
}
resolvingRepository := profile.NewResolvingRepository(repository)
for _, entry := range metadata {
if strings.HasPrefix(entry.ID, "promptkit-") {
return Set{}, fmt.Errorf("catalog %s: %s uses the reserved promptkit- profile ID prefix", source.Name, entry.Path)
}
if profileIDs[entry.ID] {
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
}
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
}
value, err := repository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
}
if err := rejectSecretKeys(value.ExtraParams); err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
}
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
}
if resolved.BackendID != definition.ID {
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
}
profileIDs[entry.ID] = true
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
}
if loaded.Profiles == nil {
loaded.Profiles = repository
} else {
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
}
loaded.Backends = append(loaded.Backends, definition)
}
sort.Strings(loaded.profileIDs)
return loaded, nil
}
func validateSource(source Source, names map[string]bool) error {
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
return errors.New("catalog source name must not be blank")
}
if names[source.Name] {
return fmt.Errorf("duplicate catalog source name %q", source.Name)
}
if source.FS == nil {
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
}
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
}
if strings.TrimSpace(source.ExpectedBackendID) == "" {
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
}
return validateLayout(source)
}
func validateLayout(source Source) error {
manifestPath := path.Join(source.Root, "backend.json")
profilesRoot := path.Join(source.Root, "profiles")
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if assetPath == source.Root {
if !entry.IsDir() {
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
return nil
}
if entry.IsDir() {
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
if assetPath == manifestPath && entry.Type().IsRegular() {
return nil
}
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
})
}
func loadBackend(source Source) (domain.Backend, error) {
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
}
type manifest struct {
SchemaVersion *int `json:"schema_version"`
ID *string `json:"id"`
Endpoint *string `json:"endpoint"`
APIKeyEnv *string `json:"api_key_env"`
ConcurrencyLimit *int `json:"concurrency_limit"`
QueueCapacity *int `json:"queue_capacity"`
ExtraParams json.RawMessage `json:"extra_params"`
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var value manifest
if err := decoder.Decode(&value); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
}
if *value.ID != source.ExpectedBackendID {
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
}
if strings.TrimSpace(*value.APIKeyEnv) == "" {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
}
extraParams, err := decodeExtraParams(value.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
}
if err := rejectSecretKeys(extraParams); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
}
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
}
return normalized, nil
}
func decodeExtraParams(data []byte) (map[string]any, error) {
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
return nil, nil
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
var value map[string]any
if err := decoder.Decode(&value); err != nil || value == nil {
return nil, errors.New("extra parameters must be an object or null")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return nil, errors.New("extra parameters must contain exactly one JSON value")
}
return value, nil
}
func containsField(fields []string, target string) bool {
for _, field := range fields {
if field == target {
return true
}
}
return false
}
func rejectSecretKeys(value map[string]any) error {
return rejectSecretValue(value)
}
func rejectSecretValue(value any) error {
switch value := value.(type) {
case map[string]any:
for key, child := range value {
switch strings.ToLower(key) {
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
return errors.New("prohibited key")
}
if err := rejectSecretValue(child); err != nil {
return err
}
}
case []any:
for _, child := range value {
if err := rejectSecretValue(child); err != nil {
return err
}
}
}
return nil
}

View File

@@ -1,356 +0,0 @@
package catalog
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"strings"
"testing"
"testing/fstest"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
)
func TestLoadPublishedCatalogs(t *testing.T) {
loaded, err := Load(
Source{Name: "OpenRouter", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
Source{Name: "Rakestrawhome", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
)
if err != nil {
t.Fatalf("load published catalogs: %v", err)
}
if loaded.Profiles == nil || len(loaded.profileIDs) == 0 {
t.Fatalf("published catalogs have no usable profiles: %#v", loaded)
}
if _, err := loaded.Profiles.GetProfile(context.Background(), "gemini-flash-latest"); err != nil {
t.Fatalf("load maintained standard-profile base: %v", err)
}
expectedBackends := map[string]struct {
endpoint string
apiKeyEnv string
concurrencyLimit int
queueCapacity int
}{
"openrouter": {
endpoint: "https://openrouter.ai/api/v1",
apiKeyEnv: "OPENROUTER_API_KEY",
concurrencyLimit: 16,
queueCapacity: 1024,
},
"rakestrawhome": {
endpoint: "https://inference.ai.rakestrawhome.com/v1",
apiKeyEnv: "RAKESTRAWHOME_INFERENCE_API_KEY",
concurrencyLimit: 4,
queueCapacity: 1024,
},
}
if len(loaded.Backends) != len(expectedBackends) {
t.Fatalf("published backend count = %d, want %d", len(loaded.Backends), len(expectedBackends))
}
for _, definition := range loaded.Backends {
expected, found := expectedBackends[definition.ID]
if !found {
t.Fatalf("unexpected published backend %q", definition.ID)
}
if definition.Endpoint != expected.endpoint ||
definition.APIKeyEnv != expected.apiKeyEnv ||
definition.ConcurrencyLimit != expected.concurrencyLimit ||
definition.QueueCapacity != expected.queueCapacity ||
!definition.QueueCapacitySet ||
definition.ExtraParams != nil {
t.Fatalf("unexpected published backend definition: %#v", definition)
}
}
}
func TestLoadRejectsInvalidSources(t *testing.T) {
for name, sources := range map[string][]Source{
"none": nil,
"blank name": {{Name: " ", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root}},
"duplicate name": {
{Name: "same", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: openrouter.Root},
{Name: "same", ExpectedBackendID: "rakestrawhome", FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
},
"nil filesystem": {{Name: "missing", ExpectedBackendID: "openrouter", Root: openrouter.Root}},
"invalid root": {{Name: "invalid-root", ExpectedBackendID: "openrouter", FS: openrouter.FS(), Root: "."}},
"blank expected backend": {{Name: "blank-backend", ExpectedBackendID: " ", FS: openrouter.FS(), Root: openrouter.Root}},
} {
t.Run(name, func(t *testing.T) {
if _, err := Load(sources...); err == nil {
t.Fatal("expected error")
}
})
}
}
func TestLoadRejectsInvalidLayouts(t *testing.T) {
tests := map[string]func(fstest.MapFS){
"unexpected file": func(fsys fstest.MapFS) {
fsys["catalog/notes.txt"] = &fstest.MapFile{Data: []byte("unexpected")}
},
"unexpected directory": func(fsys fstest.MapFS) {
fsys["catalog/unexpected"] = &fstest.MapFile{Mode: fs.ModeDir}
},
"nonregular manifest": func(fsys fstest.MapFS) {
fsys["catalog/backend.json"].Mode = fs.ModeSymlink
},
"wrong profile extension": func(fsys fstest.MapFS) {
fsys["catalog/profiles/extra.yaml"] = &fstest.MapFile{Data: []byte(validProfile("extra", "one"))}
},
"nonregular profile": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Mode = fs.ModeSymlink
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
mutate(fsys)
if _, err := Load(testSource("one", fsys)); err == nil {
t.Fatal("expected invalid layout error")
}
})
}
}
func TestLoadRejectsInvalidManifests(t *testing.T) {
tests := map[string]string{
"malformed": `{`,
"trailing value": validManifest("one", "TEST_API_KEY", "null") + `{}`,
"missing fields": `{"schema_version":1,"id":"one"}`,
"unsupported version": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"schema_version":1`, `"schema_version":2`, 1),
"unknown field": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"extra_params":null`, `"extra_params":null,"unknown":true`, 1),
"blank API key env": validManifest("one", " ", "null"),
"invalid API key env": validManifest("one", "LEAK-MARKER", "null"),
"invalid endpoint": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `https://one.example/v1`, `ftp://leak-marker.invalid/v1`, 1),
"zero concurrency": strings.Replace(validManifest("one", "TEST_API_KEY", "null"), `"concurrency_limit":2`, `"concurrency_limit":0`, 1),
"non-object parameters": validManifest("one", "TEST_API_KEY", `[]`),
"secret parameter": validManifest("one", "TEST_API_KEY", `{"nested":{"token":"leak-marker"}}`),
}
for name, manifest := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
fsys["catalog/backend.json"].Data = []byte(manifest)
_, err := Load(testSource("one", fsys))
if err == nil {
t.Fatal("expected invalid manifest error")
}
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
t.Fatalf("catalog error exposed manifest content: %v", err)
}
})
}
}
func TestLoadPreservesManifestJSONNumbers(t *testing.T) {
fsys := validCatalogFS("one")
fsys["catalog/backend.json"].Data = []byte(validManifest(
"one",
"TEST_API_KEY",
`{"large":9007199254740993,"nested":[1.25]}`,
))
loaded, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("load catalog: %v", err)
}
if got := loaded.Backends[0].ExtraParams["large"]; got != json.Number("9007199254740993") {
t.Fatalf("large JSON integer = %#v, want preserved json.Number", got)
}
nested := loaded.Backends[0].ExtraParams["nested"].([]any)
if nested[0] != json.Number("1.25") {
t.Fatalf("nested JSON number = %#v, want preserved json.Number", nested[0])
}
}
func TestLoadRejectsInvalidCatalogProfiles(t *testing.T) {
tests := map[string]func(fstest.MapFS){
"empty": func(fsys fstest.MapFS) {
delete(fsys, "catalog/profiles/one-profile.yml")
fsys["catalog/profiles"] = &fstest.MapFile{Mode: fs.ModeDir}
},
"malformed": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: [")
},
"raw API key": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key: leak-marker\n")
},
"endpoint field": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "endpoint: ''\n")
},
"API key environment field": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "api_key_env: ''\n")
},
"owner mismatch": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "other"))
},
"missing base": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: leak-marker\n")
},
"cyclic base": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte("id: one-profile\nbase_profile: second\n")
fsys["catalog/profiles/second.yml"] = &fstest.MapFile{Data: []byte("id: second\nbase_profile: one-profile\n")}
},
"secret profile parameter": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "extra_params:\n nested:\n password: leak-marker\n")
},
"unknown field is redacted": func(fsys fstest.MapFS) {
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile("one-profile", "one") + "leak_marker: leak-marker\n")
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
mutate(fsys)
_, err := Load(testSource("one", fsys))
if err == nil {
t.Fatal("expected invalid profile error")
}
if strings.Contains(strings.ToLower(err.Error()), "leak-marker") {
t.Fatalf("catalog error exposed profile content: %v", err)
}
})
}
}
func TestLoadReservesPromptkitProfilePrefixForCatalogs(t *testing.T) {
tests := map[string]struct {
profileID string
wantError bool
}{
"maintained name": {
profileID: "promptkit-openrouter-standard",
wantError: true,
},
"other reserved name": {
profileID: "promptkit-example",
wantError: true,
},
"ordinary name": {
profileID: "ordinary-profile",
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
fsys := validCatalogFS("one")
fsys["catalog/profiles/one-profile.yml"].Data = []byte(validProfile(test.profileID, "one"))
_, err := Load(testSource("one", fsys))
if (err != nil) != test.wantError {
t.Fatalf("Load() error = %v, want error %t", err, test.wantError)
}
if err != nil && !strings.Contains(err.Error(), "catalog one: one-profile.yml") {
t.Fatalf("Load() error = %v, want catalog source and path context", err)
}
})
}
}
func TestLoadRejectsCrossCatalogConflicts(t *testing.T) {
t.Run("duplicate backend", func(t *testing.T) {
second := catalogFS("one", map[string]string{
"catalog/profiles/second.yml": validProfile("second", "one"),
}, "null")
_, err := Load(
Source{Name: "first", ExpectedBackendID: "one", FS: validCatalogFS("one"), Root: "catalog"},
Source{Name: "second", ExpectedBackendID: "one", FS: second, Root: "catalog"},
)
if err == nil {
t.Fatal("expected duplicate backend error")
}
})
t.Run("duplicate profile", func(t *testing.T) {
first := catalogFS("one", map[string]string{
"catalog/profiles/shared.yml": validProfile("shared", "one"),
}, "null")
second := catalogFS("two", map[string]string{
"catalog/profiles/shared.yml": validProfile("shared", "two"),
}, "null")
_, err := Load(testSource("one", first), testSource("two", second))
if err == nil {
t.Fatal("expected duplicate profile error")
}
})
t.Run("cross-catalog base", func(t *testing.T) {
first := catalogFS("one", map[string]string{
"catalog/profiles/base.yml": validProfile("base", "one"),
}, "null")
second := catalogFS("two", map[string]string{
"catalog/profiles/child.yml": "id: child\nbase_profile: base\n",
}, "null")
_, err := Load(testSource("one", first), testSource("two", second))
if err == nil {
t.Fatal("expected cross-catalog base error")
}
})
}
func TestLoadReturnsDefensiveCatalogValues(t *testing.T) {
fsys := catalogFS("one", map[string]string{
"catalog/profiles/one-profile.yml": validProfile("one-profile", "one") + "extra_params:\n nested:\n value: profile\n",
}, `{"nested":{"value":"backend"}}`)
loaded, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("load catalog: %v", err)
}
loaded.Backends[0].ExtraParams["nested"].(map[string]any)["value"] = "changed"
profileValue, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
if err != nil {
t.Fatalf("load profile: %v", err)
}
profileValue.ExtraParams["nested"].(map[string]any)["value"] = "changed"
again, err := Load(testSource("one", fsys))
if err != nil {
t.Fatalf("reload catalog: %v", err)
}
if got := again.Backends[0].ExtraParams["nested"].(map[string]any)["value"]; got != "backend" {
t.Fatalf("backend mutation escaped returned set: %#v", got)
}
againProfile, err := loaded.Profiles.GetProfile(context.Background(), "one-profile")
if err != nil {
t.Fatalf("reload profile: %v", err)
}
if got := againProfile.ExtraParams["nested"].(map[string]any)["value"]; got != "profile" {
t.Fatalf("profile mutation escaped returned value: %#v", got)
}
}
func testSource(id string, fsys fs.FS) Source {
return Source{Name: id, ExpectedBackendID: id, FS: fsys, Root: "catalog"}
}
func validCatalogFS(id string) fstest.MapFS {
return catalogFS(id, map[string]string{
"catalog/profiles/" + id + "-profile.yml": validProfile(id+"-profile", id),
}, "null")
}
func catalogFS(id string, profiles map[string]string, extraParams string) fstest.MapFS {
fsys := fstest.MapFS{
"catalog/backend.json": &fstest.MapFile{Data: []byte(validManifest(id, "TEST_API_KEY", extraParams))},
}
for name, content := range profiles {
fsys[name] = &fstest.MapFile{Data: []byte(content)}
}
return fsys
}
func validManifest(id, apiKeyEnv, extraParams string) string {
return fmt.Sprintf(
`{"schema_version":1,"id":%q,"endpoint":%q,"api_key_env":%q,"concurrency_limit":2,"queue_capacity":3,"extra_params":%s}`,
id,
"https://"+id+".example/v1",
apiKeyEnv,
extraParams,
)
}
func validProfile(id, backendID string) string {
return fmt.Sprintf("id: %s\nbackend: %s\nmodel: test-model\n", id, backendID)
}
var _ fs.FS = openrouter.FS()

View File

@@ -0,0 +1,343 @@
package configuration
import (
"context"
"fmt"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/domain"
)
const benchmarkDefinitionCount = 128
var (
constructionResult *loaded
selectedPrompt *domain.PromptDefinition
selectedPromptHash string
selectedPromptInspection *domain.PromptInspection
selectedProfile *domain.ExecutionProfile
selectedProfileIdentity string
)
type configurationShape struct {
main configkit.Input
underlays []configkit.Underlay
inputEntries int
effectiveEntries int
tiers int
}
func benchmarkPrompt(profileID string) map[string]any {
return map[string]any{
"default_profile": profileID,
"messages": []any{map[string]any{"role": "user", "content": "Hello {{.name}}"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
func benchmarkRecords(values map[string]any) []any {
records := make([]any, 0, len(values))
for _, id := range sortedKeys(values) {
value := values[id].(map[string]any)
record := make(map[string]any, len(value)+1)
for key, field := range value {
record[key] = field
}
record["id"] = id
records = append(records, record)
}
return records
}
func independentDefinitions(count int) configurationShape {
profiles := make(map[string]any, count)
prompts := make(map[string]any, count)
for index := range count {
profileID := fmt.Sprintf("profile-%03d", index)
promptID := fmt.Sprintf("prompt-%03d", index)
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
prompts[promptID] = benchmarkPrompt(profileID)
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)})},
inputEntries: count * 2,
effectiveEntries: count * 2,
tiers: 1,
}
}
func fileBackedDefinitions(count int) configurationShape {
profiles := make(map[string]any, count)
prompts := make(map[string]any, count)
for index := range count {
profileID := fmt.Sprintf("profile-%03d", index)
promptID := fmt.Sprintf("prompt-%03d", index)
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
prompt := benchmarkPrompt(profileID)
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
prompts[promptID] = prompt
}
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
})
return configurationShape{
main: configkit.Input{
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)}),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
},
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
}
}
func sharedBaseDefinitions(count int) configurationShape {
profiles := make(map[string]any, count+1)
profiles["base"] = map[string]any{
"endpoint": "https://example.test/v1", "model": "shared-model",
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
}
for index := range count {
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": map[string]any{"id": "base"}, "temperature": float64(index%10) / 10}
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
"profiles": benchmarkRecords(profiles),
"prompts": benchmarkRecords(map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}),
})},
inputEntries: count + 2,
effectiveEntries: count + 2,
tiers: 1,
}
}
func layeredDefinitions(count int) configurationShape {
base := make(map[string]any, count)
operator := make(map[string]any, count)
programmatic := make(map[string]any, count)
final := make(map[string]any, count)
for index := range count {
id := fmt.Sprintf("profile-%03d", index)
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
operator[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model"}
programmatic[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
final[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}, "temperature": float64(index%10) / 10}
}
return configurationShape{
underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": benchmarkRecords(base)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": benchmarkRecords(operator)})}}},
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": benchmarkRecords(programmatic)})}}},
},
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
"profiles": benchmarkRecords(final),
"prompts": benchmarkRecords(map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}),
})},
inputEntries: count*4 + 1,
effectiveEntries: count + 1,
tiers: 4,
}
}
func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
for _, count := range []int{4, 16, 64} {
b.Run(fmt.Sprintf("collections_%d", count), func(b *testing.B) {
collections := make(map[string]any, count)
for index := range count {
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
"prompts": []any{
map[string]any{"id": "example",
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
},
},
}
}
namespace := configkit.NewFilesystem("collection benchmark", fstest.MapFS{
"body.md": {Data: []byte("Hello")},
})
main := configkit.Input{
Source: configkit.Mapping("collections", map[string]any{"collections": collections}),
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "."},
}
b.ReportAllocs()
for b.Loop() {
result, err := load(main, nil)
if err != nil {
b.Fatal(err)
}
constructionResult = result
}
b.ReportMetric(float64(count), "collections/op")
})
}
}
func BenchmarkConfigurationConstruction(b *testing.B) {
for _, scenario := range []struct {
name string
shape configurationShape
}{
{name: "independent", shape: independentDefinitions(benchmarkDefinitionCount)},
{name: "file_backed", shape: fileBackedDefinitions(64)},
{name: "shared_base", shape: sharedBaseDefinitions(benchmarkDefinitionCount)},
{name: "layered", shape: layeredDefinitions(benchmarkDefinitionCount)},
} {
b.Run(scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
result, err := load(scenario.shape.main, scenario.shape.underlays)
if err != nil {
b.Fatal(err)
}
constructionResult = result
}
b.ReportMetric(float64(scenario.shape.inputEntries), "input-entries/op")
b.ReportMetric(float64(scenario.shape.effectiveEntries), "effective-entries/op")
b.ReportMetric(float64(scenario.shape.tiers), "tiers/op")
})
}
}
func BenchmarkConfigurationSelection(b *testing.B) {
shape := independentDefinitions(benchmarkDefinitionCount)
result, err := load(shape.main, nil)
if err != nil {
b.Fatal(err)
}
b.Run("prompt", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
definition, _, err := result.prompt("prompt-127")
if err != nil {
b.Fatal(err)
}
selectedPrompt = definition
}
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
})
b.Run("profile", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
profile, err := result.GetProfile(context.Background(), "profile-127")
if err != nil {
b.Fatal(err)
}
selectedProfile = profile
}
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
})
}
func inspectionExtraParams(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func inspectionDefinitions(extraParams map[string]any) configurationShape {
profiles := map[string]any{
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
}
prompt := benchmarkPrompt("small")
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
prompts := map[string]any{
"inspection": prompt,
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
"profiles": benchmarkRecords(profiles),
"prompts": benchmarkRecords(prompts),
})},
inputEntries: 2, effectiveEntries: 2, tiers: 1,
}
}
func BenchmarkInspectionSelection(b *testing.B) {
for _, scenario := range []struct {
name string
entries int
}{
{name: "small_extra", entries: 2},
{name: "nested_extra", entries: 64},
} {
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
result, err := load(shape.main, nil)
if err != nil {
b.Fatal(err)
}
ctx := context.Background()
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
if err != nil || definition.ID != "inspection" || hash == "" ||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
}
profile, err := result.GetProfile(ctx, "small")
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
}
inspection, err := result.GetPromptInspection(ctx, "inspection")
if err != nil || inspection.PromptID != "inspection" ||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
}
identity, err := result.GetProfileIdentity(ctx, "small")
if err != nil || identity != "small" {
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
}
b.Run("prompt/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
if err != nil {
b.Fatal(err)
}
selectedPrompt = definition
selectedPromptHash = hash
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
inspection, err := result.GetPromptInspection(ctx, "inspection")
if err != nil {
b.Fatal(err)
}
selectedPromptInspection = inspection
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("profile/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
profile, err := result.GetProfile(ctx, "small")
if err != nil {
b.Fatal(err)
}
selectedProfile = profile
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
identity, err := result.GetProfileIdentity(ctx, "small")
if err != nil {
b.Fatal(err)
}
selectedProfileIdentity = identity
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
}
}

View File

@@ -0,0 +1,82 @@
package configuration
import (
"io"
"strconv"
"strings"
"go.maximumdirect.net/configkit"
)
// Inspection is the read-only subset of the retained ConfigKit result used by
// the facade. It does not expose typed configuration or resource contexts.
type Inspection interface {
Metadata() configkit.Metadata
Sources() []configkit.SourceInfo
Diagnostics() []configkit.Diagnostic
Explain(configkit.Path) (configkit.Explanation, bool)
WriteEffective(io.Writer) error
WriteExplanation(io.Writer, configkit.Path) error
capturedResult() *configkit.Result[document]
}
type inspectionResult struct {
*configkit.Result[document]
}
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
return inspection.Result
}
// Inspection returns only the retained result's inspection operations.
func (state *loaded) Inspection() Inspection {
if state == nil {
return nil
}
return inspectionResult{Result: state.result}
}
// Diff compares two captured ConfigKit results without reconstructing them.
func Diff(left, right Inspection) ([]configkit.Difference, error) {
var leftResult, rightResult *configkit.Result[document]
if left != nil {
leftResult = left.capturedResult()
}
if right != nil {
rightResult = right.capturedResult()
}
return configkit.Diff(leftResult, rightResult)
}
func secretProviderPath(path configkit.Path) bool {
pointer := path.String()
if pointer == "" {
return false
}
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
if !providerExtraParamsPath(parts) {
return false
}
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
switch key {
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
return true
default:
return false
}
}
func providerExtraParamsPath(parts []string) bool {
if len(parts) >= 4 && sequenceIndex(parts[1]) &&
(parts[0] == "backends" || parts[0] == "profiles") && parts[2] == "extra_params" {
return true
}
return len(parts) >= 6 && parts[0] == "collections" && parts[1] != "" &&
(parts[2] == "backends" || parts[2] == "profiles") &&
sequenceIndex(parts[3]) && parts[4] == "extra_params"
}
func sequenceIndex(segment string) bool {
index, err := strconv.Atoi(segment)
return err == nil && index >= 0 && strconv.Itoa(index) == segment
}

View File

@@ -0,0 +1,32 @@
package configuration
import (
"testing"
"go.maximumdirect.net/configkit"
)
func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) {
tests := []struct {
name string
path configkit.Path
want bool
}{
{name: "backend nested secret", path: configkit.Keys("backends").Index(0).Key("extra_params").Key("nested").Key("Token"), want: true},
{name: "profile nested secret", path: configkit.Keys("profiles").Index(1).Key("extra_params").Key("authorization"), want: true},
{name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends").Index(2).Key("extra_params").Key("nested").Key("Token"), want: true},
{name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles").Index(0).Key("extra_params").Key("authorization"), want: true},
{name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false},
{name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false},
{name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
{name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false},
{name: "ordinary provider value", path: configkit.Keys("profiles").Index(0).Key("extra_params").Key("provider").Key("mode"), want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := secretProviderPath(test.path); got != test.want {
t.Fatalf("secretProviderPath(%q) = %t, want %t", test.path, got, test.want)
}
})
}
}

View File

@@ -0,0 +1,72 @@
package configuration
import (
"context"
"strings"
"testing"
"go.maximumdirect.net/configkit"
)
func TestTargetInterpretationPreservesComposedNestedValues(t *testing.T) {
state, err := load(mapping("padded target", map[string]any{
"backends": []any{map[string]any{
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 1,
"smoke_tests": map[string]any{
"reachability": map[string]any{"endpoint": " https://example.invalid/health "},
"authentication": map[string]any{"kind": " NONE ", "endpoint": " https://example.invalid/key "},
},
}},
"profiles": []any{map[string]any{"id": "partial", "backend": "local", "service_tier": " standard ", "api_key_env": " TOKEN "}},
}), nil)
if err != nil {
t.Fatal(err)
}
backend, err := state.GetBackend("local")
if err != nil || backend.Endpoint != "https://example.invalid/v1" || backend.QueueCapacity != 1024 ||
backend.SmokeTests == nil || *backend.SmokeTests.Reachability.Endpoint != "https://example.invalid/health" ||
*backend.SmokeTests.Authentication.Kind != "none" || *backend.SmokeTests.Authentication.Endpoint != "https://example.invalid/key" {
t.Fatalf("interpreted backend = %+v, %v", backend, err)
}
profile, err := state.GetProfile(context.Background(), "partial")
if err != nil || profile.ResolvedTarget.ServiceTier != "standard" || profile.ResolvedTarget.APIKeyEnv != "TOKEN" {
t.Fatalf("interpreted profile = %+v, %v", profile, err)
}
composed := state.result.Config.Backends[0]
if *composed.Endpoint != " HTTPS://example.invalid/v1 " || composed.QueueCapacity != nil ||
*composed.SmokeTests.Reachability.Endpoint != " https://example.invalid/health " ||
*composed.SmokeTests.Authentication.Kind != " NONE " || *composed.SmokeTests.Authentication.Endpoint != " https://example.invalid/key " ||
*state.result.Config.Profiles[0].ServiceTier != " standard " || *state.result.Config.Profiles[0].APIKeyEnv != " TOKEN " {
t.Fatalf("composed values were mutated: %+v", state.result.Config)
}
path := configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint")
field, found := state.result.Metadata().Lookup(path)
if !found || len(field.Origin.Locations) == 0 || field.Origin.Operation == configkit.ChangeNormalization {
t.Fatalf("nested field origin changed: %+v", field)
}
}
func TestPromptInterpretationPreservesComposedOutputAndRejectsDuplicateNames(t *testing.T) {
definition := map[string]any{
"id": "example", "inputs": []any{map[string]any{"name": " document "}},
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": " schema.json "},
}
state, err := load(mapping("prompt metadata", map[string]any{"prompts": []any{definition}}), nil)
if err != nil {
t.Fatal(err)
}
selected, inspection, err := state.prompt("example")
if err != nil || selected.Validation.SchemaPath != "schema.json" || inspection.OutputContract.SchemaPath != "schema.json" || selected.Inputs[0].Name != "document" {
t.Fatalf("interpreted prompt = %+v, %+v, %v", selected, inspection, err)
}
composed := state.result.Config.Prompts[0]
if *composed.Output.SchemaPath != " schema.json " || composed.Inputs[0].Name != " document " {
t.Fatalf("composed prompt changed: %+v", composed)
}
definition["inputs"] = []any{map[string]any{"name": " document "}, map[string]any{"name": "document"}}
_, err = load(mapping("duplicate inputs", map[string]any{"prompts": []any{definition}}), nil)
if err == nil || !strings.Contains(err.Error(), "/prompts/0/inputs/1") || !strings.Contains(err.Error(), "duplicated") {
t.Fatalf("duplicate normalized names were not rejected: %v", err)
}
}

View File

@@ -0,0 +1,84 @@
package configuration
import (
"context"
"go.maximumdirect.net/promptkit/internal/domain"
)
// CollectionIdentity is the captured metadata for one final collection entry.
type CollectionIdentity struct {
Namespace string
ReleaseVersion *string
}
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
profiles := append([]string(nil), state.profileIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return profiles, nil
}
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
backends := append([]string(nil), state.backendIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return backends, nil
}
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
selected := kinds
if len(selected) == 0 {
selected = []string{"backend", "profile", "prompt"}
}
set := make(map[string]struct{})
for _, kind := range selected {
for _, namespace := range state.namespaces[kind] {
set[namespace] = struct{}{}
}
}
namespaces := sortedKeys(set)
if err := ctx.Err(); err != nil {
return nil, err
}
return namespaces, nil
}
func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
collections := make([]CollectionIdentity, len(state.collections))
for index, collection := range state.collections {
collections[index] = CollectionIdentity{
Namespace: collection.Namespace,
ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return collections, nil
}
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return prompts, nil
}

View File

@@ -0,0 +1,157 @@
package configuration
import (
"context"
"errors"
"reflect"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "http://localhost:8080/v1"}},
"collections": map[string]any{
"z": map[string]any{"profiles": []any{map[string]any{"id": "profile", "backend": "local", "model": "z-model"}}, "prompts": []any{listingPromptDefinition("prompt")}},
"partial": map[string]any{"profiles": []any{map[string]any{"id": "profile", "backend": "local"}}},
"a": map[string]any{"profiles": []any{map[string]any{"id": "profile", "backend": "local", "model": "a-model"}}, "prompts": []any{listingPromptDefinition("prompt")}},
" z": map[string]any{"profiles": []any{map[string]any{"id": "profile ", "backend": "local", "model": "spaced-model"}}, "prompts": []any{listingPromptDefinition("prompt ")}},
},
}), nil)
if err != nil {
t.Fatal(err)
}
backends, err := state.ListBackends(context.Background())
if err != nil {
t.Fatal(err)
}
wantBackends := []string{"local"}
if !reflect.DeepEqual(backends, wantBackends) {
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
wantProfiles := []string{" z/profile ", "a/profile", "partial/profile", "z/profile"}
if !reflect.DeepEqual(profiles, wantProfiles) {
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
}
prompts, err := state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []domain.PromptIdentity{
{ID: " z/prompt "},
{ID: "a/prompt"},
{ID: "z/prompt"},
}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
}
}
func TestLoadedListsAreIndependent(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.test/v1"}},
"profiles": []any{map[string]any{"id": "profile"}},
"prompts": []any{listingPromptDefinition("example")},
}), nil)
if err != nil {
t.Fatal(err)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
profiles = append(profiles, "caller profile")
profiles[0] = "changed profile"
backends, err := state.ListBackends(context.Background())
if err != nil {
t.Fatal(err)
}
backends[0] = "changed backend"
prompts, err := state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
prompts[0].ID = "changed prompt"
profiles, err = state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(profiles, []string{"profile"}) {
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
}
backends, err = state.ListBackends(context.Background())
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
}
prompts, err = state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []domain.PromptIdentity{{ID: "example"}}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
}
}
func TestLoadedListsEmptyInventory(t *testing.T) {
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
if err != nil {
t.Fatal(err)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil || len(profiles) != 0 {
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
}
backends, err := state.ListBackends(context.Background())
if err != nil || len(backends) != 0 {
t.Fatalf("empty backend listing = %#v, %v", backends, err)
}
prompts, err := state.ListPrompts(context.Background())
if err != nil || len(prompts) != 0 {
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
}
}
func TestLoadedListsHonorCanceledContext(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"profiles": []any{map[string]any{"id": "partial"}},
"prompts": []any{listingPromptDefinition("example")},
}), nil)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
profiles, err := state.ListProfiles(ctx)
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
}
backends, err := state.ListBackends(ctx)
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
}
prompts, err := state.ListPrompts(ctx)
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
}
}
func listingPromptDefinition(id string) map[string]any {
return map[string]any{
"id": id,
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}
}

View File

@@ -0,0 +1,611 @@
// Package configuration adapts ConfigKit's effective configuration to
// Promptkit's definition and execution boundaries.
package configuration
import (
"fmt"
"net/http"
"regexp"
"sort"
"strings"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/backend"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm"
)
type document struct {
Version *string `yaml:"version"`
Backends []backendConfig `yaml:"backends"`
Profiles []profileConfig `yaml:"profiles"`
Prompts []promptConfig `yaml:"prompts"`
Collections map[string]collectionConfig `yaml:"collections"`
}
type collectionConfig struct {
ReleaseVersion *string `yaml:"release_version"`
Backends []backendConfig `yaml:"backends"`
Profiles []profileConfig `yaml:"profiles"`
Prompts []promptConfig `yaml:"prompts"`
}
type backendConfig struct {
ID string `yaml:"id"`
Endpoint *string `yaml:"endpoint"`
APIKeyEnv *string `yaml:"api_key_env"`
ConcurrencyLimit *int `yaml:"concurrency_limit"`
QueueCapacity *int `yaml:"queue_capacity"`
ExtraParams map[string]any `yaml:"extra_params"`
SmokeTests *backendSmokeTestConfig `yaml:"smoke_tests"`
}
type backendSmokeTestConfig struct {
DefaultProfile *string `yaml:"default_profile"`
Reachability *reachabilityProbeConfig `yaml:"reachability"`
Authentication *authenticationProbeConfig `yaml:"authentication"`
}
type reachabilityProbeConfig struct {
Endpoint *string `yaml:"endpoint"`
}
type authenticationProbeConfig struct {
Kind *string `yaml:"kind"`
Endpoint *string `yaml:"endpoint"`
}
type profileConfig struct {
ID string `yaml:"id"`
Backend *string `yaml:"backend"`
Endpoint *string `yaml:"endpoint"`
Model *string `yaml:"model"`
Temperature *float64 `yaml:"temperature"`
TopP *float64 `yaml:"top_p"`
MaxTokens *int `yaml:"max_tokens"`
TimeoutSeconds *int `yaml:"timeout_seconds"`
ServiceTier *string `yaml:"service_tier"`
ReasoningEffort *string `yaml:"reasoning_effort"`
APIKeyEnv *string `yaml:"api_key_env"`
APIKeyRequired *bool `yaml:"api_key_required"`
ExtraParams map[string]any `yaml:"extra_params"`
}
type promptConfig struct {
ID string `yaml:"id"`
DefaultProfile *string `yaml:"default_profile"`
Description *string `yaml:"description"`
SessionID *string `yaml:"session_id"`
Inputs []promptInputConfig `yaml:"inputs"`
Messages []promptMessageConfig `yaml:"messages"`
Output *promptOutputConfig `yaml:"output"`
}
type promptInputConfig struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType *string `yaml:"content_type"`
Description *string `yaml:"description"`
}
type promptMessageConfig struct {
Role *string `yaml:"role"`
Content *string `yaml:"content"`
ContentFile *string `yaml:"content_file"`
CacheControl *cacheControlConfig `yaml:"cache_control"`
}
type cacheControlConfig struct {
Type *string `yaml:"type"`
TTL *string `yaml:"ttl"`
}
type promptOutputConfig struct {
Format *string `yaml:"format"`
ValidationMode *string `yaml:"validation_mode"`
SchemaPath *string `yaml:"schema_path"`
RepairAttempts int `yaml:"repair_attempts"`
}
// fieldState distinguishes omission, an explicit value, and an explicit null.
// Inherited values are explicit because ConfigKit has already expanded them.
type fieldState uint8
const (
fieldAbsent fieldState = iota
fieldValue
fieldNull
)
type loaded struct {
result *configkit.Result[document]
interpreted document
backends map[string]map[string]fieldState
profiles map[string]map[string]fieldState
promptMetadata map[string][]promptInputPresence
prompts map[string]*promptEntry
registry *backend.Registry
indexedProfiles map[string]domain.ExecutionProfile
backendIDs []string
profileIDs []string
promptIDs []domain.PromptIdentity
namespaces map[string][]string
collections []CollectionIdentity
}
type resourceBases struct {
backends configkit.Path
profiles configkit.Path
prompts configkit.Path
}
type resourceSet struct {
isCollection bool
namespace string
bases resourceBases
backends []backendConfig
profiles []profileConfig
prompts []promptConfig
}
func rootResourceBases() resourceBases {
return resourceBases{
backends: configkit.Keys("backends"),
profiles: configkit.Keys("profiles"),
prompts: configkit.Keys("prompts"),
}
}
func resourceSets(config *document) []resourceSet {
sets := make([]resourceSet, 0, len(config.Collections)+1)
sets = append(sets, resourceSet{
bases: rootResourceBases(), backends: config.Backends, profiles: config.Profiles, prompts: config.Prompts,
})
for _, namespace := range sortedKeys(config.Collections) {
collection := config.Collections[namespace]
base := configkit.Keys("collections", namespace)
sets = append(sets, resourceSet{
isCollection: true,
namespace: namespace,
bases: resourceBases{
backends: base.Key("backends"),
profiles: base.Key("profiles"),
prompts: base.Key("prompts"),
},
backends: collection.Backends,
profiles: collection.Profiles,
prompts: collection.Prompts,
})
}
return sets
}
func indexedResourceID(namespace, localKey string) string {
qualified, valid := domain.QualifyLocalResourceID(namespace, localKey)
if valid {
return qualified
}
if namespace == "" {
return localKey
}
return namespace + "/" + localKey
}
func indexedBackends(config *document) map[string]backendConfig {
backends := make(map[string]backendConfig)
for _, resources := range resourceSets(config) {
for _, definition := range resources.backends {
backends[indexedResourceID(resources.namespace, definition.ID)] = definition
}
}
return backends
}
func indexedProfiles(config *document) map[string]profileConfig {
profiles := make(map[string]profileConfig)
for _, resources := range resourceSets(config) {
for _, definition := range resources.profiles {
profiles[indexedResourceID(resources.namespace, definition.ID)] = definition
}
}
return profiles
}
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
// Load captures one immutable effective configuration for internal consumers.
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
return load(main, underlays)
}
func configurationDefinition(state *loaded) configkit.Definition[document] {
return configkit.Definition[document]{
SequenceIdentities: []configkit.SequenceIdentity{
configkit.SequenceIdentityAt(configkit.Keys("backends"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("profiles"), "id"),
configkit.SequenceIdentityAt(configkit.Keys("prompts"), "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("backends")}, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("profiles")}, "id"),
configkit.SequenceIdentityMatching(configkit.SequenceSelector{configkit.SequenceLiteralKey("collections"), configkit.SequenceAnyMappingKey(), configkit.SequenceLiteralKey("prompts")}, "id"),
},
Version: &configkit.VersionRule{
Path: configkit.Keys("version"),
Check: checkConfigurationVersion,
},
Sensitive: secretProviderPath,
Resolve: func(config *document, resolution *configkit.Resolution) error {
for _, resources := range resourceSets(config) {
captureFields(resources.backends, func(v backendConfig) string { return v.ID }, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
}
return nil
},
Validate: func(config *document, validation *configkit.Validation) {
state.interpreted = interpretConfiguration(config, validation)
validateEffective(&state.interpreted, validation)
},
}
}
// load performs one ConfigKit lifecycle over the selected source tiers.
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
state := &loaded{
backends: make(map[string]map[string]fieldState),
profiles: make(map[string]map[string]fieldState),
promptMetadata: make(map[string][]promptInputPresence),
}
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
Underlays: underlays,
SourceVersionMode: configkit.SourceVersionModeIfPresent,
Definition: configurationDefinition(state),
})
if err != nil {
return nil, fmt.Errorf("load configuration: %w", err)
}
state.result = result
if err := state.materializePrompts(); err != nil {
return nil, fmt.Errorf("load configuration: %w", err)
}
if err := state.buildExecutionProfiles(); err != nil {
return nil, fmt.Errorf("load configuration: %w", err)
}
state.backendIDs = sortedKeys(indexedBackends(&state.interpreted))
state.profileIDs = sortedKeys(state.indexedProfiles)
for _, id := range sortedKeys(state.prompts) {
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
}
state.namespaces = map[string][]string{
"backend": resourceNamespaces(state.backendIDs),
"profile": resourceNamespaces(state.profileIDs),
"prompt": promptNamespaces(state.promptIDs),
}
for _, namespace := range sortedKeys(result.Config.Collections) {
collection := result.Config.Collections[namespace]
state.collections = append(state.collections, CollectionIdentity{
Namespace: namespace, ReleaseVersion: stringPointerValue(collection.ReleaseVersion),
})
}
return state, nil
}
func resourceNamespaces(ids []string) []string {
namespaces := make(map[string]struct{})
for _, id := range ids {
if namespace, _, qualified := strings.Cut(id, "/"); qualified {
namespaces[namespace] = struct{}{}
}
}
return sortedKeys(namespaces)
}
func promptNamespaces(prompts []domain.PromptIdentity) []string {
ids := make([]string, len(prompts))
for index := range prompts {
ids[index] = prompts[index].ID
}
return resourceNamespaces(ids)
}
func checkConfigurationVersion(version configkit.Version, validation *configkit.Validation) error {
if !version.Present {
return nil
}
if version.Null || version.Text != "2" {
validation.Error(configkit.Keys("version"), "unsupported configuration specification version")
}
return nil
}
type promptInputPresence struct {
contentType fieldState
description fieldState
}
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
for promptIndex, prompt := range prompts {
inputs := prompt.Inputs
states := make([]promptInputPresence, len(inputs))
base := basePath.Index(promptIndex).Key("inputs")
for index := range inputs {
path := base.Index(index)
states[index] = promptInputPresence{
contentType: captureFieldState(path.Key("content_type"), resolution),
description: captureFieldState(path.Key("description"), resolution),
}
}
destination[indexedResourceID(namespace, prompt.ID)] = states
}
}
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
info, exists := resolution.EntryInfo(path)
switch {
case !exists || !info.Explicit:
return fieldAbsent
case info.Null:
return fieldNull
default:
return fieldValue
}
}
func captureFields[T any](entries []T, identity func(T) string, basePath configkit.Path, namespace string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
for index, entry := range entries {
states := make(map[string]fieldState, len(fields))
for _, field := range fields {
info, exists := resolution.EntryInfo(basePath.Index(index).Key(field))
switch {
case !exists || !info.Explicit:
states[field] = fieldAbsent
case info.Null:
states[field] = fieldNull
default:
states[field] = fieldValue
}
}
destination[indexedResourceID(namespace, identity(entry))] = states
}
}
func sortedKeys[T any](entries map[string]T) []string {
keys := make([]string, 0, len(entries))
for key := range entries {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func interpretConfiguration(config *document, validation *configkit.Validation) document {
interpreted := *config
interpreted.Backends = interpretBackends(config.Backends, configkit.Keys("backends"), validation)
interpreted.Profiles = interpretProfiles(config.Profiles, configkit.Keys("profiles"), validation)
interpreted.Prompts = interpretPrompts(config.Prompts)
if config.Collections != nil {
interpreted.Collections = make(map[string]collectionConfig, len(config.Collections))
for namespace, original := range config.Collections {
collection := original
base := configkit.Keys("collections", namespace)
collection.Backends = interpretBackends(original.Backends, base.Key("backends"), validation)
collection.Profiles = interpretProfiles(original.Profiles, base.Key("profiles"), validation)
collection.Prompts = interpretPrompts(original.Prompts)
interpreted.Collections[namespace] = collection
}
}
return interpreted
}
func interpretBackends(source []backendConfig, basePath configkit.Path, validation *configkit.Validation) []backendConfig {
if source == nil {
return nil
}
backends := append([]backendConfig{}, source...)
for index := range backends {
backend := &backends[index]
base := basePath.Index(index)
if backend.Endpoint != nil {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
if err != nil {
validation.Error(base.Key("endpoint"), err.Error())
} else {
backend.Endpoint = &endpoint
}
}
if backend.APIKeyEnv != nil {
value := strings.TrimSpace(*backend.APIKeyEnv)
backend.APIKeyEnv = &value
}
if backend.SmokeTests != nil {
smokeTests := *backend.SmokeTests
backend.SmokeTests = &smokeTests
smokePath := base.Key("smoke_tests")
if smokeTests.Reachability != nil {
reachability := *smokeTests.Reachability
smokeTests.Reachability = &reachability
normalizeDiagnosticEndpoint(&reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), validation)
}
if smokeTests.Authentication != nil {
authentication := *smokeTests.Authentication
smokeTests.Authentication = &authentication
if authentication.Kind != nil {
kind := strings.ToLower(strings.TrimSpace(*authentication.Kind))
authentication.Kind = &kind
}
normalizeDiagnosticEndpoint(&authentication.Endpoint, smokePath.Key("authentication").Key("endpoint"), validation)
}
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
capacity := 1024
backend.QueueCapacity = &capacity
}
}
return backends
}
func normalizeDiagnosticEndpoint(value **string, path configkit.Path, validation *configkit.Validation) {
if value == nil || *value == nil {
return
}
normalized := strings.TrimSpace(**value)
request, err := http.NewRequest(http.MethodGet, normalized, nil)
if normalized == "" || err != nil || request.URL == nil ||
(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
request.URL.Hostname() == "" {
validation.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
return
}
*value = &normalized
}
func interpretProfiles(source []profileConfig, basePath configkit.Path, validation *configkit.Validation) []profileConfig {
if source == nil {
return nil
}
profiles := append([]profileConfig{}, source...)
for index := range profiles {
profile := &profiles[index]
base := basePath.Index(index)
if profile.Endpoint != nil && *profile.Endpoint != "" {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
if err != nil {
validation.Error(base.Key("endpoint"), err.Error())
} else {
profile.Endpoint = &endpoint
}
}
for _, value := range []**string{&profile.ServiceTier, &profile.ReasoningEffort, &profile.APIKeyEnv} {
if *value != nil {
normalized := strings.TrimSpace(**value)
*value = &normalized
}
}
}
return profiles
}
func validateEffective(config *document, validation *configkit.Validation) {
sets := resourceSets(config)
backendIDs := make(map[string]struct{})
for _, resources := range sets {
if resources.isCollection && !domain.IsValidResourceNamespace(resources.namespace) {
validation.Error(configkit.Keys("collections", resources.namespace), "collection namespace must be nonblank and contain no slash")
}
for _, backend := range resources.backends {
backendIDs[indexedResourceID(resources.namespace, backend.ID)] = struct{}{}
}
}
for _, resources := range sets {
for index, backend := range resources.backends {
base := resources.bases.backends.Index(index)
if !domain.IsValidLocalResourceKey(backend.ID) {
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
}
if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required")
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
}
if backend.QueueCapacity != nil {
if *backend.QueueCapacity < 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
}
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
}
}
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
if smokeTests := backend.SmokeTests; smokeTests != nil {
smokePath := base.Key("smoke_tests")
if smokeTests.DefaultProfile != nil {
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
}
}
if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil {
validation.Error(kindPath, "authentication kind is required")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
}
}
}
}
for index, profile := range resources.profiles {
base := resources.bases.profiles.Index(index)
if !domain.IsValidLocalResourceKey(profile.ID) {
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
}
if profile.Backend != nil && *profile.Backend != "" {
if !domain.IsValidResourceID(*profile.Backend) {
validation.Error(base.Key("backend"), "backend reference is malformed")
} else if _, exists := backendIDs[*profile.Backend]; !exists {
validation.Error(base.Key("backend"), "backend is not defined")
}
}
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
validation.Error(base.Key("model"), "model must not be blank")
}
if profile.Temperature != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
validation.Error(base.Key("temperature"), err.Error())
}
}
if profile.TopP != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
validation.Error(base.Key("top_p"), err.Error())
}
}
if profile.MaxTokens != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
validation.Error(base.Key("max_tokens"), err.Error())
}
}
if profile.TimeoutSeconds != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
validation.Error(base.Key("timeout_seconds"), err.Error())
}
}
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
}
for index, prompt := range resources.prompts {
base := resources.bases.prompts.Index(index)
if !domain.IsValidLocalResourceKey(prompt.ID) {
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
}
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
}
}
}
}
func validateExtraParams(values map[string]any, path configkit.Path, backend bool, validation *configkit.Validation) {
if values == nil {
return
}
if _, err := jsonvalue.CopyMap(values); err != nil {
validation.Error(path, "extra parameters must be a JSON-compatible map with nonempty keys")
return
}
if backend {
for key := range values {
if llm.IsReservedOpenAIChatRequestField(key) {
validation.Error(path.Key(key), "extra parameter conflicts with a request field")
}
}
}
}

View File

@@ -0,0 +1,814 @@
package configuration
import (
"context"
"errors"
"strings"
"testing"
"go.maximumdirect.net/configkit"
)
func mapping(label string, values map[string]any) configkit.Input {
return configkit.Input{Source: configkit.Mapping(label, values)}
}
func TestLoadIdentifiedResourceLists(t *testing.T) {
for _, header := range []bool{false, true} {
values := map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1"}},
"collections": map[string]any{"team": map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1"}},
}},
}
if header {
values["version"] = "2"
}
loaded, err := load(mapping("identified resources", values), nil)
if err != nil {
t.Fatalf("load identified resources with header %t: %v", header, err)
}
if loaded.result.Config.Backends[0].ID != "local" || loaded.result.Config.Collections["team"].Backends[0].ID != "local" {
t.Fatal("identified root and collection resources were not retained")
}
}
}
func TestIdentifiedResourcesComposeAcrossNamespacesAndKinds(t *testing.T) {
resources := func() map[string]any {
return map[string]any{
"backends": []any{
map[string]any{"id": "shared", "endpoint": "https://example.invalid/v1"},
map[string]any{"id": "child", "extends": map[string]any{"id": "shared"}},
},
"profiles": []any{
map[string]any{"id": "shared", "model": "model"},
map[string]any{"id": "child", "extends": map[string]any{"id": "shared"}},
},
"prompts": []any{
map[string]any{"id": "shared", "messages": []any{map[string]any{"role": "user", "content": "inherited"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"}},
map[string]any{"id": "child", "extends": map[string]any{"id": "shared"}},
},
}
}
values := resources()
values["collections"] = map[string]any{"*": resources(), "123": resources()}
state, err := load(mapping("identified namespaces", values), nil)
if err != nil {
t.Fatal(err)
}
for _, prefix := range []string{"", "*/", "123/"} {
backend, err := state.registry.GetBackend(prefix + "child")
if err != nil || backend.ID != prefix+"child" || backend.Endpoint != "https://example.invalid/v1" {
t.Fatalf("inherited backend %q = %+v, %v", prefix, backend, err)
}
profile, err := state.GetProfileIdentity(context.Background(), prefix+"child")
if err != nil || profile != prefix+"child" {
t.Fatalf("inherited profile %q = %q, %v", prefix, profile, err)
}
definition, _, err := state.prompt(prefix + "child")
if err != nil || definition.ID != prefix+"child" || len(definition.Templates) != 1 || definition.Templates[0].Content != "inherited" {
t.Fatalf("inherited prompt %q = %+v, %v", prefix, definition, err)
}
}
}
func TestIdentifiedResourcesRejectInvalidInheritance(t *testing.T) {
for _, kind := range []string{"backends", "profiles", "prompts"} {
for name, records := range map[string][]any{
"missing": {map[string]any{"id": "child", "extends": map[string]any{"id": "missing"}}},
"self": {map[string]any{"id": "child", "extends": map[string]any{"id": "child"}}},
"cycle": {
map[string]any{"id": "first", "extends": map[string]any{"id": "second"}},
map[string]any{"id": "second", "extends": map[string]any{"id": "first"}},
},
} {
t.Run(kind+"/"+name, func(t *testing.T) {
_, err := load(mapping("invalid inheritance", map[string]any{kind: records}), nil)
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want an upstream inheritance diagnostic", err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Code == configkit.CodeInheritanceMissing || diagnostic.Code == configkit.CodeInheritanceCycle {
return
}
}
t.Fatalf("missing inheritance diagnostic: %+v", loadErr.Diagnostics())
})
}
}
}
func TestIdentifiedResourcesRejectInvalidLocalIDs(t *testing.T) {
for _, kind := range []string{"backends", "profiles", "prompts"} {
for _, test := range []struct {
name string
record map[string]any
}{
{name: "missing", record: map[string]any{}},
{name: "null", record: map[string]any{"id": nil}},
{name: "non-string", record: map[string]any{"id": 42}},
{name: "blank", record: map[string]any{"id": " "}},
{name: "slash", record: map[string]any{"id": "not/local"}},
} {
t.Run(kind+"/"+test.name, func(t *testing.T) {
_, err := load(mapping("invalid resource ID", map[string]any{kind: []any{test.record}}), nil)
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) || len(loadErr.Diagnostics()) == 0 {
t.Fatalf("load error = %v, want a path-aware error", err)
}
})
}
}
}
func TestIdentifiedResourcesRejectDuplicateStrictPeers(t *testing.T) {
for _, kind := range []string{"backends", "profiles", "prompts"} {
for _, namespace := range []string{"", "*"} {
t.Run(kind+"/"+namespace, func(t *testing.T) {
values := func() map[string]any {
records := []any{map[string]any{"id": "duplicate"}}
if namespace == "" {
return map[string]any{kind: records}
}
return map[string]any{"collections": map[string]any{namespace: map[string]any{kind: records}}}
}
_, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{
mapping("first", values()), mapping("second", values()),
}}})
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want an upstream duplicate diagnostic", err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Code == configkit.CodeComposeConflict {
return
}
}
t.Fatalf("missing duplicate conflict: %+v", loadErr.Diagnostics())
})
}
}
}
func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
for _, kind := range []string{"backends", "profiles", "prompts"} {
for _, namespace := range []string{"", "*"} {
t.Run(kind+"/"+namespace, func(t *testing.T) {
definition := configurationDefinition(&loaded{})
definition.Resolve = func(config *document, resolution *configkit.Resolution) error {
path := configkit.Keys(kind)
if namespace == "" {
switch kind {
case "backends":
config.Backends = []backendConfig{{ID: "same"}, {ID: "same"}}
case "profiles":
config.Profiles = []profileConfig{{ID: "same"}, {ID: "same"}}
case "prompts":
config.Prompts = []promptConfig{{ID: "same"}, {ID: "same"}}
}
} else {
collection := collectionConfig{}
switch kind {
case "backends":
collection.Backends = []backendConfig{{ID: "same"}, {ID: "same"}}
case "profiles":
collection.Profiles = []profileConfig{{ID: "same"}, {ID: "same"}}
case "prompts":
collection.Prompts = []promptConfig{{ID: "same"}, {ID: "same"}}
}
config.Collections = map[string]collectionConfig{namespace: collection}
path = configkit.Keys("collections", namespace, kind)
}
resolution.RecordNormalization(path, "supply resolved records")
return nil
}
input := configkit.Input{Source: configkit.NoSource()}
if namespace != "" {
input = mapping("existing collection", map[string]any{"collections": map[string]any{
namespace: map[string]any{kind: []any{}},
}})
}
_, err := configkit.LoadLayered(input, configkit.LayeredOptions[document]{Definition: definition})
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want a final identity diagnostic", err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Code == configkit.CodeSequenceConflict {
return
}
}
t.Fatalf("missing final identity conflict: %+v", loadErr.Diagnostics())
})
}
}
}
func TestIdentifiedResourceLayerOperations(t *testing.T) {
base := mapping("base profiles", map[string]any{"profiles": []any{
map[string]any{"id": "first", "model": "original", "service_tier": "standard"},
map[string]any{"id": "second", "model": "retained"},
}})
for _, test := range []struct {
name string
mode string
records []any
models map[string]string
}{
{name: "whole record upsert", records: []any{map[string]any{"id": "first", "model": "updated"}}, models: map[string]string{"first": "updated", "second": "retained"}},
{name: "empty upsert", records: []any{}, models: map[string]string{"first": "original", "second": "retained"}},
{name: "append", mode: "append", records: []any{map[string]any{"id": "third", "model": "added"}}, models: map[string]string{"first": "original", "second": "retained", "third": "added"}},
{name: "replace", mode: "replace", records: []any{map[string]any{"id": "third", "model": "replacement"}}, models: map[string]string{"third": "replacement"}},
{name: "clear", mode: "replace", records: []any{}, models: map[string]string{}},
} {
t.Run(test.name, func(t *testing.T) {
values := map[string]any{"profiles": test.records}
if test.mode != "" {
values["config"] = map[string]any{"sequences": []any{map[string]any{
"destination": []any{"profiles"}, "mode": test.mode,
}}}
}
state, err := load(mapping("overlay profiles", values), []configkit.Underlay{{Inputs: []configkit.Input{base}}})
if err != nil {
t.Fatal(err)
}
profiles := indexedProfiles(&state.result.Config)
if len(profiles) != len(test.models) {
t.Fatalf("profiles = %+v, want IDs and models %+v", profiles, test.models)
}
for id, model := range test.models {
profile, ok := profiles[id]
if !ok || profile.Model == nil || *profile.Model != model {
t.Fatalf("profile %q = %+v, want model %q", id, profile, model)
}
}
if test.name == "whole record upsert" && profiles["first"].ServiceTier != nil {
t.Fatalf("upsert retained a displaced field: %+v", profiles["first"])
}
})
}
}
func TestIdentifiedResourceLayersRejectDisplacedParentAndDuplicateAppend(t *testing.T) {
base := mapping("base", map[string]any{"profiles": []any{
map[string]any{"id": "shared", "model": "original"},
}})
for _, test := range []struct {
name string
values map[string]any
code configkit.Code
}{
{
name: "self extension cannot recover displaced record",
values: map[string]any{"profiles": []any{
map[string]any{"id": "shared", "extends": map[string]any{"id": "shared"}},
}},
code: configkit.CodeInheritanceCycle,
},
{
name: "append cannot duplicate an identity",
values: map[string]any{
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"profiles"}, "mode": "append"}}},
"profiles": []any{map[string]any{"id": "shared", "model": "other"}},
},
code: configkit.CodeSequenceConflict,
},
} {
t.Run(test.name, func(t *testing.T) {
_, err := load(mapping("overlay", test.values), []configkit.Underlay{{Inputs: []configkit.Input{base}}})
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want structured diagnostic", err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Code == test.code {
return
}
}
t.Fatalf("missing %s diagnostic: %+v", test.code, loadErr.Diagnostics())
})
}
}
func TestLoadChecksOptionalConfigurationVersion(t *testing.T) {
for _, test := range []struct {
name string
version any
include bool
valid bool
}{
{name: "string two", version: "2", include: true, valid: true},
{name: "integer two", version: int64(2), include: true, valid: true},
{name: "omitted", valid: true},
{name: "explicit null", include: true},
{name: "unsupported string", version: "1", include: true},
} {
t.Run(test.name, func(t *testing.T) {
values := map[string]any{}
if test.include {
values["version"] = test.version
}
loaded, err := load(mapping("main", values), nil)
if test.valid {
if err != nil {
t.Fatalf("load accepted version: %v", err)
}
if test.include && (loaded.result.Config.Version == nil || *loaded.result.Config.Version != "2") {
t.Fatalf("typed version = %v, want 2", loaded.result.Config.Version)
}
if !test.include && loaded.result.Config.Version != nil {
t.Fatalf("omitted version = %q, want nil", *loaded.result.Config.Version)
}
return
}
var loadErr *configkit.LoadError
if err == nil || !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want ConfigKit load error", err)
}
found := false
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Path.String() == "/version" && diagnostic.Message == "unsupported configuration specification version" {
found = true
}
}
if !found {
t.Fatalf("load diagnostics = %+v, want safe version diagnostic", loadErr.Diagnostics())
}
})
}
}
func TestLoadRejectsMalformedHeadersAndLegacyResourceMappings(t *testing.T) {
for name, values := range map[string]map[string]any{
"boolean header": {"version": true},
"fractional header": {"version": 2.5},
"sequence header": {"version": []any{2}},
"backend mapping": {"backends": map[string]any{"local": map[string]any{"endpoint": "https://example.invalid/v1"}}},
"profile mapping": {"profiles": map[string]any{"local": map[string]any{}}},
"prompt mapping": {"prompts": map[string]any{"local": map[string]any{}}},
} {
t.Run(name, func(t *testing.T) {
_, err := load(mapping(name, values), nil)
var loadErr *configkit.LoadError
if !errors.As(err, &loadErr) || len(loadErr.Diagnostics()) == 0 {
t.Fatalf("load error = %v, want structured rejection", err)
}
})
}
}
func TestNestedConfigRemainsApplicationData(t *testing.T) {
state, err := load(mapping("nested config", map[string]any{
"config": map[string]any{},
"profiles": []any{map[string]any{"id": "example", "extra_params": map[string]any{
"config": map[string]any{"imports": "provider data"},
}}},
}), nil)
if err != nil {
t.Fatal(err)
}
profile, err := state.GetProfile(context.Background(), "example")
if err != nil || profile.ResolvedTarget.ExtraParams["config"].(map[string]any)["imports"] != "provider data" {
t.Fatalf("nested application data = %+v, %v", profile, err)
}
}
func TestLoadConsumesAcceptedUnderlayVersionAndValidatesHeaderlessPayload(t *testing.T) {
underlay := mapping("declared underlay", map[string]any{
"version": int64(2),
})
loaded, err := load(mapping("headerless main", map[string]any{}), []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatalf("load accepted non-main declaration: %v", err)
}
if loaded.result.Config.Version != nil {
t.Fatalf("consumed non-main declaration remained in effective payload: %q", *loaded.result.Config.Version)
}
invalidFragment := mapping("headerless prompt fragment", map[string]any{
"prompts": []any{map[string]any{"id": "incomplete", "description": "still validated"}},
})
if _, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{invalidFragment}}}); err == nil || !strings.Contains(err.Error(), "at least one message is required") {
t.Fatalf("headerless fragment validation error = %v, want final prompt validation", err)
}
}
func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
base := mapping("defaults", map[string]any{
"version": int64(2),
"backends": []any{map[string]any{"id": "local.v1", "endpoint": "http://localhost:8080/v1", "concurrency_limit": 2}},
"collections": map[string]any{"base": map[string]any{
"release_version": "",
"backends": []any{map[string]any{"id": "local.v1", "endpoint": "http://localhost:8080/v1", "concurrency_limit": 3}},
"profiles": []any{
map[string]any{"id": "one", "backend": "local.v1", "temperature": 0.8, "top_p": 0.9, "api_key_required": true,
"extra_params": map[string]any{"provider": map[string]any{"first": 1, "second": 2}}},
map[string]any{"id": "child.v1", "extends": map[string]any{"id": "one"}},
map[string]any{"id": "namespaced", "backend": "base/local.v1", "model": "collection-model"},
},
}, "invoice": map[string]any{
"release_version": "r7",
"prompts": []any{
map[string]any{"id": "summary",
"messages": []any{map[string]any{"role": "user", "content": "Summarize."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
},
map[string]any{"id": "version",
"messages": []any{map[string]any{"role": "user", "content": "Nested ID."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
},
},
}},
})
overlay := mapping("operator", map[string]any{
"backends": []any{map[string]any{"id": "local.v1", "endpoint": "http://localhost:8080/v1", "concurrency_limit": 2, "queue_capacity": 0}},
"collections": map[string]any{"base": map[string]any{"profiles": []any{map[string]any{"id": "child.v1", "extends": map[string]any{"id": "one"},
"temperature": 0, "top_p": nil, "api_key_required": false,
"extra_params": map[string]any{"provider": map[string]any{"second": 3}},
}}}},
})
loaded, err := load(mapping("main", map[string]any{"version": "2"}), []configkit.Underlay{
{Inputs: []configkit.Input{base}}, {Inputs: []configkit.Input{overlay}},
})
if err != nil {
var loadErr *configkit.LoadError
if errors.As(err, &loadErr) {
t.Fatalf("load: %v; diagnostics: %+v", err, loadErr.Diagnostics())
}
t.Fatal(err)
}
profile := loaded.result.Config.Collections["base"].Profiles[1]
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
t.Fatalf("effective values were not retained: %+v", profile)
}
if loaded.profiles["base/child.v1"]["temperature"] != fieldValue || loaded.profiles["base/child.v1"]["top_p"] != fieldNull || loaded.profiles["base/child.v1"]["api_key_required"] != fieldValue {
t.Fatalf("effective presence was lost: %+v", loaded.profiles["base/child.v1"])
}
provider := profile.ExtraParams["provider"].(map[string]any)
if provider["first"] != int64(1) || provider["second"] != int64(3) {
t.Fatalf("recursive overlay failed: %+v", provider)
}
backend := loaded.result.Config.Backends[0]
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
}
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
t.Fatalf("collection backend was not indexed by its full ID: %v", err)
}
collectionProfile, exists := loaded.indexedProfiles["base/namespaced"]
if !exists || collectionProfile.ID != "base/namespaced" || collectionProfile.ResolvedTarget == nil || collectionProfile.ResolvedTarget.BackendID != "base/local.v1" {
t.Fatalf("collection profile snapshot = %+v, exists %t", collectionProfile, exists)
}
if loaded.result.Config.Collections["invoice"].Prompts[0].Messages[0].Content == nil {
t.Fatal("literal prompt content was not decoded")
}
if loaded.result.Config.Version == nil || *loaded.result.Config.Version != "2" ||
loaded.result.Config.Collections["base"].ReleaseVersion == nil || *loaded.result.Config.Collections["base"].ReleaseVersion != "" ||
loaded.result.Config.Collections["invoice"].ReleaseVersion == nil || *loaded.result.Config.Collections["invoice"].ReleaseVersion != "r7" {
t.Fatalf("root or collection version metadata was not retained: %+v", loaded.result.Config)
}
if _, _, err := loaded.prompt("invoice/version"); err != nil {
t.Fatalf("resource named version was not indexed under its qualified identity: %v", err)
}
}
func TestLoadRejectsInvalidCollectionAndResourceKeys(t *testing.T) {
for _, test := range []struct {
name string
values map[string]any
message string
}{
{
name: "namespace with separator",
values: map[string]any{"collections": map[string]any{"bad/name": map[string]any{}}},
message: "collection namespace must be nonblank and contain no slash",
},
{
name: "root resource with separator",
values: map[string]any{"backends": []any{
map[string]any{"id": "bad/name", "endpoint": "https://example.test/v1"},
}},
message: "backend ID must be a nonblank local key without a slash",
},
{
name: "collection resource with separator",
values: map[string]any{"collections": map[string]any{"good": map[string]any{
"prompts": []any{map[string]any{"id": "bad/name",
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}},
}}},
message: "prompt ID must be a nonblank local key without a slash",
},
} {
t.Run(test.name, func(t *testing.T) {
_, err := load(mapping("invalid names", test.values), nil)
var loadErr *configkit.LoadError
if err == nil || !errors.As(err, &loadErr) {
t.Fatalf("load error = %v, want ConfigKit validation error", err)
}
for _, diagnostic := range loadErr.Diagnostics() {
if diagnostic.Message == test.message {
return
}
}
t.Fatalf("load diagnostics = %+v, want message %q", loadErr.Diagnostics(), test.message)
})
}
}
func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
loaded, err := load(mapping("backend", map[string]any{"backends": []any{
map[string]any{"id": "local", "endpoint": " HTTP://localhost:8080/v1 ", "api_key_env": " TOKEN_NAME ",
"concurrency_limit": 2, "queue_capacity": nil},
}}), nil)
if err != nil {
t.Fatal(err)
}
backend, err := loaded.GetBackend("local")
if err != nil || backend.Endpoint != "http://localhost:8080/v1" ||
backend.APIKeyEnv != "TOKEN_NAME" || backend.QueueCapacity != 1024 {
t.Fatalf("backend normalization/default was not applied: %+v", backend)
}
composed := loaded.result.Config.Backends[0]
if composed.Endpoint == nil || *composed.Endpoint != " HTTP://localhost:8080/v1 " ||
composed.APIKeyEnv == nil || *composed.APIKeyEnv != " TOKEN_NAME " || composed.QueueCapacity != nil {
t.Fatalf("composed backend was changed: %+v", composed)
}
if loaded.backends["local"]["queue_capacity"] != fieldNull {
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
}
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
if !found || field.Origin.Operation == configkit.ChangeNormalization || len(field.Origin.Locations) == 0 {
t.Fatalf("endpoint origin was not preserved: %+v", field)
}
}
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
for _, test := range []struct {
name string
smokeTests any
include bool
inherited bool
}{
{name: "omitted"},
{name: "null", smokeTests: nil, include: true, inherited: true},
} {
t.Run(test.name, func(t *testing.T) {
backend := map[string]any{"id": "local", "endpoint": "https://backend.example/v1"}
if test.include {
backend["smoke_tests"] = test.smokeTests
}
main := mapping("backend", map[string]any{
"backends": []any{backend},
})
var underlays []configkit.Underlay
if test.inherited {
underlays = []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
"backends": []any{map[string]any{"id": "local",
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{"default_profile": "inherited-profile"},
}},
})}}}
}
loaded, err := load(main, underlays)
if err != nil {
t.Fatal(err)
}
if got := loaded.result.Config.Backends[0].SmokeTests; got != nil {
t.Fatalf("smoke_tests = %+v, want nil", got)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
if selected.SmokeTests != nil {
t.Fatalf("indexed smoke_tests = %+v, want nil", selected.SmokeTests)
}
})
}
}
func TestLoadMergesBackendSmokeTestsAndRetainsFallbacks(t *testing.T) {
base := mapping("defaults", map[string]any{
"backends": []any{map[string]any{"id": "catalog",
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": "catalog-profile",
"reachability": map[string]any{"endpoint": " https://probe.example/ready?check=1 "},
"authentication": map[string]any{
"kind": " OPENROUTER_KEY ",
"endpoint": "https://probe.example/key?scope=current",
},
},
}},
})
overlay := mapping("operator", map[string]any{
"backends": []any{map[string]any{"id": "local", "extends": map[string]any{"id": "catalog"},
"smoke_tests": map[string]any{
"default_profile": "not-yet-listed",
"authentication": map[string]any{"kind": " NONE ", "endpoint": nil},
},
}},
})
loaded, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{
{Inputs: []configkit.Input{base}},
{Inputs: []configkit.Input{overlay}},
})
if err != nil {
t.Fatalf("load effective diagnostic metadata: %v", err)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
metadata := selected.SmokeTests
if selected.Endpoint != "https://backend.example/v1" {
t.Fatalf("effective backend endpoint = %q", selected.Endpoint)
}
if metadata == nil || metadata.DefaultProfile == nil || *metadata.DefaultProfile != "not-yet-listed" {
t.Fatalf("default profile overlay was not retained: %+v", metadata)
}
if metadata.Reachability == nil || metadata.Reachability.Endpoint == nil ||
*metadata.Reachability.Endpoint != "https://probe.example/ready?check=1" {
t.Fatalf("full reachability destination was not normalized: %+v", metadata.Reachability)
}
if metadata.Authentication == nil || metadata.Authentication.Kind == nil || *metadata.Authentication.Kind != "none" ||
metadata.Authentication.Endpoint != nil {
t.Fatalf("authentication overlay or null fallback was not retained: %+v", metadata.Authentication)
}
}
func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
for _, test := range []struct {
name string
data map[string]any
path string
}{
{
name: "blank default profile",
data: map[string]any{"default_profile": " \t "},
path: "/backends/0/smoke_tests/default_profile",
},
{
name: "authentication kind required",
data: map[string]any{"authentication": map[string]any{}},
path: "/backends/0/smoke_tests/authentication/kind",
},
{
name: "unknown authentication kind",
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
path: "/backends/0/smoke_tests/authentication/kind",
},
{
name: "non-http reachability URL",
data: map[string]any{"reachability": map[string]any{"endpoint": "ftp://probe.example/ready"}},
path: "/backends/0/smoke_tests/reachability/endpoint",
},
{
name: "blank authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": " "}},
path: "/backends/0/smoke_tests/authentication/endpoint",
},
{
name: "relative authentication URL",
data: map[string]any{"authentication": map[string]any{"kind": "none", "endpoint": "/key"}},
path: "/backends/0/smoke_tests/authentication/endpoint",
},
} {
t.Run(test.name, func(t *testing.T) {
_, err := load(mapping("backend", map[string]any{
"backends": []any{map[string]any{"id": "local",
"endpoint": "https://backend.example/v1",
"smoke_tests": test.data,
}},
}), nil)
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("expected final-value validation error, got %v", err)
}
for _, diagnostic := range loadError.Diagnostics() {
if diagnostic.Path.String() == test.path {
return
}
}
t.Fatalf("expected diagnostic at %s, got %+v", test.path, loadError.Diagnostics())
})
}
}
func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
underlay := mapping("defaults", map[string]any{"backends": []any{
map[string]any{"id": "local", "endpoint": "http://localhost:8080/v1", "api_key_env": "OLD_KEY"},
}})
loaded, err := load(mapping("backend", map[string]any{"backends": []any{
map[string]any{"id": "local", "endpoint": "http://localhost:8080/v1", "concurrency_limit": 1},
}}), []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatal(err)
}
backend, err := loaded.GetBackend("local")
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity != nil || loaded.backends["local"]["queue_capacity"] != fieldAbsent {
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
}
}
func TestLoadCapturesProfilePresence(t *testing.T) {
loaded, err := load(mapping("profile", map[string]any{"profiles": []any{
map[string]any{"id": "local", "temperature": 0, "top_p": nil, "api_key_required": false},
}}), nil)
if err != nil {
t.Fatal(err)
}
profile := loaded.result.Config.Profiles[0]
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
t.Fatalf("profile values were not preserved: %+v", profile)
}
fields := loaded.profiles["local"]
if fields["temperature"] != fieldValue || fields["top_p"] != fieldNull || fields["api_key_required"] != fieldValue || fields["model"] != fieldAbsent {
t.Fatalf("profile presence was not preserved: %+v", fields)
}
}
func TestLoadRejectsStrictGroupConflict(t *testing.T) {
_, err := load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{
mapping("first", map[string]any{"profiles": []any{map[string]any{"id": "shared", "model": "one"}}}),
mapping("second", map[string]any{"profiles": []any{map[string]any{"id": "shared", "model": "one"}}}),
}}})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("expected ConfigKit load error, got %v", err)
}
}
func TestLoadValidatesOnlyFinalDomainValues(t *testing.T) {
underlay := mapping("defaults", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "not a URL"}},
"profiles": []any{map[string]any{"id": "partial", "backend": "local"}},
})
main := mapping("operator", map[string]any{"backends": []any{map[string]any{"id": "local", "endpoint": "http://localhost:8080/v1"}}})
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatalf("replaced lower-layer domain value should not fail: %v", err)
}
if loaded.result.Config.Profiles[0].Model != nil {
t.Fatal("unused partial profile unexpectedly gained a model")
}
_, err = load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{{Inputs: []configkit.Input{
mapping("invalid", map[string]any{"backends": []any{map[string]any{"id": "unused", "endpoint": "not a URL"}}}),
}}})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("expected path-aware effective backend failure, got %v", err)
}
if diagnostics := loadError.Diagnostics(); len(diagnostics) != 1 || diagnostics[0].Path.String() != "/backends/0/endpoint" {
t.Fatalf("expected endpoint diagnostic, got %+v", diagnostics)
}
}
func TestLoadValidatesOnlyFinalBackendSmokeTestValues(t *testing.T) {
underlay := mapping("defaults", map[string]any{
"backends": []any{map[string]any{"id": "local",
"endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": " ",
"reachability": map[string]any{"endpoint": "ftp://invalid.example/"},
"authentication": map[string]any{"kind": "unsupported"},
},
}},
})
main := mapping("operator", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://backend.example/v1",
"smoke_tests": map[string]any{
"default_profile": "dangling-profile",
"reachability": map[string]any{"endpoint": nil},
"authentication": map[string]any{"kind": "none"},
},
}},
})
loaded, err := load(main, []configkit.Underlay{{Inputs: []configkit.Input{underlay}}})
if err != nil {
t.Fatalf("replaced lower metadata should not fail: %v", err)
}
selected, err := loaded.registry.GetBackend("local")
if err != nil {
t.Fatal(err)
}
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
*selected.SmokeTests.DefaultProfile != "dangling-profile" {
t.Fatalf("final dangling default profile was not retained: %+v", selected.SmokeTests)
}
}
func TestLoadKeepsSyntaxErrorsFromLowerTiers(t *testing.T) {
_, err := load(mapping("operator", map[string]any{"backends": []any{map[string]any{"id": "local", "endpoint": "http://localhost:8080"}}}),
[]configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Bytes("broken", []byte("backends: ["))}}}})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("expected lower-tier parsing error, got %v", err)
}
}

View File

@@ -0,0 +1,395 @@
package configuration
import (
"context"
"errors"
"fmt"
"io"
"strings"
"text/template"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/filecatalog"
"go.maximumdirect.net/promptkit/internal/prompt"
"go.maximumdirect.net/promptkit/internal/promptdef"
)
var (
errPromptNotFound = errors.New("prompt definition not found")
)
type promptEntry struct {
definition domain.PromptDefinition
inspection domain.PromptInspection
requirementInputs []prompt.InputMetadata
hash string
session *template.Template
messages []*template.Template
}
func interpretPrompts(source []promptConfig) []promptConfig {
if source == nil {
return nil
}
prompts := append([]promptConfig{}, source...)
for index := range prompts {
definition := &prompts[index]
for _, value := range []**string{&definition.Description, &definition.SessionID} {
if *value != nil {
normalized := strings.TrimSpace(**value)
*value = &normalized
}
}
if definition.Inputs != nil {
definition.Inputs = append([]promptInputConfig{}, definition.Inputs...)
for index := range definition.Inputs {
input := &definition.Inputs[index]
input.Name = strings.TrimSpace(input.Name)
for _, value := range []**string{&input.ContentType, &input.Description} {
if *value != nil {
normalized := strings.TrimSpace(**value)
*value = &normalized
}
}
}
}
if definition.Output != nil {
output := *definition.Output
definition.Output = &output
if output.SchemaPath != nil {
normalized := strings.TrimSpace(*output.SchemaPath)
output.SchemaPath = &normalized
}
}
}
return prompts
}
func (state *loaded) materializePrompts() error {
state.prompts = make(map[string]*promptEntry)
var metadata *configkit.Metadata
lookupOrigin := func(path configkit.Path) (configkit.FieldInfo, bool) {
if metadata == nil {
captured := state.result.Metadata()
metadata = &captured
}
return metadata.Lookup(path)
}
for _, resources := range resourceSets(&state.interpreted) {
for index, raw := range resources.prompts {
promptBase := resources.bases.prompts.Index(index)
if strings.TrimSpace(raw.ID) == "" {
return fmt.Errorf("%s: prompt ID must not be blank", promptBase)
}
fullID := indexedResourceID(resources.namespace, raw.ID)
entry, err := state.materializePrompt(fullID, raw, promptBase, lookupOrigin)
if err != nil {
return err
}
state.prompts[fullID] = entry
}
}
return nil
}
func (state *loaded) materializePrompt(id string, raw promptConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (*promptEntry, error) {
if len(raw.Messages) == 0 {
return nil, fmt.Errorf("%s: at least one message is required", base.Key("messages"))
}
definition := domain.PromptDefinition{ID: id}
if raw.DefaultProfile != nil {
if strings.TrimSpace(*raw.DefaultProfile) == "" {
return nil, fmt.Errorf("%s: default profile must not be blank", base.Key("default_profile"))
}
definition.DefaultProfile = *raw.DefaultProfile
}
if raw.Description != nil {
definition.Description = *raw.Description
}
if raw.SessionID != nil {
definition.SessionID = *raw.SessionID
}
seenInputs := make(map[string]bool, len(raw.Inputs))
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
entry := &promptEntry{
messages: make([]*template.Template, 0, len(raw.Messages)),
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
}
for index, input := range raw.Inputs {
path := base.Key("inputs").Index(index)
if input.Name == "" {
return nil, fmt.Errorf("%s: input name is required", path.Key("name"))
}
if seenInputs[input.Name] {
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
}
seenInputs[input.Name] = true
contentType := ""
if input.ContentType != nil {
contentType = *input.ContentType
}
description := ""
if input.Description != nil {
description = *input.Description
}
definition.Inputs = append(definition.Inputs, domain.PromptInput{
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
})
presence := state.promptMetadata[id][index]
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
if presence.contentType == fieldValue {
metadata.ContentType = stringPointer(contentType)
}
if presence.description == fieldValue {
metadata.Description = stringPointer(description)
}
entry.requirementInputs = append(entry.requirementInputs, metadata)
}
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
for index, message := range raw.Messages {
path := base.Key("messages").Index(index)
if message.Role == nil {
return nil, fmt.Errorf("%s: role is required", path.Key("role"))
}
role, err := domain.NormalizeMessageRole(*message.Role)
if err != nil {
return nil, fmt.Errorf("%s: %w", path.Key("role"), err)
}
if (message.Content != nil) == (message.ContentFile != nil) {
return nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
}
content := ""
if message.Content != nil {
content = *message.Content
} else {
var err error
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
if err != nil {
return nil, err
}
}
var control *domain.CacheControl
if message.CacheControl != nil {
if message.CacheControl.Type == nil {
return nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
}
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
if message.CacheControl.TTL != nil {
control.TTL = *message.CacheControl.TTL
}
var err error
control, err = domain.NormalizeCacheControl(control)
if err != nil {
return nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
}
}
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
if err != nil {
return nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
}
entry.messages = append(entry.messages, compiled)
definition.Templates = append(definition.Templates, domain.PromptMessageTemplate{
Role: role, Content: content, CacheControl: control,
})
}
if raw.Output == nil {
return nil, fmt.Errorf("%s: output is required", base.Key("output"))
}
if raw.Output.Format == nil || raw.Output.ValidationMode == nil {
return nil, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
}
contract := domain.OutputContract{
Format: domain.OutputFormat(*raw.Output.Format),
ValidationMode: domain.ValidationMode(*raw.Output.ValidationMode),
RepairAttempts: raw.Output.RepairAttempts,
}
if raw.Output.SchemaPath != nil {
contract.SchemaPath = *raw.Output.SchemaPath
}
if err := domain.ValidateOutputContract(contract); err != nil {
return nil, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
}
definition.OutputFormat = contract.Format
definition.Validation = contract
if raw.SessionID != nil {
var err error
entry.session, err = prompt.CompileTemplate("session_id", definition.SessionID)
if err != nil {
return nil, fmt.Errorf("%s: invalid template syntax: %w", base.Key("session_id"), err)
}
}
hash, err := promptdef.HashDefinition(&definition)
if err != nil {
return nil, fmt.Errorf("%s: hash prompt definition: %w", base, err)
}
entry.definition = definition
entry.hash = hash
entry.inspection = domain.PromptInspection{
PromptID: id, PromptHash: hash,
DefaultProfileID: definition.DefaultProfile, Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
OutputContract: contract,
}
return entry, nil
}
func stringPointer(value string) *string {
copy := value
return &copy
}
func stringPointerValue(value *string) *string {
if value == nil {
return nil
}
return stringPointer(*value)
}
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
if strings.TrimSpace(name) == "" {
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
}
field, found := lookupOrigin(fieldPath)
if !found || len(field.Origin.Locations) != 1 {
return "", fmt.Errorf("%s: content_file needs one contributing source with a filesystem context", fieldPath)
}
sourceID := field.Origin.Locations[0].SourceID
context, found := state.result.SourceContext(sourceID)
if !found || context.FS == nil {
return "", fmt.Errorf("%s: content_file source %s has no filesystem context", fieldPath, sourceID)
}
resolved, _, err := filecatalog.ResolveFSPath(".", context.BaseDir, name)
if err != nil {
return "", fmt.Errorf("%s: content_file path is outside its source root or invalid: %w", fieldPath, err)
}
file, err := context.FS.Open(resolved)
if err != nil {
return "", fmt.Errorf("%s: open content_file from source %s: %w", fieldPath, sourceID, err)
}
content, readErr := io.ReadAll(file)
closeErr := file.Close()
if readErr != nil {
return "", fmt.Errorf("%s: read content_file from source %s: %w", fieldPath, sourceID, readErr)
}
if closeErr != nil {
return "", fmt.Errorf("%s: close content_file from source %s: %w", fieldPath, sourceID, closeErr)
}
return string(content), nil
}
func (state *loaded) lookupPromptEntry(id string) (*promptEntry, error) {
if state == nil || strings.TrimSpace(id) == "" {
return nil, errPromptNotFound
}
entry := state.prompts[id]
if entry == nil {
return nil, errPromptNotFound
}
return entry, nil
}
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
for index := range definition.Templates {
if control := definition.Templates[index].CacheControl; control != nil {
copy := *control
definition.Templates[index].CacheControl = &copy
}
}
return &definition
}
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &inspection
}
func promptLookupError(err error) error {
if errors.Is(err, errPromptNotFound) {
return promptdef.ErrPromptDefinitionNotFound
}
return err
}
func (state *loaded) prompt(id string) (*domain.PromptDefinition, domain.PromptInspection, error) {
entry, err := state.lookupPromptEntry(id)
if err != nil {
return nil, domain.PromptInspection{}, err
}
return ownPromptDefinition(entry.definition), *ownPromptInspection(entry.inspection), nil
}
// GetPromptSnapshot returns an independently owned captured definition and its
// construction-time hash.
func (state *loaded) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error) {
if err := ctx.Err(); err != nil {
return nil, "", err
}
entry, err := state.lookupPromptEntry(id)
if err != nil {
return nil, "", promptLookupError(err)
}
return ownPromptDefinition(entry.definition), entry.hash, nil
}
// GetPromptInspection returns an independently owned inspection of one
// captured prompt without copying its execution templates.
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
entry, err := state.lookupPromptEntry(id)
if err != nil {
return nil, promptLookupError(err)
}
return ownPromptInspection(entry.inspection), nil
}
// GetPromptDefinition implements the runner's prompt lookup boundary.
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
definition, _, err := state.GetPromptSnapshot(ctx, id)
return definition, err
}
// GetCompiledTemplates returns construction-time parses for one prompt ID.
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
if state == nil {
return prompt.CompiledTemplates{}, false
}
entry := state.prompts[id]
if entry == nil {
return prompt.CompiledTemplates{}, false
}
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
}
// GetRequirementSnapshot returns captured templates and input metadata for
// one prompt ID. Returned metadata and slices are independently
// owned; the compiled templates are shared immutable construction-time parses.
func (state *loaded) GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool) {
if state == nil {
return prompt.RequirementSnapshot{}, false
}
entry := state.prompts[id]
if entry == nil {
return prompt.RequirementSnapshot{}, false
}
snapshot := prompt.RequirementSnapshot{
Templates: prompt.CompiledTemplates{
Session: entry.session,
Messages: append([]*template.Template(nil), entry.messages...),
},
Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)),
}
for index, input := range entry.requirementInputs {
snapshot.Inputs[index] = input
if input.ContentType != nil {
snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType)
}
if input.Description != nil {
snapshot.Inputs[index].Description = stringPointer(*input.Description)
}
}
return snapshot, true
}

View File

@@ -0,0 +1,422 @@
package configuration
import (
"context"
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/promptdef"
)
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
files := fstest.MapFS{
"app/main.yaml": {Data: []byte("config:\n imports:\n - path: ../catalog/base.yaml\n required: true\ncollections:\n docs:\n prompts:\n - id: child\n extends: {id: base}\n other:\n prompts:\n - id: child\n messages:\n - role: user\n content_file: ./body.md\n output:\n format: text\n validation_mode: none\n")},
"app/body.md": {Data: []byte("Replacement {{.name}}")},
"catalog/base.yaml": {Data: []byte("collections:\n docs:\n release_version: r1\n prompts:\n - id: base\n description: ' description '\n session_id: ' session '\n inputs:\n - name: ' document '\n content_type: ' text/plain '\n description: ' document source '\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
"other/definition.yaml": {Data: []byte("collections:\n other:\n prompts:\n - id: child\n messages:\n - role: user\n content_file: ./body.md\n output:\n format: text\n validation_mode: none\n")},
"other/body.md": {Data: []byte("Other collection {{.name}}")},
}
namespace := configkit.NewFilesystem("embedded", files)
state, err := load(configkit.Input{
Source: namespace.File("app/main.yaml"), Imports: true,
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."},
}, []configkit.Underlay{{Inputs: []configkit.Input{{Source: namespace.File("other/definition.yaml"), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}}})
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
definition, inspection, err := state.prompt("docs/child")
if err != nil {
t.Fatal(err)
}
if definition.Templates[0].Content != "Captured {{.name}}" || inspection.PromptHash == "" {
t.Fatalf("inherited file content was not captured: %+v, %+v", definition, inspection)
}
if definition.Description != "description" || definition.SessionID != "session" || len(definition.Inputs) != 1 ||
definition.Inputs[0].Name != "document" || definition.Inputs[0].ContentType != "text/plain" || definition.Inputs[0].Description != "document source" {
t.Fatalf("inherited metadata was not interpreted: %+v", definition)
}
message := definition.Templates[0]
if message.Role != "user" || message.CacheControl == nil || message.CacheControl.Type != "ephemeral" || message.CacheControl.TTL != "1h" {
t.Fatalf("message metadata was not normalized: %+v", message)
}
childIndex := -1
for index, item := range state.result.Config.Collections["docs"].Prompts {
if item.ID == "child" {
childIndex = index
}
}
if childIndex < 0 {
t.Fatal("inherited prompt missing from effective list")
}
composed := state.result.Config.Collections["docs"].Prompts[childIndex]
if *composed.Description != " description " || *composed.SessionID != " session " || composed.Inputs[0].Name != " document " ||
*composed.Inputs[0].ContentType != " text/plain " || *composed.Inputs[0].Description != " document source " {
t.Fatalf("composed metadata was changed: %+v", composed)
}
field, ok := state.result.Metadata().Lookup(configkit.Keys("collections", "docs", "prompts").Index(childIndex).Key("messages").Index(0).Key("content_file"))
if !ok || len(field.Origin.Locations) != 1 {
t.Fatalf("content origin was lost during normalization: %+v", field)
}
var originPath string
for _, source := range state.result.Sources() {
if source.ID == field.Origin.Locations[0].SourceID {
originPath = source.Path
}
}
if originPath != "catalog/base.yaml" {
t.Fatalf("content origin = %q, want catalog/base.yaml", originPath)
}
other, otherInspection, err := state.prompt("other/child")
if err != nil || other.Templates[0].Content != "Replacement {{.name}}" {
t.Fatalf("other collection did not retain its source context: %+v, %v", other, err)
}
files["catalog/body.md"].Data = []byte("Changed")
files["app/body.md"].Data = []byte("Changed replacement")
files["other/body.md"].Data = []byte("Changed other collection")
again, againInspection, err := state.prompt("docs/child")
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
}
definition.Templates[0].Content = "caller mutation"
third, _, err := state.prompt("docs/child")
if err != nil || third.Templates[0].Content != "Captured {{.name}}" {
t.Fatalf("caller mutation escaped into index: %+v, %v", third, err)
}
otherAgain, otherAgainInspection, err := state.prompt("other/child")
if err != nil || otherAgain.Templates[0].Content != "Replacement {{.name}}" || otherAgainInspection.PromptHash != otherInspection.PromptHash {
t.Fatalf("other captured definition changed after source mutation: %+v, %+v, %v", otherAgain, otherAgainInspection, err)
}
}
func TestLoadHeaderlessImportedItem(t *testing.T) {
files := fstest.MapFS{
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
"item.yaml": {Data: []byte("id: example\nmessages:\n - role: user\n content: hello\noutput:\n format: text\n validation_mode: basic\n")},
}
namespace := configkit.NewFilesystem("items", files)
state, err := load(configkit.Input{Source: namespace.File("main.yaml"), Imports: true,
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}, nil)
if err != nil {
t.Fatal(err)
}
definition, _, err := state.prompt("example")
if err != nil || definition.Templates[0].Content != "hello" {
t.Fatalf("headerless item = %+v, %v", definition, err)
}
}
func TestLoadUsesExplicitMappingContextAndClosesContent(t *testing.T) {
filesystem := &countingFS{FS: fstest.MapFS{
"assets/body.md": {Data: []byte("From mapping")},
}}
namespace := configkit.NewFilesystem("mapping-context", filesystem)
source := configkit.Mapping("application", promptWithFile("body.md"))
state, err := load(configkit.Input{
Source: source,
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
}, nil)
if err != nil {
t.Fatal(err)
}
definition, _, err := state.prompt("example")
if err != nil || definition.Templates[0].Content != "From mapping" || filesystem.contentOpened != 1 || filesystem.contentClosed != 1 {
t.Fatalf("mapping context or handle ownership failed: %+v, opens=%d closes=%d, %v", definition, filesystem.contentOpened, filesystem.contentClosed, err)
}
_, err = load(mapping("contextless", promptWithFile("body.md")), nil)
if err == nil || !strings.Contains(err.Error(), "/prompts/0/messages/0/content_file") {
t.Fatalf("expected semantic path for contextless resource, got %v", err)
}
}
func TestLoadCapturesNativeContentFile(t *testing.T) {
root := t.TempDir()
definitionPath := filepath.Join(root, "definition.yaml")
contentPath := filepath.Join(root, "body.md")
if err := os.WriteFile(definitionPath, []byte("prompts:\n - id: example\n messages:\n - role: user\n content_file: body.md\n output:\n format: text\n validation_mode: basic\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(contentPath, []byte("Native body"), 0o600); err != nil {
t.Fatal(err)
}
state, err := load(configkit.Input{Source: configkit.File(definitionPath), Filesystem: &configkit.FilesystemContext{Root: root}}, nil)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(contentPath, []byte("Modified"), 0o600); err != nil {
t.Fatal(err)
}
definition, _, err := state.prompt("example")
if err != nil || definition.Templates[0].Content != "Native body" {
t.Fatalf("native content was not captured: %+v, %v", definition, err)
}
}
func TestLoadRejectsMalformedStaticSyntax(t *testing.T) {
_, err := load(mapping("bad-template", map[string]any{"prompts": []any{map[string]any{"id": "example",
"messages": []any{map[string]any{"role": "user", "content": "{{if}}"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}), nil)
if err == nil || !strings.Contains(err.Error(), "invalid template syntax") {
t.Fatalf("expected construction-time syntax error, got %v", err)
}
}
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
makePrompt := func(content string) map[string]any {
return map[string]any{
"default_profile": "partial",
"inputs": []any{map[string]any{
"name": "document", "required": true,
"content_type": " text/plain ", "description": " source document ",
}},
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
selectedPrompt := makePrompt("selected")
selectedPrompt["id"] = "selected"
state, err := load(mapping("prompt inspection lookup", map[string]any{
"profiles": []any{map[string]any{"id": "partial"}},
"prompts": []any{selectedPrompt},
}), nil)
if err != nil {
t.Fatal(err)
}
{
inspection, err := state.GetPromptInspection(context.Background(), "selected")
if err != nil {
t.Fatalf("inspect prompt ID: %v", err)
}
definition, hash, err := state.GetPromptSnapshot(context.Background(), "selected")
if err != nil {
t.Fatalf("snapshot prompt ID: %v", err)
}
if inspection.PromptID != definition.ID ||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
t.Fatalf("inspection did not match the selected snapshot: inspection=%+v definition=%+v hash=%q", inspection, definition, hash)
}
if len(definition.Inputs) != len(inspection.Inputs) || definition.Inputs[0].Name != inspection.Inputs[0].Name {
t.Fatalf("definition and inspection inputs diverged: definition=%+v inspection=%+v", definition.Inputs, inspection.Inputs)
}
}
owned, err := state.GetPromptInspection(context.Background(), "selected")
if err != nil {
t.Fatal(err)
}
owned.Inputs[0].Name = "caller mutation"
owned.Inputs[0].Description = "caller mutation"
again, err := state.GetPromptInspection(context.Background(), "selected")
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
}
for _, id := range []string{"missing", " "} {
inspection, err := state.GetPromptInspection(context.Background(), id)
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Errorf("inspection lookup %q=(%+v, %v), want not found", id, inspection, err)
}
definition, _, err := state.GetPromptSnapshot(context.Background(), id)
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Errorf("snapshot lookup %q=(%+v, %v), want not found", id, definition, err)
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := state.GetPromptInspection(ctx, "selected"); !errors.Is(err, context.Canceled) {
t.Errorf("canceled inspection selection error=%v", err)
}
}
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
state, err := load(mapping("empty-content", map[string]any{"prompts": []any{map[string]any{"id": " exact.id ",
"inputs": []any{map[string]any{"name": " note ", "required": true}},
"messages": []any{map[string]any{"role": " USER ", "content": "", "content_file": nil,
"cache_control": map[string]any{"type": " ephemeral ", "ttl": "1h "}}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
definition, _, err := state.prompt(" exact.id ")
if err != nil || definition.ID != " exact.id " || definition.Templates[0].Content != "" || definition.Templates[0].Role != "user" ||
definition.Inputs[0].Name != "note" || definition.Templates[0].CacheControl.TTL != "1h" {
t.Fatalf("explicit empty content or literal ID was changed: %+v, %v", definition, err)
}
if _, _, err := state.prompt("exact.id"); !errors.Is(err, errPromptNotFound) {
t.Fatalf("trimmed prompt identity unexpectedly matched: %v", err)
}
_, err = load(mapping("obsolete-field", map[string]any{"prompts": []any{map[string]any{
"version": "v1",
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "ok"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}), nil)
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("obsolete identity field should fail strict decoding: %v", err)
}
_, err = load(mapping("null-prompt", map[string]any{"prompts": []any{nil}}), nil)
if err == nil {
t.Fatal("null prompt collection was accepted")
}
}
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
state, err := load(mapping("requirement-metadata", map[string]any{
"prompts": []any{map[string]any{
"id": "example",
"session_id": "{{.session}}",
"inputs": []any{
map[string]any{"name": "absent"},
map[string]any{"name": "null", "content_type": nil, "description": nil},
map[string]any{"name": "empty", "content_type": "", "description": ""},
map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "},
},
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}},
}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
snapshot, ok := state.GetRequirementSnapshot("example")
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
}
for _, index := range []int{0, 1} {
if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil {
t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index])
}
}
if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" ||
snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" {
t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2])
}
if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" ||
snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" {
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
}
definition, inspection, err := state.prompt("example")
if err != nil {
t.Fatal(err)
}
if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" ||
inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" {
t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection)
}
*snapshot.Inputs[2].ContentType = "caller mutation"
snapshot.Inputs[3].Name = "caller mutation"
snapshot.Templates.Messages[0] = nil
again, ok := state.GetRequirementSnapshot("example")
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
}
}
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
variants := []struct {
name string
fields map[string]any
}{
{name: "absent"},
{name: "null", fields: map[string]any{"content_type": nil, "description": nil}},
{name: "empty", fields: map[string]any{"content_type": "", "description": ""}},
}
var expectedHash string
for _, variant := range variants {
t.Run(variant.name, func(t *testing.T) {
input := map[string]any{"name": "document"}
for name, value := range variant.fields {
input[name] = value
}
state, err := load(mapping("hash-"+variant.name, map[string]any{
"prompts": []any{map[string]any{
"id": "same",
"inputs": []any{input},
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}},
}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
_, inspection, err := state.prompt("same")
if err != nil {
t.Fatal(err)
}
if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" {
t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs)
}
if expectedHash == "" {
expectedHash = inspection.PromptHash
} else if inspection.PromptHash != expectedHash {
t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash)
}
})
}
}
func promptWithFile(name string) map[string]any {
return map[string]any{"prompts": []any{map[string]any{
"id": "example",
"messages": []any{map[string]any{"role": "user", "content_file": name}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}}
}
type countingFS struct {
fs.FS
contentOpened int
contentClosed int
}
func (filesystem *countingFS) Open(name string) (fs.File, error) {
file, err := filesystem.FS.Open(name)
if err != nil {
return nil, err
}
if name == "assets/body.md" {
filesystem.contentOpened++
return &countingFile{File: file, owner: filesystem}, nil
}
return file, nil
}
type countingFile struct {
fs.File
owner *countingFS
}
func (file *countingFile) Close() error {
file.owner.contentClosed++
return file.File.Close()
}

View File

@@ -0,0 +1,215 @@
package configuration
import (
"context"
"fmt"
"go.maximumdirect.net/promptkit/internal/backend"
"go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
profilepkg "go.maximumdirect.net/promptkit/internal/profile"
)
func (state *loaded) buildExecutionProfiles() error {
backendConfigs := indexedBackends(&state.interpreted)
profileConfigs := indexedProfiles(&state.interpreted)
definitions := make(map[string]domain.Backend, len(backendConfigs))
for _, id := range sortedKeys(backendConfigs) {
config := backendConfigs[id]
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil {
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
}
if config.SmokeTests.Authentication != nil {
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
Kind: config.SmokeTests.Authentication.Kind,
Endpoint: config.SmokeTests.Authentication.Endpoint,
}
}
definition.SmokeTests = smokeTests
}
if config.Endpoint != nil {
definition.Endpoint = *config.Endpoint
}
if config.APIKeyEnv != nil {
definition.APIKeyEnv = *config.APIKeyEnv
}
if config.ConcurrencyLimit != nil {
definition.ConcurrencyLimit = *config.ConcurrencyLimit
}
if config.QueueCapacity != nil {
definition.QueueCapacity = *config.QueueCapacity
definition.QueueCapacitySet = definition.ConcurrencyLimit > 0
}
definitions[id] = definition
}
registry, err := backend.NewEffectiveRegistry(definitions)
if err != nil {
return err
}
state.registry = registry
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
for _, id := range sortedKeys(profileConfigs) {
config := profileConfigs[id]
fields := state.profiles[id]
target := defaults.ExecutionTargetDefault()
if config.Backend != nil && *config.Backend != "" {
selected, err := registry.GetBackend(*config.Backend)
if err != nil {
return fmt.Errorf("profile %q backend: %w", id, err)
}
target.BackendID = selected.ID
target.Endpoint = selected.Endpoint
target.APIKeyEnv = selected.APIKeyEnv
target.ExtraParams = selected.ExtraParams
}
presence := domain.ExecutionTargetPresence{}
if fields["endpoint"] != fieldAbsent {
target.Endpoint = valueOrZero(config.Endpoint)
}
if fields["model"] != fieldAbsent {
target.Model = valueOrZero(config.Model)
}
if fields["temperature"] != fieldAbsent {
target.Temperature = valueOrZero(config.Temperature)
presence.Temperature = fields["temperature"] == fieldValue
}
if fields["top_p"] != fieldAbsent {
target.TopP = valueOrZero(config.TopP)
presence.TopP = fields["top_p"] == fieldValue
}
if fields["max_tokens"] != fieldAbsent {
target.MaxTokens = valueOrZero(config.MaxTokens)
presence.MaxTokens = fields["max_tokens"] == fieldValue
}
if fields["timeout_seconds"] == fieldValue {
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
presence.TimeoutSeconds = true
}
if fields["service_tier"] != fieldAbsent {
target.ServiceTier = valueOrZero(config.ServiceTier)
}
if fields["reasoning_effort"] != fieldAbsent {
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
}
if fields["api_key_env"] != fieldAbsent {
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
}
if fields["api_key_required"] != fieldAbsent {
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
}
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
target.APIKeyEnv = ""
}
switch fields["extra_params"] {
case fieldNull:
target.ExtraParams = nil
case fieldValue:
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
if err != nil {
return fmt.Errorf("profile %q extra parameters: %w", id, err)
}
target.ExtraParams = merged
}
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
state.indexedProfiles[id] = profile
}
return nil
}
func valueOrZero[T any](value *T) T {
if value == nil {
var zero T
return zero
}
return *value
}
// overlayProviderMap applies only the backend-to-profile JSON-object default
// merge. ConfigKit has already resolved source layering and inheritance.
func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
merged, err := jsonvalue.CopyMap(lower)
if err != nil {
return nil, err
}
if merged == nil {
merged = make(map[string]any)
}
for key, value := range upper {
if child, ok := value.(map[string]any); ok {
if existing, ok := merged[key].(map[string]any); ok {
combined, err := overlayProviderMap(existing, child)
if err != nil {
return nil, err
}
merged[key] = combined
continue
}
}
copied, err := jsonvalue.Copy(value)
if err != nil {
return nil, err
}
merged[key] = copied
}
return merged, nil
}
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return domain.ExecutionProfile{}, err
}
if state == nil {
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
}
profile, found := state.indexedProfiles[id]
if !found {
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
}
return profile, nil
}
// GetProfile returns an independently owned indexed profile for the internal
// runner's lookup boundary.
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
profile, err := state.lookupProfile(ctx, id)
if err != nil {
return nil, err
}
target := *profile.ResolvedTarget
params, err := jsonvalue.CopyMap(target.ExtraParams)
if err != nil {
return nil, err
}
target.ExtraParams = params
profile.ResolvedTarget = &target
return &profile, nil
}
// GetProfileIdentity returns the exact lookup ID for a captured profile
// without copying its execution target.
func (state *loaded) GetProfileIdentity(ctx context.Context, id string) (string, error) {
if _, err := state.lookupProfile(ctx, id); err != nil {
return "", err
}
return id, nil
}
// GetBackend returns an independently owned effective backend definition.
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
if state == nil {
return domain.Backend{}, fmt.Errorf("backend registry is not configured")
}
return state.registry.GetBackend(id)
}
// CapacityPolicies returns the selected backend admission policies.
func (state *loaded) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
if state == nil {
return nil
}
return state.registry.CapacityPolicies()
}

View File

@@ -0,0 +1,218 @@
package configuration
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/llm"
profilepkg "go.maximumdirect.net/promptkit/internal/profile"
"go.maximumdirect.net/promptkit/internal/prompt"
"go.maximumdirect.net/promptkit/internal/usecase"
)
func TestLoadIndexesIdentifiedTargets(t *testing.T) {
values := map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://root.example/v1"}},
"profiles": []any{map[string]any{"id": "partial", "backend": "local"}},
"collections": map[string]any{},
}
collections := values["collections"].(map[string]any)
for _, namespace := range []string{"north", "south"} {
collections[namespace] = map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://" + namespace + ".example/v1"}},
"profiles": []any{map[string]any{"id": "partial", "backend": namespace + "/local"}},
}
}
state, err := load(mapping("identified targets", values), nil)
if err != nil {
t.Fatal(err)
}
for id, backendID := range map[string]string{"partial": "local", "north/partial": "north/local", "south/partial": "south/local"} {
profile, err := state.GetProfile(context.Background(), id)
if err != nil || profile.ID != id || profile.ResolvedTarget == nil || profile.ResolvedTarget.Model != "" || profile.ResolvedTarget.BackendID != backendID {
t.Fatalf("partial profile %q = %+v, %v", id, profile, err)
}
}
}
func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
state, err := load(mapping("partial profile", map[string]any{
"profiles": []any{map[string]any{"id": "partial"}},
"prompts": []any{map[string]any{
"id": "example", "messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}},
}), nil)
if err != nil {
t.Fatal(err)
}
runner := usecase.NewRunner(state, state, nil, prompt.NewGoRenderer(), nil, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "example", ProfileID: "partial",
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
})
if err != nil {
t.Fatal(err)
}
if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Endpoint != "https://example.invalid/v1" ||
prepared.EffectiveModelParams.Model != "request-model" {
t.Fatalf("request did not complete the loaded profile: %+v", prepared)
}
}
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
state, err := load(mapping("effective-targets", map[string]any{
"backends": []any{
map[string]any{"id": "first", "endpoint": "https://first.example/v1", "api_key_env": "FIRST_KEY",
"extra_params": map[string]any{"source": "first", "nested": map[string]any{"old": true}}},
map[string]any{"id": "second", "endpoint": "https://second.example/v1", "api_key_env": "SECOND_KEY",
"concurrency_limit": 2, "queue_capacity": 0,
"extra_params": map[string]any{"source": "second", "nested": map[string]any{"backend": true}}},
},
"profiles": []any{
map[string]any{"id": "base", "backend": "first", "model": "model", "temperature": 0.8,
"top_p": 0.9, "max_tokens": 100, "timeout_seconds": 30, "api_key_required": true,
"extra_params": map[string]any{"nested": map[string]any{"profile": true}}},
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "backend": "second", "temperature": 0.0,
"top_p": nil, "max_tokens": 0, "timeout_seconds": nil, "api_key_env": nil,
"api_key_required": false,
"extra_params": map[string]any{"nested": map[string]any{"child": nil}}},
map[string]any{"id": "direct-key", "backend": "second", "api_key_required": true},
map[string]any{"id": "named-key", "backend": "second", "api_key_required": true, "api_key_env": "CUSTOM_KEY"},
map[string]any{"id": "no-provider-map", "backend": "second", "extra_params": nil},
map[string]any{"id": "no-timeout", "backend": "second", "timeout_seconds": 0},
map[string]any{"id": "partial"},
map[string]any{"id": "endpoint-only", "endpoint": "https://standalone.example/v1"},
map[string]any{"id": "cleared-endpoint", "extends": map[string]any{"id": "base"}, "endpoint": nil},
},
}), nil)
if err != nil {
t.Fatal(err)
}
selected, err := state.registry.GetBackend("second")
if err != nil || selected.QueueCapacity != 0 || !selected.QueueCapacitySet {
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
}
child, err := state.GetProfile(context.Background(), "child")
if err != nil {
t.Fatal(err)
}
if _, err := state.GetProfile(context.Background(), " child "); err == nil {
t.Fatal("profile lookup silently changed a literal ID")
}
target := child.ResolvedTarget
if target.BackendID != "second" || target.Endpoint != "https://second.example/v1" || target.Model != "model" ||
target.Temperature != 0 || target.TopP != 0 || target.MaxTokens != 0 ||
target.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds || target.APIKeyEnv != "" || target.APIKeyRequired {
t.Fatalf("effective child target is wrong: %+v", target)
}
if !child.TargetPresence.Temperature || !child.TargetPresence.MaxTokens || child.TargetPresence.TopP || child.TargetPresence.TimeoutSeconds {
t.Fatalf("numeric presence was not retained: %+v", child.TargetPresence)
}
wantParams := map[string]any{"source": "second", "nested": map[string]any{
"backend": true, "profile": true, "child": nil,
}}
if !reflect.DeepEqual(target.ExtraParams, wantParams) {
t.Fatalf("backend/profile provider map overlay failed: %+v", target.ExtraParams)
}
if _, exists := target.ExtraParams["nested"].(map[string]any)["old"]; exists {
t.Fatal("previous backend derived parameter leaked into child")
}
for id, wantEnv := range map[string]string{"direct-key": "", "named-key": "CUSTOM_KEY"} {
profile, err := state.GetProfile(context.Background(), id)
if err != nil || profile.ResolvedTarget.APIKeyEnv != wantEnv {
t.Fatalf("credential source for %s: %+v, %v", id, profile, err)
}
}
noMap, _ := state.GetProfile(context.Background(), "no-provider-map")
if noMap.ResolvedTarget.ExtraParams != nil {
t.Fatalf("explicit null provider map did not suppress backend defaults: %+v", noMap.ResolvedTarget.ExtraParams)
}
noTimeout, _ := state.GetProfile(context.Background(), "no-timeout")
if noTimeout.ResolvedTarget.TimeoutSeconds != 0 || !noTimeout.TargetPresence.TimeoutSeconds {
t.Fatalf("explicit zero timeout was lost: %+v", noTimeout)
}
partial, _ := state.GetProfile(context.Background(), "partial")
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
}
identity, err := state.GetProfileIdentity(context.Background(), "partial")
if err != nil || identity != "partial" {
t.Fatalf("partial profile identity=(%q, %v)", identity, err)
}
if _, err := state.GetProfileIdentity(context.Background(), "missing"); !errors.Is(err, profilepkg.ErrProfileNotFound) {
t.Fatalf("missing profile identity error=%v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := state.GetProfileIdentity(ctx, "partial"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled profile identity lookup error=%v", err)
}
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {
t.Fatalf("endpoint-only profile was not retained: %+v", endpointOnly.ResolvedTarget)
}
clearedEndpoint, _ := state.GetProfile(context.Background(), "cleared-endpoint")
if clearedEndpoint.ResolvedTarget.Endpoint != "" || clearedEndpoint.ResolvedTarget.BackendID != "first" {
t.Fatalf("explicit null recovered backend endpoint: %+v", clearedEndpoint.ResolvedTarget)
}
child.ResolvedTarget.ExtraParams["source"] = "mutated"
again, _ := state.GetProfile(context.Background(), "child")
if again.ResolvedTarget.ExtraParams["source"] != "second" {
t.Fatal("profile lookup exposed snapshot ownership")
}
}
func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
payloads := make(chan map[string]any, 1)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
var payload map[string]any
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
writer.WriteHeader(http.StatusBadRequest)
return
}
payloads <- payload
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer server.Close()
state, err := load(mapping("wire-controls", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": server.URL + "/v1"}},
"profiles": []any{map[string]any{
"id": "direct", "backend": "local", "model": "example", "temperature": 0.0, "max_tokens": 0, "top_p": nil,
}},
}), nil)
if err != nil {
t.Fatal(err)
}
profile, err := state.GetProfile(context.Background(), "direct")
if err != nil {
t.Fatal(err)
}
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
})
if err != nil {
t.Fatal(err)
}
payload := <-payloads
if payload["temperature"] != float64(0) || payload["max_tokens"] != float64(0) {
t.Fatalf("explicit zeros were omitted from outbound request: %+v", payload)
}
if _, exists := payload["top_p"]; exists {
t.Fatalf("explicit null top_p was sent: %+v", payload)
}
}

View File

@@ -3,7 +3,7 @@ package defaults
import ( import (
"time" "time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
const ( const (

Some files were not shown because too many files have changed in this diff Show More