Compare commits
404 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5ea60e9b96 | |||
| ba5ac651e2 | |||
| 6f45fd9fd0 | |||
| 79e23ecdba | |||
| 63e2968038 | |||
| 1408001fce | |||
| df8384426d | |||
| 8758c350e9 | |||
| 4a9def2138 | |||
| 40f8a68253 | |||
| 08d3d0ea3b | |||
| 56795bb49a | |||
| 891feb8c27 | |||
| 424d35c798 | |||
| bab3422e62 | |||
| cf03bca753 | |||
| 92e1e225b2 | |||
| 74a77b3c32 | |||
| 3a948a1e02 | |||
| eb66f82b4f | |||
| 4d5841e087 | |||
| 34c0cf3f14 | |||
| 76c8789e65 | |||
| 38a2dcca52 | |||
| dad302f1ff | |||
| 645bff20d7 | |||
| 248d0dbcf5 | |||
| 515649fee0 | |||
| 07aaee8ef6 | |||
| 49ea85680e | |||
| 16f8197a2d | |||
| 2a726d60ff | |||
| d3523000bf | |||
| bb8ec2c792 | |||
| f80ec1c42c | |||
| 649bb7dcf2 | |||
| 55276c9429 | |||
| aae523c916 | |||
| dfe9806228 | |||
| 63c96b7d61 | |||
| e1af2d6d9c | |||
| bd6b2848bb | |||
| eae4d18a9c | |||
| 2c436d14d9 | |||
| 1bfb6eb301 | |||
| 37582e9c30 | |||
| 4120d784a2 | |||
| 3bc16bffe6 | |||
| e9a12d43c2 | |||
| 9bed36282a | |||
| 950d3ccfb2 | |||
| 6fbbaa9cfa | |||
| 2d96dfa696 | |||
| be5a508554 | |||
| e8420d229e | |||
| 596e985cff | |||
| fbe3d427a2 | |||
| e775a3c116 | |||
| adfccb5370 | |||
| 0603a4f0dd | |||
| eaf9a76fd3 | |||
| 3d4c4c940e | |||
| 6668673c8d | |||
| 019e077ba9 | |||
| 718cd05043 | |||
| dd40171c01 | |||
| e415ad43ce | |||
| ae1efc8049 | |||
| 0f8e0e49b1 | |||
| ffce326dd7 | |||
| 804a4d32b2 | |||
| 67d446f611 | |||
| bcdd6fe06c | |||
| 382ccfb14b | |||
| 7406616794 | |||
| 4939fdb283 | |||
| fcbff80ab2 | |||
| d3bc439bb8 | |||
| 3ec68d509c | |||
| 57b62d391e | |||
| 8831ae35f7 | |||
| 3de3e60c0c | |||
| a8698d45a8 | |||
| 2b9badb3a5 | |||
| 79699acbb9 | |||
| 4cc5a55579 | |||
| d42b131de2 | |||
| 2066579e09 | |||
| 0e4f590f5a | |||
| f17e0620a5 | |||
| 5ad1b7aa6e | |||
| 3ace6ddb57 | |||
| 65546d794e | |||
| 892cb74e6e | |||
| 64cf84c9b5 | |||
| c23393b2cc | |||
| 5aae5df5fb | |||
| aaaa2816a4 | |||
| 5c135c5b28 | |||
| 31cc0069fc | |||
| 68daa61867 | |||
| 159d369b9b | |||
| 23fb55f9df | |||
| a07e95fd17 | |||
| c4d7cf8cd4 | |||
| 5a6f73e0bb | |||
| c149f7274c | |||
| e5c42de06d | |||
| 04569e44df | |||
| 26530808e0 | |||
| fcd0485340 | |||
| 5cc3fa0168 | |||
| b423de12c3 | |||
| 72955aaaa3 | |||
| 85dae9b443 | |||
| 70db0ad614 | |||
| 37a48f2e20 | |||
| 6c14379026 | |||
| 45bbf7bb2c | |||
| ee810b8fbe | |||
| bc95d1f9fd | |||
| c948fef8fc | |||
| cb180e8728 | |||
| de2a7fcd6f | |||
| f20020d9f4 | |||
| 9396183bdb | |||
| dba6d7d68c | |||
| d515a7e49b | |||
| 171fb11f4e | |||
| c9cbe7d3b3 | |||
| 7df7e420aa | |||
| 8ff9aa3efd | |||
| 78b78ea00b | |||
| 63e9f0b731 | |||
| 20b0f78af2 | |||
| 101a2840d4 | |||
| f083ad5309 | |||
| 19a0604874 | |||
| ea31e71280 | |||
| 27966c1686 | |||
| ea63a7b0ff | |||
| b5b9b1f209 | |||
| 129959b24c | |||
| dfe9d6566f | |||
| bf3d2e19bd | |||
| 4cbd4e2da0 | |||
| b0a16ccce8 | |||
| 8add7c49e6 | |||
| 8563379216 | |||
| 951739eda0 | |||
| a0f696181f | |||
| 3951dcf1f3 | |||
| 363920deb2 | |||
| 06315062b3 | |||
| c669f14610 | |||
| 52287ab2d1 | |||
| a43e0f17ee | |||
| ffaaf552c8 | |||
| d7007eb072 | |||
| 4f3212bfb6 | |||
| 4faf43cc29 | |||
| c9663ec63c | |||
| d8c98ac575 | |||
| 5ee602cc4b | |||
| 96c619f7d4 | |||
| b3b5ec3820 | |||
| 5dd1418953 | |||
| f5c900cb0a | |||
| c5f3de13f3 | |||
| 88725e79ab | |||
| 6c015a17fa | |||
| 9463c9463a | |||
| 1fbd175e53 | |||
| 4b4ce43796 | |||
| d78474a3bf | |||
| acaf2ccd8b | |||
| 97e5739bfb | |||
| 90f796aef7 | |||
| 0dec3dcec0 | |||
| 448e64dc40 | |||
| de46935a0e | |||
| c69703b2b6 | |||
| f660b6e0fc | |||
| 5f56c65427 | |||
| cfa3a85308 | |||
| 1ff6d778af | |||
| 3f93763c9c | |||
| 4aeb58f913 | |||
| ac1e150168 | |||
| ff57dbf8b3 | |||
| beb18cc9d6 | |||
| 7203e5d6a1 | |||
| 356563a6e5 | |||
| 6673284312 | |||
| 6f42e1e007 | |||
| cee9bbc7b2 | |||
| 7db5572b62 | |||
| a91bc1608b | |||
| e7a3665443 | |||
| 59e8010623 | |||
| ada5377f32 | |||
| df208e132b | |||
| 8c0fb15727 | |||
| 2e2fba4993 | |||
| 68668cee56 | |||
| f745b14b71 | |||
| dbba1485a7 | |||
| 7c435032b6 | |||
| e834e7d806 | |||
| a4901cbb0e | |||
| 5dcfc2e953 | |||
| a02561d40f | |||
| 6112ee8fb4 | |||
| 0aceff8c08 | |||
| cddb2b389c | |||
| e056cb1607 | |||
| 1e122f0473 | |||
| 2f0f5fa607 | |||
| 7d500ee62f | |||
| 7be677c80b | |||
| 1c075bd510 | |||
| b8dc8a49a9 | |||
| cd373ee46b | |||
| e881bbdcf4 | |||
| 106c808442 | |||
| 4ec35f5ca6 | |||
| 6a1a32ae49 | |||
| e85ea2db35 | |||
| 3ef2a9e18e | |||
| a85a6788b4 | |||
| 21fea46fb3 | |||
| 62bf82a56b | |||
| bd3dfa992c | |||
| 425f53a62e | |||
| fcc94eb9e9 | |||
| 95730a3c11 | |||
| c542b81666 | |||
| e8e112186e | |||
| aa08d46ef6 | |||
| 668d09997a | |||
| 69203dcb12 | |||
| 3b760310a6 | |||
| b39460a507 | |||
| 5177db0611 | |||
| 31f1e6a89d | |||
| f7085d76f6 | |||
| cf0bfd4d76 | |||
| 6b1323d1fe | |||
| fb87fabc2e | |||
| 5adc1c1e64 | |||
| 0d27252999 | |||
| d9f6aad305 | |||
| fda0338134 | |||
| 2b3d165ce2 | |||
| 43f55edb98 | |||
| 4950e1dee2 | |||
| d240364776 | |||
| 6f57f2f30a | |||
| 2c50eca033 | |||
| ac44a4f325 | |||
| 9e05c3ac8b | |||
| 2c83925c88 | |||
| e0cd63923d | |||
| 75204d6134 | |||
| 5ce6e01f27 | |||
| 5cec25a40c | |||
| 855aae6daa | |||
| 43abbcb414 | |||
| 9f33fed823 | |||
| 7a98feb399 | |||
| 626134e5e1 | |||
| e7ed315671 | |||
| d7d9a7dd65 | |||
| 18f6a474e0 | |||
| fb7ebe6a6c | |||
| a0441fb2d1 | |||
| 32547e1e29 | |||
| bcc9ebd2e0 | |||
| c7d4318122 | |||
| 175140ae5a | |||
| 66b98831d3 | |||
| 3e3cc677fb | |||
| 8896d8733b | |||
| 7bbdb55888 | |||
| 79dfd97afa | |||
| 995c309560 | |||
| ea4fec9b1b | |||
| 1de3e1fe38 | |||
| 061f01c081 | |||
| 937b9723d4 | |||
| 367af5feca | |||
| fd73cd0662 | |||
| 81a55bc62c | |||
| 2bc976de10 | |||
| fc568aa0b2 | |||
| b0bfe857fe | |||
| 2ccbee3758 | |||
| 0eee3ec962 | |||
| 130c4cc447 | |||
| f661176b06 | |||
| 3e67743fbf | |||
| 6b8eefd18b | |||
| 682cb84c82 | |||
| dbc345e566 | |||
| afcfae2aea | |||
| eaca50763b | |||
| cb83b8aca6 | |||
| 0b026861a5 | |||
| da8496b45b | |||
| e01b23259a | |||
| 500b0aa1c8 | |||
| fd6e9742a1 | |||
| a6f4d67e15 | |||
| c3ce15a4ab | |||
| 8e8f61f53f | |||
| b7565affe5 | |||
| aba890aa86 | |||
| 0b271267a0 | |||
| a4e97bf102 | |||
| 6ae005ed3a | |||
| 07a55e45f2 | |||
| 2b3875008b | |||
| a51d12d898 | |||
| 9178f3eb91 | |||
| a70fd7bb65 | |||
| 7091b51c87 | |||
| 8627c751a4 | |||
| 6fff3d822c | |||
| a1f12f3185 | |||
| 165cef7aec | |||
| 0221cd4610 | |||
| dfd8da843e | |||
| 29f104fbc0 | |||
| 8669294cf6 | |||
| c9b623b374 | |||
| 15f79ce7b1 | |||
| aa54b8d1c4 | |||
| 60c58260ec | |||
| 8dfe35a1f0 | |||
| 13b60a72ac | |||
| ee40a1bdd2 | |||
| 473fc1f6d7 | |||
| a10a9053b7 | |||
| ec3245c97e | |||
| 8cbb7c0daf | |||
| d40a010538 | |||
| 0cd833513c | |||
| 5e58b6944b | |||
| 0a85ac8cd1 | |||
| 89af2a1281 | |||
| 683689a4f8 | |||
| ddda2f8767 | |||
| 190ec69345 | |||
| b5866dd749 | |||
| c7490b783b | |||
| c7d1b0dca7 | |||
| 9e5ed08635 | |||
| 4bddbbe8f4 | |||
| 1516631a30 | |||
| cf997c2034 | |||
| a0ec67c256 | |||
| 75891b6561 | |||
| 5dc6370a97 | |||
| 4cda3674c4 | |||
| 5ea6f06a6d | |||
| 116bd0e39b | |||
| 208ff42915 | |||
| 972a103a97 | |||
| 2f735aedf9 | |||
| f4c721d934 | |||
| b7a951685b | |||
| abb2461244 | |||
| b268c55783 | |||
| f0182fb2c9 | |||
| 8d05b594e2 | |||
| edaaecc982 | |||
| 2e81c9e61b | |||
| 2b723760d1 | |||
| c5aed7a451 | |||
| d91afc61a4 | |||
| 6eeb0ee9c0 | |||
| b9c232ee63 | |||
| 7291d23587 | |||
| da1d196dc5 | |||
| b0de49a87a | |||
| 27ff580dae | |||
| ece764f19e | |||
| f91a02548e | |||
| 8fbd058b90 | |||
| c8f9a4def9 | |||
| d8b6c0b471 | |||
| 52cd94fcd1 | |||
| f1f72dfe8f | |||
| d4676bc870 | |||
| 64769883ae | |||
| 635674c066 | |||
| 50bc35516a | |||
| 18309f50db | |||
| 60784e7875 | |||
| f3148bfe52 | |||
| f315620bf7 | |||
| 97357d62b9 | |||
| 0c3651fceb | |||
| 5a5d47d36e |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -22,7 +22,7 @@
|
|||||||
go.work
|
go.work
|
||||||
go.work.sum
|
go.work.sum
|
||||||
|
|
||||||
# Local ReleaseKit executable
|
# Local ReleaseKit delivery credentials
|
||||||
.releasekit/
|
.releasekit/
|
||||||
|
|
||||||
# env file
|
# env file
|
||||||
|
|||||||
76
README.md
76
README.md
@@ -4,7 +4,7 @@ Promptkit is a reusable Go library for preparing and executing prompt-defined
|
|||||||
LLM workflows. Its module path is:
|
LLM workflows. Its module path is:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
gitea.maximumdirect.net/eric/promptkit
|
go.maximumdirect.net/promptkit
|
||||||
```
|
```
|
||||||
|
|
||||||
The root `promptkit` package provides the supported public engine. Consumers
|
The root `promptkit` package provides the supported public engine. Consumers
|
||||||
@@ -21,15 +21,23 @@ Run the maintained offline preparation example from the repository root:
|
|||||||
go run ./examples/go-library/prepare
|
go run ./examples/go-library/prepare
|
||||||
```
|
```
|
||||||
|
|
||||||
It loads a repository-local prompt, supplies a programmatic profile and inline
|
It composes two explicitly selected synthetic collections, imports
|
||||||
input, inspects the captured configuration, and prints deterministic
|
source-relative prompt content, supplies an inline input, and prepares through
|
||||||
preparation metadata without contacting a provider or requiring credentials.
|
a scoped view. The example also demonstrates collection inventories and
|
||||||
|
namespace-filtered discovery, without contacting a provider or requiring
|
||||||
|
credentials.
|
||||||
|
The example loads application-owned settings once and selects no bundled
|
||||||
|
underlay, keeping inventories limited to its own offline fixtures.
|
||||||
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
||||||
ordered file/embedded sources, programmatic mappings, and how to replace an
|
ordered file/embedded sources, programmatic mappings, and how to replace an
|
||||||
engine when source files change. Read the
|
engine when source files change. Applications adapting the prior keyed
|
||||||
[example source](examples/go-library/prepare/main.go), the
|
configuration or versioned-prompt format can consult the
|
||||||
|
[configuration migration guide](docs/consumers/configuration-migration.md).
|
||||||
|
See the [example source](examples/go-library/prepare/main.go), the
|
||||||
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
||||||
[framework format reference](docs/formats.md) to build a consumer workflow.
|
[framework format reference](docs/formats.md) to build a consumer workflow.
|
||||||
|
The consumer guide also covers explicit, live smoke diagnostics and interpreting
|
||||||
|
their factual reports; they are separate from the offline quickstart.
|
||||||
|
|
||||||
Contributors should start with the [development guide](docs/development.md).
|
Contributors should start with the [development guide](docs/development.md).
|
||||||
The [architecture policy](docs/policy/architecture.md) defines the library
|
The [architecture policy](docs/policy/architecture.md) defines the library
|
||||||
@@ -37,6 +45,62 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
The [v0.24.0 release notes](docs/releases/v0.24.0.md) cover application-owned
|
||||||
|
configuration loading, prompt-owned schemas, default output repair, and the
|
||||||
|
prepared message-content hash. Upgrading consumers should review the breaking
|
||||||
|
configuration and request-API changes in the
|
||||||
|
[migration guide](docs/consumers/configuration-migration.md#move-settings-under-promptkit).
|
||||||
|
|
||||||
|
The [v0.23.3 release notes](docs/releases/v0.23.3.md) cover HTTP operation-timeout
|
||||||
|
attribution, custom caller cancellation causes, and response-read error identity.
|
||||||
|
|
||||||
|
The [v0.23.2 release notes](docs/releases/v0.23.2.md) cover configuration,
|
||||||
|
validation, and release-tool dependency updates without an API migration or
|
||||||
|
Go minimum increase.
|
||||||
|
|
||||||
|
The [v0.23.1 release notes](docs/releases/v0.23.1.md) cover corrected constructor
|
||||||
|
error wrapping for downstream classification, with loading diagnostics and
|
||||||
|
underlying causes preserved.
|
||||||
|
|
||||||
|
The [v0.23.0 release notes](docs/releases/v0.23.0.md) cover per-operation
|
||||||
|
timeouts and opt-in prepared-request notifications for ordinary and coordinated
|
||||||
|
execution.
|
||||||
|
|
||||||
|
The [v0.22.0 release notes](docs/releases/v0.22.0.md) cover captured conversation
|
||||||
|
exchanges, application-directed turn coordination, and explicit execution groups.
|
||||||
|
|
||||||
|
The [v0.21.0 release notes](docs/releases/v0.21.0.md) cover ConfigKit v1.2.0
|
||||||
|
and main-relative companion imports through the existing configuration-source
|
||||||
|
API.
|
||||||
|
|
||||||
|
The [v0.20.0 release notes](docs/releases/v0.20.0.md) cover resolved resource
|
||||||
|
inspection, generic JSON/YAML reports, typed prompt sources, and backend
|
||||||
|
inheritance. Upgrading consumers should review the changed inspection result
|
||||||
|
types and profile null-fallback behavior.
|
||||||
|
|
||||||
|
The [v0.19.0 release notes](docs/releases/v0.19.0.md) cover audit remediation
|
||||||
|
and compatibility changes to artifact ownership, programmatic sources, and schemas.
|
||||||
|
|
||||||
|
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
||||||
|
OpenRouter backend and profiles, recorded configuration normalization, and
|
||||||
|
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
||||||
|
the Go 1.27 minimum remains unchanged.
|
||||||
|
|
||||||
|
The [v0.17.0 release notes](docs/releases/v0.17.0.md) cover the vanity module
|
||||||
|
path, ConfigKit v1, identified resource lists, removal of prompt-version
|
||||||
|
metadata, and the Go 1.27 minimum. This is a breaking pre-v1 migration.
|
||||||
|
|
||||||
|
Consumers using the former Go module path should follow the
|
||||||
|
[module-path migration guidance](docs/consumers/configuration-migration.md#update-go-imports).
|
||||||
|
|
||||||
|
The [v0.16.0 release notes](docs/releases/v0.16.0.md) cover explicit smoke
|
||||||
|
diagnostics, configuration collections, scoped engine views, and discovery.
|
||||||
|
Upgrading consumers should also review the current identified-resource and
|
||||||
|
ConfigKit v1 changes in the [migration guide](docs/consumers/configuration-migration.md).
|
||||||
|
|
||||||
|
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
|
||||||
|
validation of concrete requests with `Engine.ValidateRequest`.
|
||||||
|
|
||||||
Consumers upgrading to `v0.12.0` should read the
|
Consumers upgrading to `v0.12.0` should read the
|
||||||
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
||||||
configuration and public construction API migration.
|
configuration and public construction API migration.
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
||||||
|
|||||||
325
application_handoff_integration_test.go
Normal file
325
application_handoff_integration_test.go
Normal file
@@ -0,0 +1,325 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type integrationComponent struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type integrationApplication struct {
|
||||||
|
Version string `yaml:"version"`
|
||||||
|
Sibling string `yaml:"sibling"`
|
||||||
|
Components []integrationComponent `yaml:"components"`
|
||||||
|
Prompts []string `yaml:"prompts"`
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func integrationDefinition(resolves, validates *atomic.Int64) configkit.Definition[integrationApplication] {
|
||||||
|
definition := configkit.Definition[integrationApplication]{
|
||||||
|
Defaults: func() integrationApplication { return integrationApplication{Sibling: "default"} },
|
||||||
|
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(version configkit.Version, validation *configkit.Validation) error {
|
||||||
|
if version.Present && version.Text != "app-v1" {
|
||||||
|
validation.Error(configkit.Keys("version"), "invalid application version")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}},
|
||||||
|
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("components"), "id")},
|
||||||
|
Sensitive: func(path configkit.Path) bool { return path.String() == "/sibling" },
|
||||||
|
Resolve: func(_ *integrationApplication, _ *configkit.Resolution) error { resolves.Add(1); return nil },
|
||||||
|
Validate: func(_ *integrationApplication, validation *configkit.Validation) {
|
||||||
|
validates.Add(1)
|
||||||
|
validation.Warning(configkit.Keys("sibling"), "application warning")
|
||||||
|
},
|
||||||
|
}
|
||||||
|
configured := promptkit.Configure(definition, func(value *integrationApplication) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
previous := configured.Validate
|
||||||
|
configured.Validate = func(value *integrationApplication, validation *configkit.Validation) {
|
||||||
|
previous(value, validation)
|
||||||
|
if len(value.PromptKit.Backends) != 1 || value.PromptKit.Backends[0].Endpoint == nil || strings.Contains(*value.PromptKit.Backends[0].Endpoint, " ") {
|
||||||
|
validation.Error(configkit.Keys("promptkit"), "backend normalization missing")
|
||||||
|
}
|
||||||
|
if value.PromptKit.Application["section_extra"] != "section" || value.PromptKit.Backends[0].Application["backend_extra"] != "backend" || value.PromptKit.Profiles[0].Application["profile_extra"] != "profile" || value.PromptKit.Prompts[0].Application["prompt_extra"] != "prompt" || value.PromptKit.Collections["group"].Application["collection_extra"] != "collection" {
|
||||||
|
validation.Error(configkit.Keys("promptkit"), "application fields missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return configured
|
||||||
|
}
|
||||||
|
|
||||||
|
const integrationDocument = `version: app-v1
|
||||||
|
prompts: [application-only]
|
||||||
|
components: [{id: second-component}]
|
||||||
|
promptkit:
|
||||||
|
section_extra: section
|
||||||
|
backends:
|
||||||
|
- id: local
|
||||||
|
endpoint: ' https://example.test/v1 '
|
||||||
|
extra_params: {provider: {mode: original}}
|
||||||
|
backend_extra: backend
|
||||||
|
profiles:
|
||||||
|
- id: profile
|
||||||
|
backend: local
|
||||||
|
model: original-model
|
||||||
|
profile_extra: profile
|
||||||
|
prompts:
|
||||||
|
- id: example
|
||||||
|
default_profile: profile
|
||||||
|
inputs: [{name: document, required: false}]
|
||||||
|
messages: [{role: user, content: hello}]
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema: {type: object, required: [ok], properties: {ok: {type: boolean}}}
|
||||||
|
prompt_extra: prompt
|
||||||
|
collections:
|
||||||
|
group: {collection_extra: collection}
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestApplicationOwnedLoadAndCapture(t *testing.T) {
|
||||||
|
var resolves, validates atomic.Int64
|
||||||
|
definition := integrationDefinition(&resolves, &validates)
|
||||||
|
result, err := configkit.Load(configkit.Bytes("application", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Config.Sibling != "default" || result.Config.Prompts[0] != "application-only" || len(result.Diagnostics()) != 1 || result.Config.PromptKit.Prompts[0].Output.RepairAttempts == nil || *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("loaded application = %+v, diagnostics = %+v", result.Config, result.Diagnostics())
|
||||||
|
}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"ok":true}`}}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resolves.Load() != 1 || validates.Load() != 1 {
|
||||||
|
t.Fatalf("callbacks rerun during handoff: %d %d", resolves.Load(), validates.Load())
|
||||||
|
}
|
||||||
|
before, err := engine.InspectPrompt(context.Background(), "example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
app := &result.Config.PromptKit
|
||||||
|
*app.Backends[0].Endpoint = "https://changed.test/v1"
|
||||||
|
app.Backends[0].ExtraParams["provider"].(map[string]any)["mode"] = "changed"
|
||||||
|
*app.Profiles[0].Model = "changed-model"
|
||||||
|
*app.Prompts[0].Messages[0].Content = "changed"
|
||||||
|
app.Prompts[0].Inputs[0].Name = "changed"
|
||||||
|
app.Prompts[0].Output.Schema.(map[string]any)["required"].([]any)[0] = "changed"
|
||||||
|
app.Prompts = nil
|
||||||
|
app.Collections = nil
|
||||||
|
after, err := engine.InspectPrompt(context.Background(), "example")
|
||||||
|
if err != nil || !reflect.DeepEqual(before, after) {
|
||||||
|
t.Fatalf("captured prompt changed: %+v, %+v, %v", before, after, err)
|
||||||
|
}
|
||||||
|
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "example"})
|
||||||
|
if err != nil || run.Validation.Status != promptkit.ValidationPassed || len(client.requests) != 1 || client.requests[0].Prompt.Messages[0].Content != "hello" || client.requests[0].Target.Model != "original-model" {
|
||||||
|
t.Fatalf("run = %+v, requests = %+v, error = %v", run, client.requests, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(fmt.Sprintf("%+v", client.requests[0].Target.ExtraParams), "backend_extra") || strings.Contains(fmt.Sprintf("%+v", after), "prompt_extra") {
|
||||||
|
t.Fatal("application fields leaked into resources or provider parameters")
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var report bytes.Buffer
|
||||||
|
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "section_extra: section") || !strings.Contains(report.String(), "prompt_extra: prompt") {
|
||||||
|
t.Fatalf("inspection = %q, %v", report.String(), err)
|
||||||
|
}
|
||||||
|
if len(view.Diagnostics()) != 1 {
|
||||||
|
t.Fatalf("diagnostics = %+v", view.Diagnostics())
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range 4 {
|
||||||
|
wg.Go(func() {
|
||||||
|
_, err := configkit.Load(configkit.Bytes("repeat", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if resolves.Load() != 5 || validates.Load() != 5 {
|
||||||
|
t.Fatalf("reused callbacks = %d %d", resolves.Load(), validates.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandoffMountedSourcesAndExecution(t *testing.T) {
|
||||||
|
baseFS := &countingMountFS{FS: fstest.MapFS{
|
||||||
|
"base/collection.yaml": {Data: []byte("prompts:\n - id: parent\n default_profile: profile\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")},
|
||||||
|
"base/original.txt": {Data: []byte("original")}, "base/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
|
}}
|
||||||
|
operatorFS := &countingMountFS{FS: fstest.MapFS{
|
||||||
|
"operator/item.yaml": {Data: []byte("id: child\nextends: {id: parent}\nmessages: [{role: user, content_file: replacement.txt}]\n")},
|
||||||
|
"operator/replacement.txt": {Data: []byte("replacement")},
|
||||||
|
}}
|
||||||
|
base := configkit.NewFilesystem("embedded", baseFS)
|
||||||
|
operator := configkit.NewFilesystem("operator", operatorFS)
|
||||||
|
collection, err := promptkit.CollectionInput("team", base.File("base/collection.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
collection.Filesystem = &configkit.FilesystemContext{FS: &base, Root: "."}
|
||||||
|
item := configkit.Input{Source: operator.File("operator/item.yaml"), Destination: configkit.Keys("promptkit", "collections", "team", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem, Filesystem: &configkit.FilesystemContext{FS: &operator, Root: "."}}
|
||||||
|
profile := promptkit.SettingsInput(configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.test/v1", "model": "model"}}}))
|
||||||
|
type root struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
layers := []configkit.Underlay{{Inputs: []configkit.Input{profile, collection}}, {Inputs: []configkit.Input{item}}}
|
||||||
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: layers})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
beforeBase, beforeOperator := baseFS.opens, operatorFS.opens
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{}`}}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if baseFS.opens-beforeBase != 1 || operatorFS.opens-beforeOperator != 1 {
|
||||||
|
t.Fatalf("unexpected construction reads: base=%d operator=%d", baseFS.opens-beforeBase, operatorFS.opens-beforeOperator)
|
||||||
|
}
|
||||||
|
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: layers}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantPrompt, err := standalone.InspectPrompt(context.Background(), "team/child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gotPrompt, err := engine.InspectPrompt(context.Background(), "team/child")
|
||||||
|
if err != nil || !reflect.DeepEqual(gotPrompt, wantPrompt) {
|
||||||
|
t.Fatalf("resource reports = %+v, %+v, %v", gotPrompt, wantPrompt, err)
|
||||||
|
}
|
||||||
|
wantInventory, err := standalone.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gotInventory, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || !reflect.DeepEqual(gotInventory, wantInventory) {
|
||||||
|
t.Fatalf("inventories = %+v, %+v, %v", gotInventory, wantInventory, err)
|
||||||
|
}
|
||||||
|
collectionSettings := result.Config.PromptKit.Collections["team"]
|
||||||
|
*collectionSettings.Prompts[1].Output.SchemaFile = "changed.json"
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||||
|
if err != nil || prepared.Messages[0].Content != "replacement" {
|
||||||
|
t.Fatalf("prepared = %+v, %v", prepared, err)
|
||||||
|
}
|
||||||
|
standalonePrepared, err := standalone.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||||
|
if err != nil || !reflect.DeepEqual(prepared.StructuredOutput, standalonePrepared.StructuredOutput) || !reflect.DeepEqual(prepared.OutputContract, standalonePrepared.OutputContract) || !reflect.DeepEqual(prepared.Messages, standalonePrepared.Messages) {
|
||||||
|
t.Fatalf("prepared details differ: %+v, %+v, %v", prepared, standalonePrepared, err)
|
||||||
|
}
|
||||||
|
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||||
|
if err != nil || run.Validation.Status != promptkit.ValidationPassed {
|
||||||
|
t.Fatalf("run = %+v, %v", run, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandoffUsesOnlyBundlesSelectedBeforeLoad(t *testing.T) {
|
||||||
|
type root struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{promptkit.BundledOpenRouterUnderlay()}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil || len(backends) != 1 || backends[0] != "openrouter" {
|
||||||
|
t.Fatalf("bundles = %+v, %v", backends, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationFieldsComposeAndInherit(t *testing.T) {
|
||||||
|
type root struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
underlay := promptkit.SettingsInput(configkit.Mapping("defaults", map[string]any{
|
||||||
|
"section_data": map[string]any{"first": "section"},
|
||||||
|
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.test/v1", "app_data": map[string]any{"first": "backend"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "base", "model": "model", "app_data": map[string]any{"first": "profile"}}},
|
||||||
|
"prompts": []any{map[string]any{"id": "base", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "app_data": map[string]any{"first": "prompt"}}},
|
||||||
|
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"first": "collection"}}},
|
||||||
|
}))
|
||||||
|
main := promptkit.SettingsInput(configkit.Mapping("operator", map[string]any{
|
||||||
|
"section_data": map[string]any{"second": "section"},
|
||||||
|
"backends": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "backend"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "profile"}}},
|
||||||
|
"prompts": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "prompt"}}},
|
||||||
|
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"second": "collection"}}},
|
||||||
|
}))
|
||||||
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
settings := result.Config.PromptKit
|
||||||
|
for name, fields := range map[string]map[string]any{"section": settings.Application, "backend": settings.Backends[1].Application, "profile": settings.Profiles[1].Application, "prompt": settings.Prompts[1].Application, "collection": settings.Collections["team"].Application} {
|
||||||
|
key := "app_data"
|
||||||
|
if name == "section" {
|
||||||
|
key = "section_data"
|
||||||
|
}
|
||||||
|
data, ok := fields[key].(map[string]any)
|
||||||
|
if !ok || data["first"] != name || data["second"] != name {
|
||||||
|
t.Errorf("%s extension = %#v", name, fields[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandoffMissingAndNullSections(t *testing.T) {
|
||||||
|
type noSection struct {
|
||||||
|
Prompts []string `yaml:"prompts"`
|
||||||
|
}
|
||||||
|
missing, err := configkit.Load[noSection](configkit.Bytes("application-only", []byte("prompts: [application]\n")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
empty, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if prompts, err := empty.ListPrompts(context.Background()); err != nil || len(prompts) != 0 {
|
||||||
|
t.Fatalf("application root resources were consumed: %+v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
for _, source := range []string{"{}", "promptkit: null", "promptkit: {}"} {
|
||||||
|
t.Run(source, func(t *testing.T) {
|
||||||
|
type root struct {
|
||||||
|
PromptKit *promptkit.Settings `yaml:"promptkit"`
|
||||||
|
Prompts []string `yaml:"prompts"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return value.PromptKit })
|
||||||
|
result, err := configkit.Load(configkit.Bytes("missing", []byte(source)), configkit.Options[root]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || len(prompts) != 0 {
|
||||||
|
t.Fatalf("prompts = %+v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := promptkit.NewFromConfig(nil, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil result = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
90
application_input_defaults_test.go
Normal file
90
application_input_defaults_test.go
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApplicationDefaultInputInspection(t *testing.T) {
|
||||||
|
for _, derived := range []bool{false, true} {
|
||||||
|
name := "Go defaults"
|
||||||
|
if derived {
|
||||||
|
name = "derived collection defaults"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
settings := promptkit.Settings{
|
||||||
|
Profiles: []promptkit.ProfileSettings{{ID: "profile", Endpoint: new("https://example.test/v1"), Model: new("model")}},
|
||||||
|
Prompts: []promptkit.PromptSettings{{ID: "prompt", DefaultProfile: new("profile"),
|
||||||
|
Inputs: []promptkit.PromptInputSettings{{Name: "doc", Required: true, ContentType: new("text/plain"), Description: new("")}},
|
||||||
|
Messages: []promptkit.PromptMessageSettings{{Role: new("user"), Content: new("hello")}},
|
||||||
|
Output: &promptkit.PromptOutputSettings{Format: new("text"), ValidationMode: new("none")},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
definition := configkit.Definition[handoffApplication]{Defaults: func() handoffApplication { return handoffApplication{PromptKit: settings} }}
|
||||||
|
source := configkit.NoSource()
|
||||||
|
id := "prompt"
|
||||||
|
if derived {
|
||||||
|
definition.Defaults = nil
|
||||||
|
source = configkit.Bytes("derived inputs", []byte(`promptkit:
|
||||||
|
profiles: [{id: profile, endpoint: https://example.test/v1, model: model}]
|
||||||
|
collections:
|
||||||
|
team:
|
||||||
|
prompts:
|
||||||
|
- id: prompt
|
||||||
|
default_profile: profile
|
||||||
|
inputs: [{name: doc}]
|
||||||
|
messages: [{role: user, content: hello}]
|
||||||
|
output: {format: text, validation_mode: none}
|
||||||
|
`))
|
||||||
|
id = "team/prompt"
|
||||||
|
definition.Resolve = func(value *handoffApplication, resolution *configkit.Resolution) error {
|
||||||
|
input := &value.PromptKit.Collections["team"].Prompts[0].Inputs[0]
|
||||||
|
input.Required, input.ContentType, input.Description = true, new("text/plain"), new("")
|
||||||
|
base := configkit.Keys("promptkit", "collections", "team", "prompts").Index(0).Key("inputs").Index(0)
|
||||||
|
for _, field := range []string{"required", "content_type", "description"} {
|
||||||
|
resolution.RecordDerivedDefault(base.Key(field), "Application input default", base.Key("name"))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
definition = promptkit.Configure(definition, func(value *handoffApplication) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
result, err := configkit.Load(source, configkit.Options[handoffApplication]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
resource, err := engine.InspectPrompt(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input := (*resource.Inputs)[0]
|
||||||
|
if input.Required == nil || !*input.Required || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil || *input.Description != "" {
|
||||||
|
t.Fatalf("resource input = %+v", input)
|
||||||
|
}
|
||||||
|
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: id})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
required := requirements.Inputs[0]
|
||||||
|
if required.Kind != promptkit.RequirementRequired || required.ContentType == nil || *required.ContentType != "text/plain" || required.Description == nil || *required.Description != "" {
|
||||||
|
t.Fatalf("requirement = %+v", required)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{PromptID: id}
|
||||||
|
if err := engine.ValidateRequest(ctx, request); !errors.Is(err, promptkit.ErrPromptRender) {
|
||||||
|
t.Fatalf("missing input: %v", err)
|
||||||
|
}
|
||||||
|
request.Inputs = map[string]promptkit.ArtifactRef{"doc": promptkit.Inline("document")}
|
||||||
|
if err := engine.ValidateRequest(ctx, request); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
43
artifact_ownership_contract_test.go
Normal file
43
artifact_ownership_contract_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type independentArtifactReader struct{}
|
||||||
|
|
||||||
|
func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
return &promptkit.Artifact{Body: []byte(ref.Body), Hash: ref.Body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
||||||
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("ownership", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithArtifactReader(independentArtifactReader{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var group sync.WaitGroup
|
||||||
|
for i := range 24 {
|
||||||
|
group.Go(func() {
|
||||||
|
value := fmt.Sprintf("body-%d", i)
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"document": promptkit.Inline(value)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("prepare: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != value || prepared.InputHashes["document"] != value {
|
||||||
|
t.Errorf("prepared = %+v", prepared)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
}
|
||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
artifactadapter "go.maximumdirect.net/promptkit/internal/artifact"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error")
|
var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error")
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) {
|
func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) {
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -71,8 +71,9 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
message := "Hello {{.name}}"
|
message := "Hello {{.name}}"
|
||||||
response := "ok"
|
response := "ok"
|
||||||
output := map[string]any{"format": "text", "validation_mode": "none"}
|
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||||
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||||
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||||
|
var underlays []configkit.Underlay
|
||||||
var expectedMessage string
|
var expectedMessage string
|
||||||
|
|
||||||
switch name {
|
switch name {
|
||||||
@@ -91,29 +92,29 @@ func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFix
|
|||||||
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||||
}
|
}
|
||||||
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||||
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{schemaRootInput(fstest.MapFS{
|
||||||
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||||
}, "."))
|
}, ".")}})
|
||||||
default:
|
default:
|
||||||
b.Fatalf("unknown assembled workflow fixture %q", name)
|
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||||
}
|
}
|
||||||
|
|
||||||
config := map[string]any{
|
config := map[string]any{
|
||||||
"profiles": map[string]any{"profile": profile},
|
"profiles": []any{profile},
|
||||||
"prompts": map[string]any{"workflow": map[string]any{"v1": map[string]any{
|
"prompts": []any{map[string]any{"id": "workflow",
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": message}},
|
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||||
"output": output,
|
"output": output,
|
||||||
}}},
|
}},
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays, Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("assembled workflow benchmark", config),
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||||
}}}, options...)
|
}}}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
request := promptkit.RunRequest{
|
request := promptkit.RunRequest{
|
||||||
PromptID: "workflow", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
|
PromptID: "workflow", Vars: map[string]string{"name": "Ada"},
|
||||||
}
|
}
|
||||||
return assembledWorkflowFixture{
|
return assembledWorkflowFixture{
|
||||||
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
||||||
@@ -127,7 +128,7 @@ func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixtu
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
if prepared.PromptID != "workflow" || prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
|
if prepared.PromptID != "workflow" || len(prepared.Messages) != 1 ||
|
||||||
prepared.Messages[0].Content != fixture.expectedMessage {
|
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||||
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||||
}
|
}
|
||||||
|
|||||||
67
backends.go
67
backends.go
@@ -1,25 +1,31 @@
|
|||||||
package promptkit
|
package promptkit
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BackendOpenRouter identifies the maintained OpenRouter backend.
|
// BackendOpenRouter is the conventional ID for an OpenRouter backend.
|
||||||
const BackendOpenRouter = backend.OpenRouterID
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
// BackendRakestrawHome identifies the maintained Rakestrawhome backend.
|
// BackendRakestrawHome is the conventional ID for a Rakestrawhome backend.
|
||||||
const BackendRakestrawHome = backend.RakestrawHomeID
|
const BackendRakestrawHome = backend.RakestrawHomeID
|
||||||
|
|
||||||
// BackendLocal is a conventional ID for a caller-configured local backend.
|
// BackendLocal is a conventional ID for a caller-configured local backend.
|
||||||
const BackendLocal = "local"
|
const BackendLocal = "local"
|
||||||
|
|
||||||
// BackendConfig is a presence-aware convenience value for a keyed backend
|
// BackendConfig is a presence-aware convenience value for an identified backend
|
||||||
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
||||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||||
// empty map supplies an empty mapping. Use raw ConfigKit Mapping inputs for
|
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
||||||
// explicit null. BackendConfig has no stable JSON representation.
|
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
||||||
|
// inputs for explicit null. Extends names a sibling backend in the same list.
|
||||||
|
// BackendConfig has no stable JSON representation.
|
||||||
type BackendConfig struct {
|
type BackendConfig struct {
|
||||||
|
// ID is the local backend identity within its resource list.
|
||||||
|
ID string
|
||||||
|
// Extends names a sibling backend to inherit through ConfigKit.
|
||||||
|
Extends *string
|
||||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||||
Endpoint *string
|
Endpoint *string
|
||||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||||
@@ -31,17 +37,24 @@ type BackendConfig struct {
|
|||||||
QueueCapacity *int
|
QueueCapacity *int
|
||||||
// ExtraParams supplies JSON-compatible provider request defaults.
|
// ExtraParams supplies JSON-compatible provider request defaults.
|
||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
|
// SmokeTests supplies optional diagnostic defaults and HTTP probe settings.
|
||||||
|
SmokeTests *BackendSmokeTestConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
|
// BackendSource creates a labeled ConfigKit mapping containing backend records
|
||||||
|
// in caller order. IDs are explicit and duplicates are left for loading to reject.
|
||||||
// It encodes fields without validating them; NewEngine validates the final
|
// It encodes fields without validating them; NewEngine validates the final
|
||||||
// effective configuration after all selected source layers are combined.
|
// effective configuration after all selected source layers are combined.
|
||||||
// Mapping captures a copy of supported input values at creation; unsupported
|
// Mapping captures a copy of supported input values at creation; unsupported
|
||||||
// values fail when the source is loaded.
|
// values fail when the source is loaded. ExtraParams must use ConfigKit Mapping
|
||||||
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source {
|
// input forms; callers must explicitly convert numbers or containers as needed.
|
||||||
definitions := make(map[string]any, len(backends))
|
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||||
for id, config := range backends {
|
definitions := make([]any, 0, len(backends))
|
||||||
fields := make(map[string]any)
|
for _, config := range backends {
|
||||||
|
fields := map[string]any{"id": config.ID}
|
||||||
|
if config.Extends != nil {
|
||||||
|
fields["extends"] = map[string]any{"id": *config.Extends}
|
||||||
|
}
|
||||||
if config.Endpoint != nil {
|
if config.Endpoint != nil {
|
||||||
fields["endpoint"] = *config.Endpoint
|
fields["endpoint"] = *config.Endpoint
|
||||||
}
|
}
|
||||||
@@ -55,9 +68,33 @@ func BackendSource(label string, backends map[string]BackendConfig) configkit.So
|
|||||||
fields["queue_capacity"] = *config.QueueCapacity
|
fields["queue_capacity"] = *config.QueueCapacity
|
||||||
}
|
}
|
||||||
if config.ExtraParams != nil {
|
if config.ExtraParams != nil {
|
||||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
fields["extra_params"] = config.ExtraParams
|
||||||
}
|
}
|
||||||
definitions[id] = fields
|
if config.SmokeTests != nil {
|
||||||
|
smokeTests := make(map[string]any)
|
||||||
|
if config.SmokeTests.DefaultProfile != nil {
|
||||||
|
smokeTests["default_profile"] = *config.SmokeTests.DefaultProfile
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Reachability != nil {
|
||||||
|
reachability := make(map[string]any)
|
||||||
|
if config.SmokeTests.Reachability.Endpoint != nil {
|
||||||
|
reachability["endpoint"] = *config.SmokeTests.Reachability.Endpoint
|
||||||
|
}
|
||||||
|
smokeTests["reachability"] = reachability
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication != nil {
|
||||||
|
authentication := make(map[string]any)
|
||||||
|
if config.SmokeTests.Authentication.Kind != nil {
|
||||||
|
authentication["kind"] = *config.SmokeTests.Authentication.Kind
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication.Endpoint != nil {
|
||||||
|
authentication["endpoint"] = *config.SmokeTests.Authentication.Endpoint
|
||||||
|
}
|
||||||
|
smokeTests["authentication"] = authentication
|
||||||
|
}
|
||||||
|
fields["smoke_tests"] = smokeTests
|
||||||
|
}
|
||||||
|
definitions = append(definitions, fields)
|
||||||
}
|
}
|
||||||
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
||||||
}
|
}
|
||||||
|
|||||||
114
bundled_composition_contract_test.go
Normal file
114
bundled_composition_contract_test.go
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
||||||
|
underlay := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller underlay", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}},
|
||||||
|
})}
|
||||||
|
underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
||||||
|
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
|
||||||
|
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
||||||
|
}
|
||||||
|
main := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller main", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "gpt-sol-latest", "model": "main-model"},
|
||||||
|
map[string]any{"id": "custom", "extends": map[string]any{"id": "gpt-luna-latest"}, "model": "custom-model"},
|
||||||
|
map[string]any{"id": "auto", "backend": "other", "model": "other-model"},
|
||||||
|
},
|
||||||
|
"collections": map[string]any{"openrouter": map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://collection.invalid/v1"}},
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "backend": "openrouter/local", "model": "collection-model"}},
|
||||||
|
}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}, Main: &main}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspect := func(id string) *promptkit.ProfileInspection {
|
||||||
|
t.Helper()
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return profile
|
||||||
|
}
|
||||||
|
if target := inspect("gpt-sol-latest"); target.Model == nil || *target.Model != "main-model" || target.Backend != nil {
|
||||||
|
t.Fatalf("replacement = %+v", target)
|
||||||
|
}
|
||||||
|
if target := inspect("custom"); target.Model == nil || *target.Model != "custom-model" || target.Backend == nil || *target.Backend != "openrouter" || target.Endpoint != nil {
|
||||||
|
t.Fatalf("inherited = %+v", target)
|
||||||
|
}
|
||||||
|
if backend, err := engine.InspectBackend(context.Background(), "openrouter"); err != nil || backend.Endpoint != "https://local.invalid/v1" || backend.APIKeyEnv != nil {
|
||||||
|
t.Fatalf("backend replacement = %+v, %v", backend, err)
|
||||||
|
}
|
||||||
|
if target := inspect("openrouter/profile"); target.Backend == nil || *target.Backend != "openrouter/local" || target.Model == nil || *target.Model != "collection-model" {
|
||||||
|
t.Fatalf("collection = %+v", target)
|
||||||
|
}
|
||||||
|
target := inspect("auto-low")
|
||||||
|
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
||||||
|
if target.Backend == nil || *target.Backend != "other" || target.Model == nil || *target.Model != "other-model" || target.ExtraParams == nil || !reflect.DeepEqual(*target.ExtraParams, wantPlugins) {
|
||||||
|
t.Errorf("inherited tier = %+v", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledCompositionSequenceModes(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, mode, id string
|
||||||
|
wantErr bool
|
||||||
|
hasAuto bool
|
||||||
|
}{
|
||||||
|
{"duplicate append", "append", "auto", true, false},
|
||||||
|
{"distinct append", "append", "custom", false, true},
|
||||||
|
{"replace", "replace", "custom", false, false},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("sequence", map[string]any{
|
||||||
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "profiles"}, "mode": test.mode}}},
|
||||||
|
"profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
|
if test.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate rejection")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil || !slices.Contains(profiles, "custom") || slices.Contains(profiles, "auto") != test.hasAuto {
|
||||||
|
t.Fatalf("profiles = %v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) {
|
||||||
|
profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})}
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("strict group accepted duplicate records")
|
||||||
|
}
|
||||||
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replace backends", map[string]any{
|
||||||
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"promptkit", "backends"}, "mode": "replace"}}},
|
||||||
|
"backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
|
})}
|
||||||
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil {
|
||||||
|
t.Fatal("missing bundled backend was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
89
bundled_execution_contract_test.go
Normal file
89
bundled_execution_contract_test.go
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func bundledPromptConfig(client *http.Client) promptkit.Config {
|
||||||
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||||
|
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
}}})}
|
||||||
|
return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledProfilePreparationAndExecution(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
engine, err := promptkit.NewEngine(bundledPromptConfig(nil), promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileRequired) {
|
||||||
|
t.Fatalf("missing profile = %v", err)
|
||||||
|
}
|
||||||
|
temperature := 0.3
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", ProfileID: "gpt-luna-latest", Execution: &promptkit.ExecutionTargetOverride{Temperature: &temperature}}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
details := prepared.Details()
|
||||||
|
if details.SelectedProfileID != "gpt-luna-latest" || details.EffectiveModelParams.Model != "~openai/gpt-luna-latest" || details.EffectiveModelParams.BackendID != "openrouter" || details.EffectiveModelParams.Temperature != temperature {
|
||||||
|
t.Fatalf("prepared target = %+v", details.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(context.Background(), request); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 || client.requests[0].Target.Model != "~openai/gpt-luna-latest" || client.requests[0].Target.BackendID != "openrouter" {
|
||||||
|
t.Fatalf("requests = %+v", client.requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
|
for _, test := range []struct{ id, model, tier string }{
|
||||||
|
{"auto-low", "openrouter/auto", "low"}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
||||||
|
} {
|
||||||
|
t.Run(test.id, func(t *testing.T) {
|
||||||
|
var body map[string]any
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||||
|
t.Errorf("authorization = %q", request.Header.Get("Authorization"))
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(bundledPromptConfig(&http.Client{Transport: transport}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", ProfileID: test.id}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if body["model"] != test.model || body["extra_params"] != nil {
|
||||||
|
t.Fatalf("request = %+v", body)
|
||||||
|
}
|
||||||
|
if test.tier == "" {
|
||||||
|
if _, exists := body["plugins"]; exists {
|
||||||
|
t.Fatalf("unexpected plugins = %+v", body)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
want := []any{map[string]any{"id": "auto-router", "cost_tier": test.tier}}
|
||||||
|
if !reflect.DeepEqual(body["plugins"], want) {
|
||||||
|
t.Fatalf("plugins = %#v, want %#v", body["plugins"], want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
117
bundled_openrouter_contract_test.go
Normal file
117
bundled_openrouter_contract_test.go
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type rejectBundleTransport struct{}
|
||||||
|
|
||||||
|
func (rejectBundleTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||||
|
return nil, errors.New("unexpected HTTP request")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
options []promptkit.Option
|
||||||
|
enabled bool
|
||||||
|
}{
|
||||||
|
{"default", nil, true}, {"enabled", []promptkit.Option{promptkit.WithBundledOpenRouter(true)}, true},
|
||||||
|
{"disabled", []promptkit.Option{promptkit.WithBundledOpenRouter(false)}, false},
|
||||||
|
{"last disables", []promptkit.Option{promptkit.WithBundledOpenRouter(true), promptkit.WithBundledOpenRouter(false)}, false},
|
||||||
|
{"last enables", []promptkit.Option{promptkit.WithBundledOpenRouter(false), promptkit.WithBundledOpenRouter(true)}, true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: rejectBundleTransport{}}}, test.options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
collections, err := engine.ListCollections(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if (len(profiles) > 0) != test.enabled || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
||||||
|
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
||||||
|
}
|
||||||
|
if test.enabled {
|
||||||
|
if !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
|
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
||||||
|
}
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), "auto")
|
||||||
|
if err != nil || profile.Backend == nil || *profile.Backend != "openrouter" {
|
||||||
|
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
|
t.Fatal("bundle source missing")
|
||||||
|
}
|
||||||
|
if explanation, found := view.Explain(configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 {
|
||||||
|
t.Fatalf("bundle explanation = %+v", explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolCount(value bool) int {
|
||||||
|
if value {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
func fmtSources(sources []configkit.SourceInfo) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, source := range sources {
|
||||||
|
b.WriteString(source.Label)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
|
||||||
|
source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}})
|
||||||
|
groups := make([]configkit.Underlay, 1, 2)
|
||||||
|
groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}
|
||||||
|
before := append([]configkit.Underlay(nil), groups...)
|
||||||
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(groups, before) || !reflect.DeepEqual(groups[:2][1], configkit.Underlay{}) {
|
||||||
|
t.Fatal("caller underlays were modified")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) {
|
||||||
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil || !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
|
t.Fatalf("backends = %v, %v", backends, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
181
bundled_smoke_contract_test.go
Normal file
181
bundled_smoke_contract_test.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cancelingAuthenticationBody struct {
|
||||||
|
remaining int
|
||||||
|
cancel context.CancelFunc
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *cancelingAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n := min(len(p), b.remaining)
|
||||||
|
for i := range p[:n] {
|
||||||
|
p[i] = 'x'
|
||||||
|
}
|
||||||
|
b.remaining -= n
|
||||||
|
if b.remaining == 0 {
|
||||||
|
b.cancel()
|
||||||
|
return n, context.Canceled
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
func (b *cancelingAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestAuthenticationReadCancellationWinsOverOverflow(t *testing.T) {
|
||||||
|
for _, size := range []int{1024, (1 << 20) + 1} {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
body := &cancelingAuthenticationBody{remaining: size, cancel: cancel}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter"})
|
||||||
|
cancel()
|
||||||
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != promptkit.ExecutionFailureCanceled || report.Facts.CredentialsAccepted != nil || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || !body.closed {
|
||||||
|
t.Fatalf("size %d: report=%+v error=%v closed=%v", size, report, err, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
|
var urls []string
|
||||||
|
var models []string
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
urls = append(urls, request.URL.String())
|
||||||
|
body := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
|
||||||
|
if request.Method == http.MethodPost {
|
||||||
|
var payload map[string]any
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
models = append(models, payload["model"].(string))
|
||||||
|
content := "PROMPTKIT_OK"
|
||||||
|
if _, structured := payload["response_format"]; structured {
|
||||||
|
content = `{"status":"ok"}`
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body = string(encoded)
|
||||||
|
} else if strings.HasSuffix(request.URL.Path, "/key") && request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||||
|
t.Errorf("authentication header missing")
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, _ := engine.ListProfiles(context.Background())
|
||||||
|
inspectionBefore, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, kind := range []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput} {
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: kind, BackendID: "openrouter"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Failure != "" {
|
||||||
|
t.Fatalf("%s report = %+v, %v", kind, report, err)
|
||||||
|
}
|
||||||
|
if kind == promptkit.SmokeTestAuthentication && (report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted) {
|
||||||
|
t.Fatalf("authentication facts = %+v", report.Facts)
|
||||||
|
}
|
||||||
|
if kind == promptkit.SmokeTestTextInference || kind == promptkit.SmokeTestStructuredOutput {
|
||||||
|
if report.Target.ProfileID != "gpt-luna-latest" || report.Target.EffectiveSettings.Model != "~openai/gpt-luna-latest" {
|
||||||
|
t.Fatalf("target = %+v", report.Target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantURLs := []string{"https://openrouter.ai/api/v1", "https://openrouter.ai/api/v1/key", "https://openrouter.ai/api/v1/chat/completions", "https://openrouter.ai/api/v1/chat/completions"}
|
||||||
|
if !reflect.DeepEqual(urls, wantURLs) || !reflect.DeepEqual(models, []string{"~openai/gpt-luna-latest", "~openai/gpt-luna-latest"}) {
|
||||||
|
t.Fatalf("urls %v, models %v", urls, models)
|
||||||
|
}
|
||||||
|
after, _ := engine.ListProfiles(context.Background())
|
||||||
|
if !reflect.DeepEqual(before, after) {
|
||||||
|
t.Fatalf("inventory changed: %v %v", before, after)
|
||||||
|
}
|
||||||
|
inspectionAfter, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
diff, err := inspectionBefore.Diff(inspectionAfter)
|
||||||
|
if err != nil || len(diff) != 0 {
|
||||||
|
t.Fatalf("inspection changed: %+v, %v", diff, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
||||||
|
replacement := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
|
||||||
|
replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||||
|
replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replacementReport, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||||
|
if err != nil || replacementReport.Target.ProfileID != "gpt-luna-latest" || replacementReport.Target.EffectiveSettings.Model != "replacement" {
|
||||||
|
t.Fatalf("replaced default = %+v, %v", replacementReport, err)
|
||||||
|
}
|
||||||
|
replacementClient.response.Content = "unexpected"
|
||||||
|
mismatch, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||||
|
if err != nil || mismatch == nil || !mismatch.Completed {
|
||||||
|
t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
|
||||||
|
}
|
||||||
|
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("application", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
request promptkit.SmokeTestRequest
|
||||||
|
profile, model string
|
||||||
|
}{
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter"}, "custom", "custom-model"},
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt"}, "free", "openrouter/free"},
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt", ProfileID: "gpt-luna-latest"}, "gpt-luna-latest", "replacement"},
|
||||||
|
} {
|
||||||
|
test.request.Kind = promptkit.SmokeTestTextInference
|
||||||
|
report, err := engine.SmokeTest(context.Background(), test.request)
|
||||||
|
if err != nil || report.Target.ProfileID != test.profile || report.Target.EffectiveSettings.Model != test.model {
|
||||||
|
t.Fatalf("report = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(client.requests[len(client.requests)-1].Prompt.Messages[0].Content, "private application marker") {
|
||||||
|
t.Fatal("application content used by diagnostic")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeAuthenticationResponseOverflow(t *testing.T) {
|
||||||
|
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(strings.Repeat("x", (1<<20)+1)))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter", APIKey: "private-key-marker"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Reason != promptkit.SmokeTestReasonAuthenticationResponseLimitExceeded || report.Failure != promptkit.ExecutionFailureOther || report.Facts.CredentialsAccepted != nil {
|
||||||
|
t.Fatalf("overflow = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,8 +7,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
|
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
|
||||||
@@ -247,11 +247,12 @@ func newBackendCapacityEngine(
|
|||||||
promptID, content = "input-prompt", `{{input "input"}}`
|
promptID, content = "input-prompt", `{{input "input"}}`
|
||||||
inputs = []any{map[string]any{"name": "input", "required": true}}
|
inputs = []any{map[string]any{"name": "input", "required": true}}
|
||||||
}
|
}
|
||||||
backend := map[string]any{"endpoint": "http://backend.example/v1", "concurrency_limit": limit}
|
backend := map[string]any{"id": "limited", "endpoint": "http://backend.example/v1", "concurrency_limit": limit}
|
||||||
if queueCapacity != nil {
|
if queueCapacity != nil {
|
||||||
backend["queue_capacity"] = *queueCapacity
|
backend["queue_capacity"] = *queueCapacity
|
||||||
}
|
}
|
||||||
definition := map[string]any{
|
definition := map[string]any{
|
||||||
|
"id": promptID,
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
@@ -264,10 +265,10 @@ func newBackendCapacityEngine(
|
|||||||
options = append(options, promptkit.WithArtifactReader(reader))
|
options = append(options, promptkit.WithArtifactReader(reader))
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("capacity fixture", map[string]any{
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||||
"backends": map[string]any{"limited": backend},
|
"backends": []any{backend},
|
||||||
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
|
||||||
"prompts": map[string]any{promptID: map[string]any{"1": definition}},
|
"prompts": []any{definition},
|
||||||
}),
|
}),
|
||||||
}}}, options...)
|
}}}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
38
configuration.go
Normal file
38
configuration.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Configure composes PromptKit's settings rules with an application definition.
|
||||||
|
// selectSettings identifies the application's PromptKit section and must be
|
||||||
|
// non-nil. A nil selected section is left untouched. Existing resolution and
|
||||||
|
// validation callbacks run first; callers may wrap the returned callbacks to
|
||||||
|
// run additional work afterward. Defaults and version policy are preserved.
|
||||||
|
func Configure[T any](definition configkit.Definition[T], selectSettings func(*T) *Settings) configkit.Definition[T] {
|
||||||
|
previousResolve := definition.Resolve
|
||||||
|
previousValidate := definition.Validate
|
||||||
|
previousSensitive := definition.Sensitive
|
||||||
|
identities := append([]configkit.SequenceIdentity(nil), definition.SequenceIdentities...)
|
||||||
|
definition.SequenceIdentities = append(identities, configuration.SequenceIdentities()...)
|
||||||
|
definition.Resolve = func(value *T, resolution *configkit.Resolution) error {
|
||||||
|
if previousResolve != nil {
|
||||||
|
if err := previousResolve(value, resolution); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
configuration.Resolve(selectSettings(value), resolution)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
definition.Validate = func(value *T, validation *configkit.Validation) {
|
||||||
|
if previousValidate != nil {
|
||||||
|
previousValidate(value, validation)
|
||||||
|
}
|
||||||
|
configuration.Validate(selectSettings(value), validation)
|
||||||
|
}
|
||||||
|
definition.Sensitive = func(path configkit.Path) bool {
|
||||||
|
return (previousSensitive != nil && previousSensitive(path)) || configuration.Sensitive(path)
|
||||||
|
}
|
||||||
|
return definition
|
||||||
|
}
|
||||||
@@ -5,8 +5,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||||
@@ -21,7 +21,7 @@ func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
|||||||
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||||
}
|
}
|
||||||
difference := differences[0]
|
difference := differences[0]
|
||||||
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
|
if got, want := difference.Path.String(), "/promptkit/profiles"; got != want {
|
||||||
t.Fatalf("difference path = %q, want %q", got, want)
|
t.Fatalf("difference path = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||||
@@ -81,6 +81,34 @@ func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
|
||||||
|
inspect := func(endpoint string) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("backend", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": endpoint}},
|
||||||
|
})}}}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return view
|
||||||
|
}
|
||||||
|
left := inspect(" HTTPS://example.invalid/v1 ")
|
||||||
|
equal := inspect("https://example.invalid/v1")
|
||||||
|
differences, err := left.Diff(equal)
|
||||||
|
if err != nil || len(differences) != 0 {
|
||||||
|
t.Fatalf("normalized diff = %+v, %v", differences, err)
|
||||||
|
}
|
||||||
|
changed := inspect("https://other.invalid/v1")
|
||||||
|
differences, err = left.Diff(changed)
|
||||||
|
if err != nil || len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
|
t.Fatalf("changed diff = %+v, %v", differences, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
"sequence": []any{"earlier", "unchanged"},
|
"sequence": []any{"earlier", "unchanged"},
|
||||||
@@ -94,18 +122,14 @@ func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("compare presence and sequence values: %v", err)
|
t.Fatalf("compare presence and sequence values: %v", err)
|
||||||
}
|
}
|
||||||
if len(differences) != 2 {
|
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
t.Fatalf("atomic backend list difference = %+v", differences)
|
||||||
}
|
}
|
||||||
byPath := differencesByPath(differences)
|
difference := differences[0]
|
||||||
nullable, ok := byPath["/backends/local/extra_params/nullable"]
|
if !difference.Left.Present || !difference.Right.Present ||
|
||||||
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
|
strings.Contains(difference.Left.YAML, "nullable:") || !strings.Contains(difference.Right.YAML, "nullable: null") ||
|
||||||
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
|
!strings.Contains(difference.Left.YAML, "earlier") || !strings.Contains(difference.Right.YAML, "later") {
|
||||||
}
|
t.Fatalf("atomic sequence snapshots = %+v", difference)
|
||||||
sequence, ok := byPath["/backends/local/extra_params/sequence"]
|
|
||||||
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
|
|
||||||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
|
|
||||||
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,22 +147,19 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("compare sensitive values: %v", err)
|
t.Fatalf("compare sensitive values: %v", err)
|
||||||
}
|
}
|
||||||
if len(differences) != 2 {
|
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
t.Fatalf("sensitive backend list difference = %+v", differences)
|
||||||
}
|
}
|
||||||
byPath := differencesByPath(differences)
|
secret := differences[0]
|
||||||
secret, ok := byPath["/backends/local/extra_params/token"]
|
if !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
||||||
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
t.Fatalf("sensitive difference = %+v", secret)
|
||||||
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
|
|
||||||
}
|
}
|
||||||
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
||||||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
||||||
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
||||||
}
|
}
|
||||||
visible, ok := byPath["/backends/local/extra_params/visible"]
|
if !strings.Contains(secret.Left.YAML, "before-value") || !strings.Contains(secret.Right.YAML, "after-value") {
|
||||||
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
|
t.Fatalf("public sibling missing from snapshots: %+v", secret)
|
||||||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
|
|
||||||
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
differences[0].Path = configkit.Keys("caller", "mutation")
|
differences[0].Path = configkit.Keys("caller", "mutation")
|
||||||
@@ -148,18 +169,15 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
||||||
}
|
}
|
||||||
if len(again) != 2 {
|
if len(again) != 1 {
|
||||||
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
|
t.Fatalf("repeat difference count = %d, want 1: %+v", len(again), again)
|
||||||
}
|
}
|
||||||
againByPath := differencesByPath(again)
|
againByPath := differencesByPath(again)
|
||||||
if _, exists := againByPath["/caller/mutation"]; exists {
|
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||||
t.Fatal("mutating returned path reached retained inspection")
|
t.Fatal("mutating returned path reached retained inspection")
|
||||||
}
|
}
|
||||||
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
|
if _, exists := againByPath["/promptkit/backends"]; !exists {
|
||||||
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
|
t.Fatalf("original backend path was lost after mutation: %+v", again)
|
||||||
}
|
|
||||||
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
|
|
||||||
t.Fatalf("original public path was lost after mutation: %+v", again)
|
|
||||||
}
|
}
|
||||||
for _, difference := range again {
|
for _, difference := range again {
|
||||||
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
||||||
@@ -183,7 +201,7 @@ func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("compare sensitive shape replacement: %v", err)
|
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||||
}
|
}
|
||||||
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
|
if len(differences) != 1 || differences[0].Path.String() != "/promptkit/backends" {
|
||||||
t.Fatalf("shape replacement differences = %+v", differences)
|
t.Fatalf("shape replacement differences = %+v", differences)
|
||||||
}
|
}
|
||||||
difference := differences[0]
|
difference := differences[0]
|
||||||
@@ -202,15 +220,15 @@ func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationIns
|
|||||||
|
|
||||||
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
|
backend := map[string]any{"id": "local", "endpoint": "https://example.invalid/v1"}
|
||||||
if extraParams != nil {
|
if extraParams != nil {
|
||||||
backend["extra_params"] = extraParams
|
backend["extra_params"] = extraParams
|
||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping(source, map[string]any{
|
||||||
"backends": map[string]any{"local": backend},
|
"backends": []any{backend},
|
||||||
"profiles": map[string]any{"offline": map[string]any{
|
"profiles": []any{map[string]any{
|
||||||
"backend": "local",
|
"id": "offline", "backend": "local",
|
||||||
"model": model,
|
"model": model,
|
||||||
}},
|
}},
|
||||||
})}}}},
|
})}}}},
|
||||||
|
|||||||
76
configuration_handoff_test.go
Normal file
76
configuration_handoff_test.go
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type handoffApplication struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type handoffWrapper struct {
|
||||||
|
result *configkit.Result[handoffApplication]
|
||||||
|
override any
|
||||||
|
useOverride bool
|
||||||
|
inspection bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *handoffWrapper) Value(path configkit.Path) (any, bool) {
|
||||||
|
if w.useOverride {
|
||||||
|
return w.override, true
|
||||||
|
}
|
||||||
|
return w.result.Value(path)
|
||||||
|
}
|
||||||
|
func (w *handoffWrapper) InspectSubtree(path configkit.Path) (configkit.Inspection, bool) {
|
||||||
|
if !w.inspection {
|
||||||
|
return configkit.Inspection{}, false
|
||||||
|
}
|
||||||
|
return w.result.InspectSubtree(path)
|
||||||
|
}
|
||||||
|
func (w *handoffWrapper) Diagnostics() []configkit.Diagnostic { return w.result.Diagnostics() }
|
||||||
|
|
||||||
|
func TestConstructorsShareCapturedResources(t *testing.T) {
|
||||||
|
source := configkit.Bytes("prompt", []byte("promptkit:\n prompts:\n - id: example\n messages: [{role: user, content: hello}]\n output: {format: text, validation_mode: basic}\n"))
|
||||||
|
definition := promptkit.Configure(configkit.Definition[handoffApplication]{}, func(value *handoffApplication) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
result, err := configkit.Load(source, configkit.Options[handoffApplication]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wrapped := &handoffWrapper{result: result, inspection: true}
|
||||||
|
applicationEngine, err := promptkit.NewFromConfig(wrapped, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(true))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input := configkit.Input{Source: source}
|
||||||
|
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, engine := range []*promptkit.Engine{applicationEngine, standalone} {
|
||||||
|
prompt, err := engine.InspectPrompt(context.Background(), "example")
|
||||||
|
if err != nil || prompt.Messages[0].Content != "hello" || prompt.OutputContract.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("prompt = %+v, %v", prompt, err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil || len(backends) != 0 {
|
||||||
|
t.Fatalf("backends = %+v, %v", backends, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewFromConfigRejectsInvalidHandoff(t *testing.T) {
|
||||||
|
var nilResult *handoffWrapper
|
||||||
|
if _, err := promptkit.NewFromConfig(nilResult, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil result: %v", err)
|
||||||
|
}
|
||||||
|
for _, wrapper := range []*handoffWrapper{{inspection: false}, {inspection: true, useOverride: true, override: "wrong"}} {
|
||||||
|
if _, err := promptkit.NewFromConfig(wrapper, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("invalid handoff: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,13 +4,15 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ConfigurationInspection is a read-only view of one Engine's captured
|
// ConfigurationInspection is a read-only view of one Engine's captured
|
||||||
// effective configuration and source history. It does not expose typed
|
// effective configuration and source history focused on /promptkit. Metadata,
|
||||||
// configuration values, filesystem contexts, or runtime credentials.
|
// sources, and explicit path queries retain native whole-load evidence; the
|
||||||
|
// focus is not an isolation boundary. It does not expose typed configuration
|
||||||
|
// values, filesystem contexts, or runtime credentials.
|
||||||
// Report content may include application prompt and provider data, so callers
|
// Report content may include application prompt and provider data, so callers
|
||||||
// choose where and to whom to disclose it.
|
// choose where and to whom to disclose it.
|
||||||
type ConfigurationInspection struct {
|
type ConfigurationInspection struct {
|
||||||
@@ -30,7 +32,9 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
|||||||
// Diff compares this view's captured effective configuration with other. This
|
// Diff compares this view's captured effective configuration with other. This
|
||||||
// view is the earlier (left) value and other is the later (right) value. The
|
// view is the earlier (left) value and other is the later (right) value. The
|
||||||
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||||
// source or resolution history alone does not create differences. It does not
|
// source or resolution history alone does not create differences. Native
|
||||||
|
// ancestor differences are retained, while sibling-only records are excluded.
|
||||||
|
// It does not
|
||||||
// reread external sources or compare referenced file contents, resolved
|
// reread external sources or compare referenced file contents, resolved
|
||||||
// credential contents, later execution preparation, or request-specific
|
// credential contents, later execution preparation, or request-specific
|
||||||
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||||
@@ -52,6 +56,14 @@ func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
|||||||
return v.source.Metadata()
|
return v.source.Metadata()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HistoricalPaths lists native historical paths within the PromptKit focus.
|
||||||
|
func (v ConfigurationInspection) HistoricalPaths() []configkit.Path {
|
||||||
|
if v.source == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return v.source.HistoricalPaths()
|
||||||
|
}
|
||||||
|
|
||||||
// Sources returns a detached list of loaded source descriptions.
|
// Sources returns a detached list of loaded source descriptions.
|
||||||
func (v ConfigurationInspection) Sources() []configkit.SourceInfo {
|
func (v ConfigurationInspection) Sources() []configkit.SourceInfo {
|
||||||
if v.source == nil {
|
if v.source == nil {
|
||||||
@@ -78,7 +90,8 @@ func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explana
|
|||||||
return v.source.Explain(path)
|
return v.source.Explain(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteEffective writes ConfigKit's redacted effective YAML. Writer errors
|
// WriteEffective writes the focused ConfigKit-redacted effective YAML. An absent
|
||||||
|
// effective focus fails without output; explicit null renders normally. Writer errors
|
||||||
// remain available through errors.Is and errors.As. The zero view returns
|
// remain available through errors.Is and errors.As. The zero view returns
|
||||||
// ErrInvalidConfig.
|
// ErrInvalidConfig.
|
||||||
func (v ConfigurationInspection) WriteEffective(w io.Writer) error {
|
func (v ConfigurationInspection) WriteEffective(w io.Writer) error {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -11,28 +12,37 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{
|
Underlays: []configkit.Underlay{
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("defaults", map[string]any{
|
||||||
"backends": map[string]any{"local": map[string]any{
|
"backends": []any{map[string]any{
|
||||||
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY",
|
||||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||||
}},
|
}},
|
||||||
"profiles": map[string]any{
|
"collections": map[string]any{"literal": map[string]any{
|
||||||
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
"backends": []any{
|
||||||
"literal/id": map[string]any{"extends": "base.v1"},
|
map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "namespaced-private-sentinel"}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "local"}},
|
||||||
},
|
},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "base.v1", "backend": "local", "model": "default-model"},
|
||||||
|
map[string]any{"id": "id", "extends": map[string]any{"id": "base.v1"}},
|
||||||
|
},
|
||||||
|
}},
|
||||||
})}}},
|
})}}},
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("operator", map[string]any{
|
||||||
"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}},
|
"collections": map[string]any{"literal": map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
|
||||||
|
}},
|
||||||
})}}},
|
})}}},
|
||||||
},
|
},
|
||||||
}})
|
}}, promptkit.WithBundledOpenRouter(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct layered engine: %v", err)
|
t.Fatalf("construct layered engine: %v", err)
|
||||||
}
|
}
|
||||||
@@ -40,24 +50,43 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
modelPath := configkit.Keys("profiles", "base.v1", "model")
|
modelPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(0).Key("model")
|
||||||
model, found := view.Explain(modelPath)
|
model, found := view.Explain(modelPath)
|
||||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
|
||||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||||
}
|
}
|
||||||
inheritedPath := configkit.Keys("profiles", "literal/id", "model")
|
inheritedPath := configkit.Keys("promptkit", "collections", "literal", "profiles").Index(1).Key("model")
|
||||||
inherited, found := view.Explain(inheritedPath)
|
inherited, found := view.Explain(inheritedPath)
|
||||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||||
}
|
}
|
||||||
if inheritedPath.String() != "/profiles/literal~1id/model" {
|
if inheritedPath.String() != "/promptkit/collections/literal/profiles/1/model" {
|
||||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||||
}
|
}
|
||||||
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
secretPath := configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("token")
|
||||||
|
endpointPath := configkit.Keys("promptkit", "backends").Index(0).Key("endpoint")
|
||||||
|
endpoint, found := view.Explain(endpointPath)
|
||||||
|
if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") {
|
||||||
|
t.Fatalf("normalized endpoint explanation = %+v", endpoint)
|
||||||
|
}
|
||||||
secret, found := view.Explain(secretPath)
|
secret, found := view.Explain(secretPath)
|
||||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||||
}
|
}
|
||||||
|
namespacedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
|
||||||
|
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
||||||
|
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
||||||
|
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
||||||
|
}
|
||||||
|
inheritedSecretPath := configkit.Keys("promptkit", "collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
|
||||||
|
inheritedSecret, found := view.Explain(inheritedSecretPath)
|
||||||
|
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
|
||||||
|
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
|
||||||
|
}
|
||||||
|
metadata := fmt.Sprintf("%+v", view.Metadata().History(inheritedSecretPath))
|
||||||
|
if strings.Contains(metadata, "replacement-private-sentinel") || strings.Contains(metadata, "namespaced-private-sentinel") {
|
||||||
|
t.Fatal("metadata disclosed a sensitive provider value")
|
||||||
|
}
|
||||||
var effective bytes.Buffer
|
var effective bytes.Buffer
|
||||||
if err := view.WriteEffective(&effective); err != nil {
|
if err := view.WriteEffective(&effective); err != nil {
|
||||||
t.Fatalf("write effective configuration: %v", err)
|
t.Fatalf("write effective configuration: %v", err)
|
||||||
@@ -66,7 +95,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
t.Fatalf("unexpected effective report: %s", effective.String())
|
t.Fatalf("unexpected effective report: %s", effective.String())
|
||||||
}
|
}
|
||||||
var explanation bytes.Buffer
|
var explanation bytes.Buffer
|
||||||
if err := view.WriteExplanation(&explanation, secretPath); err != nil || strings.Contains(explanation.String(), "private-sentinel") {
|
if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") {
|
||||||
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,6 +122,45 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid backend", map[string]any{
|
||||||
|
"collections": map[string]any{"private": map[string]any{"backends": []any{
|
||||||
|
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
|
||||||
|
}}},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
var loadErr *configkit.LoadError
|
||||||
|
if !errors.As(err, &loadErr) {
|
||||||
|
t.Fatalf("invalid backend error = %v, want load error", err)
|
||||||
|
}
|
||||||
|
if text := fmt.Sprintf("%v %+v", err, loadErr.Diagnostics()); strings.Contains(text, "failure-private-sentinel") {
|
||||||
|
t.Fatalf("failure diagnostics disclosed a sensitive value: %s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionRetainsApplicationVersionField(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("main", map[string]any{"version": "2"}),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with accepted declaration: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
|
}
|
||||||
|
path := configkit.Keys("promptkit", "version")
|
||||||
|
field, found := inspection.Explain(path)
|
||||||
|
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
|
||||||
|
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
||||||
|
}
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") {
|
||||||
|
t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type failingInspectionWriter struct{ err error }
|
type failingInspectionWriter struct{ err error }
|
||||||
|
|
||||||
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
||||||
@@ -103,7 +171,7 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
t.Fatalf("nil engine inspection = %v", err)
|
t.Fatalf("nil engine inspection = %v", err)
|
||||||
}
|
}
|
||||||
var empty promptkit.ConfigurationInspection
|
var empty promptkit.ConfigurationInspection
|
||||||
path := configkit.Keys("profiles", "missing")
|
path := configkit.Keys("promptkit", "profiles", "missing")
|
||||||
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
|
||||||
t.Fatal("zero inspection reported configuration")
|
t.Fatal("zero inspection reported configuration")
|
||||||
}
|
}
|
||||||
@@ -117,7 +185,12 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
t.Fatalf("zero explanation writer = %v", err)
|
t.Fatalf("zero explanation writer = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
valid, err := promptkit.NewEngine(promptkit.Config{})
|
profile := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("inspection test profile", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
|
||||||
|
})}
|
||||||
|
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}},
|
||||||
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
@@ -130,14 +203,15 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("effective writer error = %v", err)
|
t.Fatalf("effective writer error = %v", err)
|
||||||
}
|
}
|
||||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) {
|
if err := view.WriteExplanation(writer, configkit.Keys("promptkit", "profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("explanation writer error = %v", err)
|
t.Fatalf("explanation writer error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
||||||
|
const promptDocument = "prompts:\n - id: captured\n default_profile: offline\n messages:\n - role: user\n content_file: body.tmpl\n output:\n format: text\n validation_mode: none\n"
|
||||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
|
||||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||||
}}
|
}}
|
||||||
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
||||||
@@ -145,7 +219,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatalf("select inputs: %v", err)
|
t.Fatalf("select inputs: %v", err)
|
||||||
}
|
}
|
||||||
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
|
||||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||||
}}
|
}}
|
||||||
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||||
@@ -156,16 +230,16 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
client := countingLLMClient{calls: &modelCalls}
|
client := countingLLMClient{calls: &modelCalls}
|
||||||
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(inputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("offline profile", map[string]any{
|
||||||
"offline": {Endpoint: &endpoint, Model: &model},
|
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
|
||||||
})}}}},
|
})}}}},
|
||||||
}}, promptkit.WithLLMClient(client))
|
}}, promptkit.WithLLMClient(client))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: mountSettingsInputs(otherInputs)}, {Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("comparison profile", map[string]any{
|
||||||
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
|
||||||
})}}}},
|
})}}}},
|
||||||
}}, promptkit.WithLLMClient(client))
|
}}, promptkit.WithLLMClient(client))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -187,7 +261,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err != nil || len(differences) != 1 {
|
if err != nil || len(differences) != 1 {
|
||||||
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||||
}
|
}
|
||||||
if differences[0].Path.String() != "/profiles/offline/model" ||
|
if differences[0].Path.String() != "/promptkit/profiles" ||
|
||||||
!strings.Contains(differences[0].Left.YAML, model) ||
|
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||||
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||||
t.Fatalf("captured difference = %+v", differences[0])
|
t.Fatalf("captured difference = %+v", differences[0])
|
||||||
@@ -199,7 +273,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
if _, found := view.Explain(configkit.Keys("promptkit", "prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
|
||||||
t.Fatal("captured prompt content path was not inspectable")
|
t.Fatal("captured prompt content path was not inspectable")
|
||||||
}
|
}
|
||||||
var report bytes.Buffer
|
var report bytes.Buffer
|
||||||
|
|||||||
60
configuration_mounts_test.go
Normal file
60
configuration_mounts_test.go
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type mountApplication struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingMountFS struct {
|
||||||
|
fs.FS
|
||||||
|
opens int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingMountFS) Open(name string) (fs.File, error) { c.opens++; return c.FS.Open(name) }
|
||||||
|
|
||||||
|
func TestSettingsAndCollectionMounts(t *testing.T) {
|
||||||
|
files := &countingMountFS{FS: fstest.MapFS{
|
||||||
|
"fragments/root.yaml": {Data: []byte("profiles:\n - id: main\n model: model\n")},
|
||||||
|
"fragments/collection.yaml": {Data: []byte("prompts:\n - id: child\n messages: [{role: user, content_file: body.txt}]\n output: {format: text, validation_mode: basic}\n")},
|
||||||
|
"fragments/item.yaml": {Data: []byte("id: item\nmessages: [{role: user, content: hello}]\noutput: {format: text, validation_mode: basic}\n")},
|
||||||
|
"fragments/body.txt": {Data: []byte("message")},
|
||||||
|
}}
|
||||||
|
namespace := configkit.NewFilesystem("mounts", files)
|
||||||
|
root := promptkit.SettingsInput(namespace.File("fragments/root.yaml"))
|
||||||
|
collection, err := promptkit.CollectionInput("docs", namespace.File("fragments/collection.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
item := configkit.Input{Source: namespace.File("fragments/item.yaml"), Destination: configkit.Keys("promptkit", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem}
|
||||||
|
for _, input := range []*configkit.Input{&root, &collection, &item} {
|
||||||
|
input.Filesystem = &configkit.FilesystemContext{FS: &namespace, Root: "."}
|
||||||
|
}
|
||||||
|
bundled := promptkit.BundledOpenRouterUnderlay()
|
||||||
|
if files.opens != 0 || root.Imports || !collection.Imports || root.Destination.String() != "/promptkit" || collection.Destination.String() != "/promptkit/collections/docs" {
|
||||||
|
t.Fatalf("mount construction read files or chose wrong defaults: opens=%d", files.opens)
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[mountApplication]{}, func(value *mountApplication) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[mountApplication]{Definition: definition, Underlays: []configkit.Underlay{bundled, {Inputs: []configkit.Input{root, collection, item}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(result.Config.PromptKit.Profiles) < 1 || len(result.Config.PromptKit.Prompts) != 1 || len(result.Config.PromptKit.Collections["docs"].Prompts) != 1 {
|
||||||
|
t.Fatalf("mounted settings = %+v", result.Config.PromptKit)
|
||||||
|
}
|
||||||
|
info, ok := result.Metadata().Lookup(configkit.Keys("promptkit", "collections", "docs", "prompts").Index(0).Key("messages").Index(0).Key("content_file"))
|
||||||
|
if !ok || len(info.Origin.Locations) != 1 {
|
||||||
|
t.Fatalf("origin = %+v", info)
|
||||||
|
}
|
||||||
|
context, ok := result.SourceContext(info.Origin.Locations[0].SourceID)
|
||||||
|
if !ok || context.FS == nil || context.BaseDir != "fragments" {
|
||||||
|
t.Fatalf("context = %+v", context)
|
||||||
|
}
|
||||||
|
}
|
||||||
99
configuration_rules_test.go
Normal file
99
configuration_rules_test.go
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type rulesApplication struct {
|
||||||
|
Version string `yaml:"version"`
|
||||||
|
Other []rulesOther `yaml:"other"`
|
||||||
|
PromptKit *promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type rulesOther struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigureComposesApplicationRules(t *testing.T) {
|
||||||
|
var events []string
|
||||||
|
definition := configkit.Definition[rulesApplication]{
|
||||||
|
Defaults: func() rulesApplication { return rulesApplication{Version: "app"} },
|
||||||
|
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(v configkit.Version, validation *configkit.Validation) error {
|
||||||
|
events = append(events, "version")
|
||||||
|
return nil
|
||||||
|
}},
|
||||||
|
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("other"), "id")},
|
||||||
|
Resolve: func(_ *rulesApplication, _ *configkit.Resolution) error {
|
||||||
|
events = append(events, "application resolve")
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Validate: func(_ *rulesApplication, _ *configkit.Validation) { events = append(events, "application validate") },
|
||||||
|
Sensitive: func(path configkit.Path) bool { return path.String() == "/application_secret" },
|
||||||
|
}
|
||||||
|
configured := promptkit.Configure(definition, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
|
||||||
|
if len(definition.SequenceIdentities) != 1 || len(configured.SequenceIdentities) != 7 {
|
||||||
|
t.Fatalf("registrations: %d, %d", len(definition.SequenceIdentities), len(configured.SequenceIdentities))
|
||||||
|
}
|
||||||
|
priorValidate := configured.Validate
|
||||||
|
configured.Validate = func(value *rulesApplication, validation *configkit.Validation) {
|
||||||
|
priorValidate(value, validation)
|
||||||
|
if value.PromptKit == nil || value.PromptKit.Backends[0].Endpoint == nil || strings.TrimSpace(*value.PromptKit.Backends[0].Endpoint) != *value.PromptKit.Backends[0].Endpoint {
|
||||||
|
validation.Error(configkit.Keys("promptkit"), "normalization did not run")
|
||||||
|
}
|
||||||
|
events = append(events, "after validate")
|
||||||
|
}
|
||||||
|
source := configkit.Bytes("rules", []byte("promptkit:\n backends:\n - id: local\n endpoint: ' https://example.test/v1/ '\n extra_params: {api_key: hidden}\nother: [{id: one}]\n"))
|
||||||
|
result, err := configkit.Load(source, configkit.Options[rulesApplication]{Definition: configured})
|
||||||
|
if err != nil {
|
||||||
|
var loadError *configkit.LoadError
|
||||||
|
if errors.As(err, &loadError) {
|
||||||
|
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
|
||||||
|
}
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Config.Version != "app" || strings.Join(events, ",") != "version,application resolve,application validate,after validate" {
|
||||||
|
t.Fatalf("callbacks: %+v, %+v", result.Config, events)
|
||||||
|
}
|
||||||
|
if !configured.Sensitive(configkit.Keys("application_secret")) || !configured.Sensitive(configkit.Keys("promptkit", "backends").Index(0).Key("extra_params").Key("api_key")) || configured.Sensitive(configkit.Keys("backends").Index(0).Key("extra_params").Key("api_key")) {
|
||||||
|
t.Fatal("sensitivity composition failed")
|
||||||
|
}
|
||||||
|
var rendered strings.Builder
|
||||||
|
if err := result.WriteEffective(&rendered); err != nil || strings.Contains(rendered.String(), "hidden") {
|
||||||
|
t.Fatalf("redaction: %q, %v", rendered.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurePropagatesResolveError(t *testing.T) {
|
||||||
|
want := errors.New("resolve failed")
|
||||||
|
definition := promptkit.Configure(configkit.Definition[rulesApplication]{Resolve: func(*rulesApplication, *configkit.Resolution) error { return want }}, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
|
||||||
|
err := definition.Resolve(&rulesApplication{}, nil)
|
||||||
|
if !errors.Is(err, want) {
|
||||||
|
t.Fatalf("resolve error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigureReusableAcrossConcurrentLoads(t *testing.T) {
|
||||||
|
definition := promptkit.Configure(configkit.Definition[rulesApplication]{}, func(v *rulesApplication) *promptkit.Settings { return v.PromptKit })
|
||||||
|
var group sync.WaitGroup
|
||||||
|
for range 8 {
|
||||||
|
group.Go(func() {
|
||||||
|
for _, source := range []string{"{}", "promptkit: {prompts: [{id: test, output: {format: text, validation_mode: basic}}]}"} {
|
||||||
|
result, err := configkit.Load(configkit.Bytes("settings", []byte(source)), configkit.Options[rulesApplication]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if result.Config.PromptKit != nil && *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
|
||||||
|
t.Error("default was not resolved")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
}
|
||||||
@@ -7,10 +7,40 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/filecatalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CollectionInput mounts source at the selected collection's configuration
|
||||||
|
// destination, /promptkit/collections/<namespace>, and enables ConfigKit imports for that
|
||||||
|
// input. An invalid namespace returns an error matching ErrInvalidConfig. The
|
||||||
|
// source is not read or validated until ConfigKit loads it. Callers may adjust
|
||||||
|
// the returned input, including its filesystem context or imports setting.
|
||||||
|
func CollectionInput(namespace string, source configkit.Source) (configkit.Input, error) {
|
||||||
|
if !domain.IsValidResourceNamespace(namespace) {
|
||||||
|
return configkit.Input{}, fmt.Errorf("%w: invalid collection namespace %q", ErrInvalidConfig, namespace)
|
||||||
|
}
|
||||||
|
return configkit.Input{
|
||||||
|
Source: source,
|
||||||
|
Destination: configkit.Keys("promptkit", "collections", namespace),
|
||||||
|
Imports: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SettingsInput mounts a wrapperless PromptKit settings fragment at /promptkit.
|
||||||
|
// It does not acquire the source; callers may adjust the returned input.
|
||||||
|
func SettingsInput(source configkit.Source) configkit.Input {
|
||||||
|
return configkit.Input{Source: source, Destination: configkit.Keys("promptkit")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BundledOpenRouterUnderlay selects the bundled OpenRouter settings fragment.
|
||||||
|
// It does not acquire its source until ConfigKit loads it.
|
||||||
|
func BundledOpenRouterUnderlay() configkit.Underlay {
|
||||||
|
return configuration.BundledOpenRouterUnderlay()
|
||||||
|
}
|
||||||
|
|
||||||
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
||||||
// directory as one strict-composition group. The returned order is deterministic;
|
// directory as one strict-composition group. The returned order is deterministic;
|
||||||
// it does not assign precedence among those files. Discovery is recursive over
|
// it does not assign precedence among those files. Discovery is recursive over
|
||||||
|
|||||||
@@ -1,14 +1,16 @@
|
|||||||
package promptkit_test
|
package promptkit_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type sourceDocument struct {
|
type sourceDocument struct {
|
||||||
@@ -16,6 +18,119 @@ type sourceDocument struct {
|
|||||||
Second string `yaml:"second"`
|
Second string `yaml:"second"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
|
||||||
|
input, err := promptkit.CollectionInput("docs", configkit.File(filepath.Join(t.TempDir(), "not-read.yaml")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if input.Destination.String() != "/promptkit/collections/docs" || !input.Imports {
|
||||||
|
t.Fatalf("collection input = %+v, want destination /promptkit/collections/docs with imports enabled", input)
|
||||||
|
}
|
||||||
|
for _, namespace := range []string{"", " ", "docs/api"} {
|
||||||
|
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Errorf("namespace %q error = %v, want ErrInvalidConfig", namespace, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
mainPath := filepath.Join(directory, "collection.yaml")
|
||||||
|
defaultsPath := filepath.Join(directory, "defaults.yaml")
|
||||||
|
if err := os.WriteFile(mainPath, []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(defaultsPath, []byte("prompts:\n - id: greeting\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input, err := promptkit.CollectionInput("docs", configkit.File(mainPath))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
|
||||||
|
Destination: configkit.Keys("promptkit", "collections", "docs"),
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with native collection import: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputUsesMainRelativeContext(t *testing.T) {
|
||||||
|
filesystem := configkit.NewFilesystem("application", fstest.MapFS{
|
||||||
|
"site/main.yaml": {Data: []byte("version: '2'\n")},
|
||||||
|
"site/prompts/greeting.yaml": {Data: []byte(`id: greeting
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: greeting.txt
|
||||||
|
output: {format: text, validation_mode: none}
|
||||||
|
`)},
|
||||||
|
"site/prompts/greeting.txt": {Data: []byte("Hello from the imported prompt's directory.")},
|
||||||
|
})
|
||||||
|
companion, err := promptkit.CollectionInput("support", configkit.Bytes("companion prompts", []byte(`
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- dir: prompts
|
||||||
|
pattern: ["*.yaml", "*.yml"]
|
||||||
|
destination: [prompts]
|
||||||
|
placement: sequence-item
|
||||||
|
required: true
|
||||||
|
`)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
companion.MainRelative = configkit.MainRelative()
|
||||||
|
main := configkit.Input{Source: filesystem.File("site/main.yaml")}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Main: &main,
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{companion}}},
|
||||||
|
}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with main-relative imports: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(t.Context(), "support/greeting")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(inspection.Messages) != 1 || inspection.Messages[0].Content != "Hello from the imported prompt's directory." {
|
||||||
|
t.Fatalf("unexpected imported messages: %+v", inspection.Messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"collection.yaml": {Data: []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n")},
|
||||||
|
"defaults.yaml": {Data: []byte("prompts:\n - id: greeting\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n")},
|
||||||
|
}
|
||||||
|
filesystem := configkit.NewFilesystem("embedded collection", files)
|
||||||
|
input, err := promptkit.CollectionInput("docs", filesystem.File("collection.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: "."}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
|
||||||
|
Destination: configkit.Keys("promptkit", "collections", "docs"),
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with embedded collection import: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package promptkit_test
|
package promptkit_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
@@ -11,8 +12,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||||
@@ -37,18 +38,19 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
cfg := simpleConfig("hello")
|
cfg := simpleConfig("hello")
|
||||||
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
|
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
|
||||||
|
endpoint, model := "http://example.test/v1", "model"
|
||||||
lowerName := name
|
lowerName := name
|
||||||
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
Source: promptkit.ProfileSource("default credentials", map[string]promptkit.ProfileConfig{
|
Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
|
||||||
"profile": {APIKeyEnv: &lowerName},
|
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
|
||||||
}),
|
}),
|
||||||
}}})
|
}}})
|
||||||
env := " " + name + " "
|
env := " " + name + " "
|
||||||
if test.clear {
|
if test.clear {
|
||||||
env = ""
|
env = ""
|
||||||
}
|
}
|
||||||
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", map[string]promptkit.ProfileConfig{
|
cfg.Sources.Main = &configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
|
||||||
"profile": {APIKeyEnv: &env},
|
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
|
||||||
})}
|
})}
|
||||||
var calls int
|
var calls int
|
||||||
var auth string
|
var auth string
|
||||||
@@ -71,21 +73,24 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
|
|
||||||
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
||||||
backendID, model := "workstation", "offline-model"
|
backendID, model := "workstation", "offline-model"
|
||||||
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
prompt := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("prompt", map[string]any{
|
||||||
"prompts": map[string]any{"hello": map[string]any{"1": map[string]any{
|
"prompts": []any{map[string]any{
|
||||||
|
"id": "hello",
|
||||||
"default_profile": "offline",
|
"default_profile": "offline",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}}},
|
}},
|
||||||
})}
|
})}
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
||||||
|
localBackend := promptkit.LocalBackend("http://localhost:8000/v1", 2)
|
||||||
|
localBackend.ID = backendID
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
{Source: promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
|
||||||
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
localBackend,
|
||||||
})},
|
})},
|
||||||
{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||||
"offline": {Backend: &backendID, Model: &model},
|
{ID: "offline", Backend: &backendID, Model: &model},
|
||||||
})},
|
})},
|
||||||
}}},
|
}}},
|
||||||
Main: &prompt,
|
Main: &prompt,
|
||||||
@@ -105,16 +110,45 @@ func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
||||||
|
openRouterID, rakestrawhomeID := promptkit.BackendOpenRouter, promptkit.BackendRakestrawHome
|
||||||
|
openRouterEndpoint, rakestrawhomeEndpoint := "https://openrouter.example/v1", "https://inference.example/v1"
|
||||||
|
model := "configured-model"
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
|
||||||
|
{ID: openRouterID, Endpoint: &openRouterEndpoint},
|
||||||
|
{ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint},
|
||||||
|
})},
|
||||||
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
|
||||||
|
{ID: "openrouter-profile", Backend: &openRouterID, Model: &model},
|
||||||
|
{ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model},
|
||||||
|
})},
|
||||||
|
}}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with explicitly configured provider IDs: %v", err)
|
||||||
|
}
|
||||||
|
for profileID, backendID := range map[string]string{
|
||||||
|
"openrouter-profile": openRouterID,
|
||||||
|
"rakestrawhome-profile": rakestrawhomeID,
|
||||||
|
} {
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||||
|
if err != nil || inspection.Backend == nil || *inspection.Backend != backendID {
|
||||||
|
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
||||||
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
||||||
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
profiles:
|
profiles:
|
||||||
application:
|
- id: application
|
||||||
endpoint: https://example.invalid/v1
|
endpoint: https://example.invalid/v1
|
||||||
model: default-model
|
model: default-model
|
||||||
prompts:
|
prompts:
|
||||||
hello:
|
- id: hello
|
||||||
"1":
|
|
||||||
default_profile: application
|
default_profile: application
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -128,16 +162,16 @@ prompts:
|
|||||||
t.Fatalf("select embedded defaults: %v", err)
|
t.Fatalf("select embedded defaults: %v", err)
|
||||||
}
|
}
|
||||||
operatorDirectory := t.TempDir()
|
operatorDirectory := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n application:\n model: operator-model\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n - id: application\n endpoint: https://example.invalid/v1\n model: operator-model\n"), 0o644); err != nil {
|
||||||
t.Fatalf("write operator input: %v", err)
|
t.Fatalf("write operator input: %v", err)
|
||||||
}
|
}
|
||||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("select operator inputs: %v", err)
|
t.Fatalf("select operator inputs: %v", err)
|
||||||
}
|
}
|
||||||
programmaticModel := "programmatic-model"
|
programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1"
|
||||||
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", map[string]promptkit.ProfileConfig{
|
programmatic := configkit.Input{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
|
||||||
"application": {Model: &programmaticModel},
|
{ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel},
|
||||||
})}
|
})}
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -149,9 +183,9 @@ prompts:
|
|||||||
{name: "programmatic", apply: true, want: "programmatic-model"},
|
{name: "programmatic", apply: true, want: "programmatic-model"},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
underlays := []configkit.Underlay{{Inputs: embedded}}
|
underlays := []configkit.Underlay{{Inputs: mountSettingsInputs(embedded)}}
|
||||||
if test.apply {
|
if test.apply {
|
||||||
underlays = append(underlays, configkit.Underlay{Inputs: operator})
|
underlays = append(underlays, configkit.Underlay{Inputs: mountSettingsInputs(operator)})
|
||||||
}
|
}
|
||||||
if test.name == "programmatic" {
|
if test.name == "programmatic" {
|
||||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
|
||||||
@@ -173,7 +207,7 @@ prompts:
|
|||||||
|
|
||||||
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("invalid", map[string]any{"profiles": map[string]any{"broken": map[string]any{"unknown": true}}}),
|
Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("invalid", map[string]any{"prompts": []any{map[string]any{"id": "broken", "output": map[string]any{"unknown": true}}}}),
|
||||||
}}})
|
}}})
|
||||||
var loadErr *configkit.LoadError
|
var loadErr *configkit.LoadError
|
||||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||||
@@ -181,6 +215,36 @@ func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPublicConstructionPreservesMalformedYAMLCause(t *testing.T) {
|
||||||
|
const sourceLabel = "prompts.yml"
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Bytes(sourceLabel, []byte("profiles: [\n")),
|
||||||
|
}}})
|
||||||
|
var loadErr *configkit.LoadError
|
||||||
|
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||||
|
t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err)
|
||||||
|
}
|
||||||
|
cause := errors.Unwrap(err)
|
||||||
|
var causeLoadErr *configkit.LoadError
|
||||||
|
if cause == nil || errors.Is(cause, promptkit.ErrInvalidConfig) || !errors.As(cause, &causeLoadErr) || causeLoadErr != loadErr {
|
||||||
|
t.Fatalf("construction cause = %v, want the original loading failure without a category branch", cause)
|
||||||
|
}
|
||||||
|
for _, diagnostic := range loadErr.Diagnostics() {
|
||||||
|
if diagnostic.Code != "yaml.invalid" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if diagnostic.Primary == nil {
|
||||||
|
t.Fatal("YAML diagnostic has no source location")
|
||||||
|
}
|
||||||
|
source, ok := loadErr.Source(diagnostic.Primary.SourceID)
|
||||||
|
if !ok || source.Label != sourceLabel {
|
||||||
|
t.Fatalf("YAML diagnostic source = %+v, found=%v, want %q", source, ok, sourceLabel)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Fatalf("loading diagnostics = %+v, want yaml.invalid", loadErr.Diagnostics())
|
||||||
|
}
|
||||||
|
|
||||||
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
||||||
main := configkit.Input{}
|
main := configkit.Input{}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
|
||||||
@@ -189,43 +253,96 @@ func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) {
|
func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T) {
|
||||||
endpoint := "http://local-openrouter.example/v1"
|
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
promptkit.BackendOpenRouter: {Endpoint: &endpoint},
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
|
||||||
})}}}},
|
{ID: backendID, Endpoint: &endpoint},
|
||||||
|
})},
|
||||||
|
{Destination: configkit.Keys("promptkit"), Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
|
||||||
|
{ID: "application-profile", Backend: &backendID, Model: &model},
|
||||||
|
})},
|
||||||
|
}}},
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine with backend overlay: %v", err)
|
t.Fatalf("construct engine with backend overlay: %v", err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3")
|
inspection, err := engine.InspectProfile(context.Background(), "application-profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect catalog profile: %v", err)
|
t.Fatalf("inspect configured profile: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
backendInspection, err := engine.InspectBackend(context.Background(), backendID)
|
||||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
if err != nil || inspection.Endpoint != nil || backendInspection.Endpoint != endpoint {
|
||||||
|
t.Fatalf("profile/backend = %+v / %+v, %v", inspection, backendInspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparationRejectsAmbiguousPromptVersion(t *testing.T) {
|
func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
||||||
definition := func(content string) map[string]any {
|
backendEndpoint := "https://backend.example/v1"
|
||||||
return map[string]any{
|
defaultProfile := "not-yet-listed"
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
reachabilityEndpoint := " https://probe.example/ready?check=1 "
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
authenticationKind := " OPENROUTER_KEY "
|
||||||
}
|
authenticationEndpoint := "https://probe.example/key?scope=current"
|
||||||
}
|
source := promptkit.BackendSource("diagnostic backend", []promptkit.BackendConfig{
|
||||||
|
{
|
||||||
|
ID: "local",
|
||||||
|
Endpoint: &backendEndpoint,
|
||||||
|
SmokeTests: &promptkit.BackendSmokeTestConfig{
|
||||||
|
DefaultProfile: &defaultProfile,
|
||||||
|
Reachability: &promptkit.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||||
|
Authentication: &promptkit.AuthenticationProbeConfig{
|
||||||
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
backendEndpoint = "https://mutated.example/v1"
|
||||||
|
defaultProfile = "mutated-profile"
|
||||||
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
|
authenticationKind = "none"
|
||||||
|
authenticationEndpoint = "https://mutated.example/key"
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("versions", map[string]any{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Destination: configkit.Keys("promptkit"), Source: source}}}},
|
||||||
"prompts": map[string]any{"hello": map[string]any{
|
|
||||||
"1": definition("first"), "2": definition("second"),
|
|
||||||
}},
|
|
||||||
})},
|
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&effective); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
contents := effective.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"https://backend.example/v1",
|
||||||
|
"not-yet-listed",
|
||||||
|
"https://probe.example/ready?check=1",
|
||||||
|
"openrouter_key",
|
||||||
|
"https://probe.example/key?scope=current",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(contents, want) {
|
||||||
|
t.Errorf("effective configuration does not contain %q:\n%s", want, contents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, unwanted := range []string{
|
||||||
|
"mutated.example",
|
||||||
|
"mutated-profile",
|
||||||
|
} {
|
||||||
|
if strings.Contains(contents, unwanted) {
|
||||||
|
t.Errorf("effective configuration unexpectedly contains %q:\n%s", unwanted, contents)
|
||||||
}
|
}
|
||||||
if result, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}); result != nil || !errors.Is(err, promptkit.ErrPromptLoad) {
|
}
|
||||||
t.Fatalf("ambiguous preparation = (%#v, %v), want ErrPromptLoad", result, err)
|
}
|
||||||
|
|
||||||
|
func TestPreparationSelectsSinglePromptDefinitionByID(t *testing.T) {
|
||||||
|
engine := newSimpleEngine(t, "selected")
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil || prepared == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Content != "selected" {
|
||||||
|
t.Fatalf("prepare prompt by ID = (%+v, %v)", prepared, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,10 +7,18 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func mountSettingsInputs(inputs []configkit.Input) []configkit.Input {
|
||||||
|
result := append([]configkit.Input(nil), inputs...)
|
||||||
|
for i := range result {
|
||||||
|
result[i].Destination = configkit.Keys("promptkit")
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
type fakeLLMClient struct {
|
type fakeLLMClient struct {
|
||||||
response *promptkit.GenerateResponse
|
response *promptkit.GenerateResponse
|
||||||
responses []*promptkit.GenerateResponse
|
responses []*promptkit.GenerateResponse
|
||||||
@@ -59,15 +67,15 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
|
|||||||
|
|
||||||
func simpleConfig(content string) promptkit.Config {
|
func simpleConfig(content string) promptkit.Config {
|
||||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("consumer test", map[string]any{
|
||||||
"profiles": map[string]any{"profile": map[string]any{
|
"profiles": []any{map[string]any{"id": "profile",
|
||||||
"endpoint": "http://example.test/v1", "model": "model",
|
"endpoint": "http://example.test/v1", "model": "model",
|
||||||
}},
|
}},
|
||||||
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
|
"prompts": []any{map[string]any{"id": "prompt",
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}}},
|
}},
|
||||||
})},
|
})},
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|||||||
97
context_identity_contract_test.go
Normal file
97
context_identity_contract_test.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type contextIdentityResponseBody struct {
|
||||||
|
ctx context.Context
|
||||||
|
cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b contextIdentityResponseBody) Read([]byte) (int, error) { <-b.ctx.Done(); return 0, b.cause }
|
||||||
|
func (contextIdentityResponseBody) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestExecutionContextAndOriginalCauseIdentities(t *testing.T) {
|
||||||
|
for _, split := range []bool{false, true} {
|
||||||
|
for _, kind := range []string{"caller cancel", "caller deadline", "body operation deadline"} {
|
||||||
|
t.Run(map[bool]string{false: "run/", true: "prepared/"}[split]+kind, func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
cause := errors.New("private original cause")
|
||||||
|
ctx, cancel := context.WithCancelCause(context.Background())
|
||||||
|
defer cancel(nil)
|
||||||
|
want := error(context.Canceled)
|
||||||
|
failure := promptkit.ExecutionFailureCanceled
|
||||||
|
if kind == "caller deadline" {
|
||||||
|
var stop context.CancelFunc
|
||||||
|
ctx, stop = context.WithTimeoutCause(context.Background(), time.Second, cause)
|
||||||
|
defer stop()
|
||||||
|
}
|
||||||
|
if kind != "caller cancel" {
|
||||||
|
want = context.DeadlineExceeded
|
||||||
|
failure = promptkit.ExecutionFailureDeadline
|
||||||
|
}
|
||||||
|
cfg := simpleConfig("message")
|
||||||
|
cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||||
|
if kind == "body operation deadline" {
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, Body: contextIdentityResponseBody{ctx: r.Context(), cause: cause}}, nil
|
||||||
|
}
|
||||||
|
if kind == "caller cancel" {
|
||||||
|
cancel(cause)
|
||||||
|
} else {
|
||||||
|
<-ctx.Done()
|
||||||
|
}
|
||||||
|
return nil, context.Cause(ctx)
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var events []promptkit.ExecutionEvent
|
||||||
|
req := simpleGenerationRequest()
|
||||||
|
req.OperationTimeout = time.Minute
|
||||||
|
if kind == "body operation deadline" {
|
||||||
|
req.OperationTimeout = time.Second
|
||||||
|
}
|
||||||
|
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
|
||||||
|
var result *promptkit.RunResult
|
||||||
|
if split {
|
||||||
|
handle, prepareErr := engine.PrepareExecution(ctx, req)
|
||||||
|
if prepareErr != nil {
|
||||||
|
t.Fatal(prepareErr)
|
||||||
|
}
|
||||||
|
result, err = engine.RunPrepared(ctx, handle)
|
||||||
|
} else {
|
||||||
|
result, err = engine.Run(ctx, req)
|
||||||
|
}
|
||||||
|
for _, identity := range []error{want, cause, promptkit.ErrLLMGenerate, promptkit.ErrTransport} {
|
||||||
|
if !errors.Is(err, identity) {
|
||||||
|
t.Errorf("error=%v, want identity %v", err, identity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if kind != "body operation deadline" {
|
||||||
|
var transportErr *url.Error
|
||||||
|
if !errors.As(err, &transportErr) {
|
||||||
|
t.Errorf("lost HTTP transport error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result != nil || errors.Is(err, promptkit.ErrOperationTimeout) != (kind == "body operation deadline") || strings.Contains(err.Error(), cause.Error()) {
|
||||||
|
t.Fatalf("incorrect outcome or disclosure: result=%v error=%v", result, err)
|
||||||
|
}
|
||||||
|
if len(events) != 3 || events[1].Failure != failure || events[2].Kind != promptkit.ExecutionFinished || events[2].Failure != failure {
|
||||||
|
t.Fatalf("attempt/terminal classification = %+v", events)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
141
continuation_contract_test.go
Normal file
141
continuation_contract_test.go
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestContinuationBranchesAcrossEngines(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
firstClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "first answer"}}
|
||||||
|
firstEngine := newSimpleEngine(t, "first question", promptkit.WithLLMClient(firstClient))
|
||||||
|
initial, err := firstEngine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, APIKey: "synthetic-first-key"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
original := initial.Exchange.Details()
|
||||||
|
secondClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "follow-up answer"}}
|
||||||
|
secondEngine := newNamespacedContractEngine(t, secondClient)
|
||||||
|
scoped, err := secondEngine.WithNamespaces(promptkit.Namespaces{Prompt: "alpha", Profile: "alpha"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: initial.Exchange, SessionID: "new-session", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON}, AppendedMessages: []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "branch one"}}}
|
||||||
|
branch, err := scoped.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request.AppendedMessages[0].Content = "branch two"
|
||||||
|
sibling, err := scoped.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
next, err := scoped.Run(ctx, promptkit.RunRequest{PromptID: "shared", ProfileID: "shared", ContinueFrom: branch.Exchange})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if branch.ParentRunID != initial.RunID || sibling.ParentRunID != initial.RunID || next.ParentRunID != branch.RunID || branch.PromptID != "alpha/shared" || branch.SelectedProfileID != "alpha/shared" || branch.SessionID != "new-session" {
|
||||||
|
t.Fatalf("ancestry or selections incorrect: %#v %#v %#v", branch, sibling, next)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(initial.Exchange.Details(), original) {
|
||||||
|
t.Fatal("branching changed source")
|
||||||
|
}
|
||||||
|
if branch.Validation.Status != promptkit.ValidationFailed || branch.ModelName != "alpha-model" {
|
||||||
|
t.Fatalf("follow-up settings: %#v", branch)
|
||||||
|
}
|
||||||
|
branchMessages := branch.Exchange.Details().Messages
|
||||||
|
siblingMessages := sibling.Exchange.Details().Messages
|
||||||
|
if len(branchMessages) != 5 || len(siblingMessages) != 5 || branchMessages[3].Content != "branch one" || siblingMessages[3].Content != "branch two" || len(next.Exchange.Details().Messages) != 7 {
|
||||||
|
t.Fatalf("branch histories = %#v / %#v", branchMessages, siblingMessages)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(branch.Exchange.Details())
|
||||||
|
if err != nil || !json.Valid(encoded) {
|
||||||
|
t.Fatalf("details JSON = %s, %v", encoded, err)
|
||||||
|
}
|
||||||
|
branchMessages[0].Content = "changed details"
|
||||||
|
branch.RawOutput = "changed result"
|
||||||
|
if initial.Exchange.Details().Messages[0].Content != "first question" || branch.Exchange.Details().Messages[4].Content != "follow-up answer" {
|
||||||
|
t.Fatal("caller mutation changed captured history")
|
||||||
|
}
|
||||||
|
for _, request := range secondClient.requests {
|
||||||
|
if request.APIKey == "synthetic-first-key" || request.Prompt.Messages[0].Content != "first question" || request.Prompt.Messages[1].Content != "first answer" {
|
||||||
|
t.Fatalf("unexpected follow-up request: %#v", request)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContinuationDoesNotReloadCapturedInputs(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
path := filepath.Join(t.TempDir(), "input.txt")
|
||||||
|
if err := os.WriteFile(path, []byte("original input"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: ""}}
|
||||||
|
engine := newSimpleEngine(t, `{{input "source"}}`, promptkit.WithLLMClient(client))
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(path)}, CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte("new input"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(path)}, ContinueFrom: initial.Exchange}
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "original input"}, {Role: promptkit.RoleAssistant, Content: ""}, {Role: promptkit.RoleUser, Content: "new input"}, {Role: promptkit.RoleAssistant, Content: ""}}
|
||||||
|
if !reflect.DeepEqual(result.Exchange.Details().Messages, want) {
|
||||||
|
t.Fatalf("history = %#v", result.Exchange.Details().Messages)
|
||||||
|
}
|
||||||
|
if result, err := engine.Run(ctx, request); err == nil || result != nil {
|
||||||
|
t.Fatalf("missing new input = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContinuationSourceSupportsConcurrentRuns(t *testing.T) {
|
||||||
|
client := &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const count = 8
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errors := make(chan error, count)
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||||
|
if err == nil && (result.Exchange == nil || result.ParentRunID != initial.RunID) {
|
||||||
|
t.Error("missing continuation ancestry")
|
||||||
|
}
|
||||||
|
errors <- err
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errors)
|
||||||
|
for err := range errors {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(initial.Exchange.Details().Messages) != 2 {
|
||||||
|
t.Fatal("source changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
212
convert.go
212
convert.go
@@ -1,15 +1,23 @@
|
|||||||
package promptkit
|
package promptkit
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||||
|
var history *domain.Exchange
|
||||||
|
if req.ContinueFrom != nil && req.ContinueFrom.internal == nil {
|
||||||
|
return domain.RunRequest{}, fmt.Errorf("continuation exchange is empty")
|
||||||
|
}
|
||||||
|
if req.ContinueFrom != nil {
|
||||||
|
history = req.ContinueFrom.internal
|
||||||
|
}
|
||||||
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
execution, err := toDomainExecutionTargetOverride(req.Execution)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.RunRequest{}, err
|
return domain.RunRequest{}, err
|
||||||
@@ -23,9 +31,17 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
callback := req.Observer
|
callback := req.Observer
|
||||||
observer = func(event domain.ExecutionEvent) { callback(fromDomainExecutionEvent(event)) }
|
observer = func(event domain.ExecutionEvent) { callback(fromDomainExecutionEvent(event)) }
|
||||||
}
|
}
|
||||||
|
var onPrepared func(context.Context, *domain.PreparedRun)
|
||||||
|
if callback := req.OnPrepared; callback != nil {
|
||||||
|
onPrepared = func(ctx context.Context, prepared *domain.PreparedRun) {
|
||||||
|
callback(ctx, *fromDomainPreparedRun(prepared))
|
||||||
|
}
|
||||||
|
}
|
||||||
return domain.RunRequest{
|
return domain.RunRequest{
|
||||||
|
OperationTimeout: req.OperationTimeout,
|
||||||
|
OnPrepared: onPrepared,
|
||||||
|
CaptureExchange: req.CaptureExchange || req.ContinueFrom != nil,
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
APIKey: req.APIKey,
|
APIKey: req.APIKey,
|
||||||
@@ -36,6 +52,7 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
AppendedMessages: appendedMessages,
|
AppendedMessages: appendedMessages,
|
||||||
TransientRetries: copyIntPtr(req.TransientRetries),
|
TransientRetries: copyIntPtr(req.TransientRetries),
|
||||||
Observer: observer,
|
Observer: observer,
|
||||||
|
ContinueFrom: history,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,8 +89,8 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return &PreparedRun{
|
return &PreparedRun{
|
||||||
|
ParentRunID: prepared.ParentRunID,
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
SelectedBackendID: prepared.SelectedBackendID,
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
@@ -83,6 +100,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
InputHashes: copyStringMap(prepared.InputHashes),
|
InputHashes: copyStringMap(prepared.InputHashes),
|
||||||
SessionID: prepared.SessionID,
|
SessionID: prepared.SessionID,
|
||||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||||
|
MessageContentHash: prepared.MessageContentHash,
|
||||||
Messages: fromDomainRenderedMessages(prepared.Messages),
|
Messages: fromDomainRenderedMessages(prepared.Messages),
|
||||||
StartTime: prepared.StartTime,
|
StartTime: prepared.StartTime,
|
||||||
EndTime: prepared.EndTime,
|
EndTime: prepared.EndTime,
|
||||||
@@ -94,13 +112,18 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
if result == nil {
|
if result == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var exchange *Exchange
|
||||||
|
if result.Exchange != nil {
|
||||||
|
exchange = &Exchange{internal: result.Exchange}
|
||||||
|
}
|
||||||
return &RunResult{
|
return &RunResult{
|
||||||
|
Exchange: exchange,
|
||||||
|
ParentRunID: result.ParentRunID,
|
||||||
RunID: result.RunID,
|
RunID: result.RunID,
|
||||||
Artifact: fromDomainArtifact(result.Artifact),
|
Artifact: fromDomainArtifact(result.Artifact),
|
||||||
RawOutput: result.RawOutput,
|
RawOutput: result.RawOutput,
|
||||||
Validation: fromDomainValidationResult(result.Validation),
|
Validation: fromDomainValidationResult(result.Validation),
|
||||||
PromptID: result.PromptID,
|
PromptID: result.PromptID,
|
||||||
PromptVersion: result.PromptVersion,
|
|
||||||
PromptHash: result.PromptHash,
|
PromptHash: result.PromptHash,
|
||||||
SessionID: result.SessionID,
|
SessionID: result.SessionID,
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
@@ -211,6 +234,12 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
|
return fromDomainExecutionTargetWithOwnedParams(target, copyAnyMap(target.ExtraParams))
|
||||||
|
}
|
||||||
|
|
||||||
|
// fromDomainExecutionTargetWithOwnedParams projects scalar settings and takes
|
||||||
|
// ownership of extraParams, which the caller has already copied.
|
||||||
|
func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, extraParams map[string]any) ExecutionTarget {
|
||||||
return ExecutionTarget{
|
return ExecutionTarget{
|
||||||
BackendID: target.BackendID,
|
BackendID: target.BackendID,
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
@@ -222,42 +251,62 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
|||||||
ServiceTier: target.ServiceTier,
|
ServiceTier: target.ServiceTier,
|
||||||
ReasoningEffort: target.ReasoningEffort,
|
ReasoningEffort: target.ReasoningEffort,
|
||||||
APIKeyEnv: target.APIKeyEnv,
|
APIKeyEnv: target.APIKeyEnv,
|
||||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
ExtraParams: extraParams,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
|
||||||
if inspection == nil {
|
out := &ProfileInspection{ProfileID: v.ID, Backend: copyStringPtr(v.Backend), Endpoint: copyStringPtr(v.Endpoint), Model: copyStringPtr(v.Model), Temperature: copyFloat64Ptr(v.Temperature), TopP: copyFloat64Ptr(v.TopP), MaxTokens: copyIntPtr(v.MaxTokens), TimeoutSeconds: copyIntPtr(v.TimeoutSeconds), ServiceTier: copyStringPtr(v.ServiceTier), ReasoningEffort: copyStringPtr(v.ReasoningEffort), APIKeyEnv: copyStringPtr(v.APIKeyEnv), APIKeyRequired: copyBoolPtr(v.APIKeyRequired)}
|
||||||
return nil
|
if v.ExtraParams != nil {
|
||||||
}
|
data := copyAnyMap(v.ExtraParams)
|
||||||
return &ProfileInspection{
|
out.ExtraParams = &data
|
||||||
ProfileID: inspection.ProfileID,
|
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
|
||||||
APIKeyRequired: inspection.APIKeyRequired,
|
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
||||||
if inspection == nil {
|
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||||
|
if v.ExtraParams != nil {
|
||||||
|
data := copyAnyMap(v.ExtraParams)
|
||||||
|
out.ExtraParams = &data
|
||||||
|
}
|
||||||
|
if v.SmokeTests != nil {
|
||||||
|
out.SmokeTests = &BackendSmokeTestConfig{DefaultProfile: copyStringPtr(v.SmokeTests.DefaultProfile)}
|
||||||
|
if v.SmokeTests.Reachability != nil {
|
||||||
|
out.SmokeTests.Reachability = &ReachabilityProbeConfig{Endpoint: copyStringPtr(v.SmokeTests.Reachability.Endpoint)}
|
||||||
|
}
|
||||||
|
if v.SmokeTests.Authentication != nil {
|
||||||
|
out.SmokeTests.Authentication = &AuthenticationProbeConfig{Kind: copyStringPtr(v.SmokeTests.Authentication.Kind), Endpoint: copyStringPtr(v.SmokeTests.Authentication.Endpoint)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
|
||||||
|
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), Schema: jsonvalue.CloneTrusted(v.Output.Schema), SchemaFile: copyStringPtr(v.Output.SchemaFile), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: v.Output.RepairAttempts}}
|
||||||
|
if v.Inputs != nil {
|
||||||
|
inputs := make([]PromptInputInspection, len(*v.Inputs))
|
||||||
|
for i, input := range *v.Inputs {
|
||||||
|
inputs[i] = PromptInputInspection{Name: input.Name, Required: copyBoolPtr(input.Required), ContentType: copyStringPtr(input.ContentType), Description: copyStringPtr(input.Description)}
|
||||||
|
}
|
||||||
|
out.Inputs = &inputs
|
||||||
|
}
|
||||||
|
out.Messages = make([]PromptMessageInspection, len(v.Messages))
|
||||||
|
for i, message := range v.Messages {
|
||||||
|
out.Messages[i] = PromptMessageInspection{Role: message.Role, Content: message.Content}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
out.Messages[i].CacheControl = &PromptCacheControlInspection{Type: CacheControlType(message.CacheControl.Type), TTL: copyStringPtr(message.CacheControl.TTL)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBoolPtr(src *bool) *bool {
|
||||||
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
value := *src
|
||||||
for i, input := range inspection.Inputs {
|
return &value
|
||||||
inputs[i] = PromptInputDefinition{
|
|
||||||
Name: input.Name,
|
|
||||||
Required: input.Required,
|
|
||||||
ContentType: input.ContentType,
|
|
||||||
Description: input.Description,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &PromptInspection{
|
|
||||||
PromptID: inspection.PromptID,
|
|
||||||
PromptVersion: inspection.PromptVersion,
|
|
||||||
PromptHash: inspection.PromptHash,
|
|
||||||
DefaultProfileID: inspection.DefaultProfileID,
|
|
||||||
Inputs: inputs,
|
|
||||||
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
@@ -269,20 +318,16 @@ func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
func toDomainOutputContractPtr(contract *OutputContractInput) *domain.OutputContractInput {
|
||||||
if contract == nil {
|
if contract == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := toDomainOutputContract(*contract)
|
return &domain.OutputContractInput{
|
||||||
return &out
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
|
||||||
return domain.OutputContract{
|
|
||||||
Format: domain.OutputFormat(contract.Format),
|
Format: domain.OutputFormat(contract.Format),
|
||||||
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||||
|
Schema: contract.Schema,
|
||||||
SchemaPath: contract.SchemaPath,
|
SchemaPath: contract.SchemaPath,
|
||||||
RepairAttempts: contract.RepairAttempts,
|
RepairAttempts: copyIntPtr(contract.RepairAttempts),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -290,6 +335,7 @@ func fromDomainOutputContract(contract domain.OutputContract) OutputContract {
|
|||||||
return OutputContract{
|
return OutputContract{
|
||||||
Format: OutputFormat(contract.Format),
|
Format: OutputFormat(contract.Format),
|
||||||
ValidationMode: ValidationMode(contract.ValidationMode),
|
ValidationMode: ValidationMode(contract.ValidationMode),
|
||||||
|
Schema: jsonvalue.CloneTrusted(contract.Schema),
|
||||||
SchemaPath: contract.SchemaPath,
|
SchemaPath: contract.SchemaPath,
|
||||||
RepairAttempts: contract.RepairAttempts,
|
RepairAttempts: contract.RepairAttempts,
|
||||||
}
|
}
|
||||||
@@ -393,88 +439,10 @@ func copyAnyMap(src map[string]any) map[string]any {
|
|||||||
if src == nil {
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make(map[string]any, len(src))
|
return jsonvalue.CloneTrusted(src).(map[string]any)
|
||||||
for k, v := range src {
|
|
||||||
out[k] = copyAny(v)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyAny(value any) any {
|
func copyAny(value any) any { return jsonvalue.CloneTrusted(value) }
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
switch v := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
return copyAnyMap(v)
|
|
||||||
case []any:
|
|
||||||
out := make([]any, len(v))
|
|
||||||
for i, item := range v {
|
|
||||||
out[i] = copyAny(item)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case []string:
|
|
||||||
return copyStringSlice(v)
|
|
||||||
case []byte:
|
|
||||||
return copyBytes(v)
|
|
||||||
default:
|
|
||||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
|
||||||
if !value.IsValid() {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
switch value.Kind() {
|
|
||||||
case reflect.Interface:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
copied := copyReflectValue(value.Elem())
|
|
||||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
|
||||||
return copied
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
out.Set(copied)
|
|
||||||
return out
|
|
||||||
case reflect.Pointer:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type().Elem())
|
|
||||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
|
||||||
return out
|
|
||||||
case reflect.Map:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
|
||||||
iter := value.MapRange()
|
|
||||||
for iter.Next() {
|
|
||||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Slice:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Array:
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
default:
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyStringSlice(src []string) []string {
|
func copyStringSlice(src []string) []string {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestToDomainAppendedMessages(t *testing.T) {
|
func TestToDomainAppendedMessages(t *testing.T) {
|
||||||
|
|||||||
82
coordinate.go
Normal file
82
coordinate.go
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ConversationTurn is delivered before each requested continuation. Result is
|
||||||
|
// nil on the initial callback; RemainingTurns counts additional runs allowed.
|
||||||
|
type ConversationTurn struct {
|
||||||
|
Exchange *Exchange
|
||||||
|
Result *RunResult
|
||||||
|
RemainingTurns int
|
||||||
|
}
|
||||||
|
|
||||||
|
// CoordinateRequest selects a starting exchange, turn budget and callback.
|
||||||
|
// Observer selects continuation runs only, independently of callback calls.
|
||||||
|
type CoordinateRequest struct {
|
||||||
|
Start *Exchange
|
||||||
|
AdditionalTurns int
|
||||||
|
Next func(context.Context, ConversationTurn) (*RunRequest, error)
|
||||||
|
Observer ExecutionObserver
|
||||||
|
}
|
||||||
|
|
||||||
|
// CoordinationOutcome distinguishes caller-directed stopping from exhaustion.
|
||||||
|
type CoordinationOutcome string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// CoordinationStopped means the callback chose not to continue.
|
||||||
|
CoordinationStopped CoordinationOutcome = "stopped"
|
||||||
|
// CoordinationExhausted means another run was requested without budget.
|
||||||
|
CoordinationExhausted CoordinationOutcome = "exhausted"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CoordinationResult contains the normal outcome or a response withheld by
|
||||||
|
// cancellation before delivery to the callback.
|
||||||
|
type CoordinationResult struct {
|
||||||
|
Outcome CoordinationOutcome `json:"outcome,omitempty"`
|
||||||
|
Undelivered *RunResult `json:"undelivered,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Coordinate advances one branch from Start using ordinary runs. Next is called
|
||||||
|
// initially even for a zero budget and receives each completed result once,
|
||||||
|
// including the final permitted response. The initial exchange is not itself a
|
||||||
|
// newly delivered result. Each started continuation consumes one turn; retries
|
||||||
|
// and structural repairs do not. Callback work runs after admission is released.
|
||||||
|
// A nil request stops; requesting a run without remaining budget exhausts it.
|
||||||
|
// Callback and execution errors retain their identities. Cancellation observed
|
||||||
|
// before delivery returns the single completed response as Undelivered;
|
||||||
|
// cancellation after callback invocation does not replay that response. Context
|
||||||
|
// cancellation takes precedence over a callback's returned action or error.
|
||||||
|
// Start must be a captured exchange, Next non-nil, and AdditionalTurns
|
||||||
|
// nonnegative; invalid values match ErrInvalidRequest. A nil engine returns
|
||||||
|
// ErrInvalidConfig. The selected request is copied and its ContinueFrom is
|
||||||
|
// replaced by the coordinator's current exchange. The request observer runs
|
||||||
|
// before the coordinator observer for each continuation.
|
||||||
|
func (e *Engine) Coordinate(ctx context.Context, req CoordinateRequest) (CoordinationResult, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return CoordinationResult{}, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
if req.Start == nil || req.Start.internal == nil || req.Next == nil || req.AdditionalTurns < 0 {
|
||||||
|
return CoordinationResult{}, fmt.Errorf("%w: invalid coordination request", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
outcome, err := usecase.Coordinate(ctx, req.Start.internal, req.AdditionalTurns,
|
||||||
|
func(ctx context.Context, turn usecase.ConversationTurn[RunResult]) (*RunRequest, error) {
|
||||||
|
return req.Next(ctx, ConversationTurn{Exchange: &Exchange{internal: turn.Exchange}, Result: turn.Result, RemainingTurns: turn.RemainingTurns})
|
||||||
|
},
|
||||||
|
func(ctx context.Context, current *domain.Exchange, selected *RunRequest) (*RunResult, *domain.Exchange, error) {
|
||||||
|
request := *selected
|
||||||
|
request.ContinueFrom = &Exchange{internal: current}
|
||||||
|
request.Observer = ComposeObservers(request.Observer, req.Observer)
|
||||||
|
result, runErr := e.Run(ctx, request)
|
||||||
|
if runErr != nil {
|
||||||
|
return nil, nil, runErr
|
||||||
|
}
|
||||||
|
return result, result.Exchange.internal, nil
|
||||||
|
})
|
||||||
|
return CoordinationResult{Outcome: CoordinationOutcome(outcome.Outcome), Undelivered: outcome.Undelivered}, err
|
||||||
|
}
|
||||||
61
coordinate_contract_test.go
Normal file
61
coordinate_contract_test.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCoordinatePublicBoundary(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, request := range []promptkit.CoordinateRequest{{}, {Start: &promptkit.Exchange{}, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }}, {Start: initial.Exchange, AdditionalTurns: -1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }}} {
|
||||||
|
if _, err := engine.Coordinate(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("invalid request error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := (*promptkit.Engine)(nil).Coordinate(ctx, promptkit.CoordinateRequest{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine error = %v", err)
|
||||||
|
}
|
||||||
|
callbacks := 0
|
||||||
|
selected := &promptkit.RunRequest{PromptID: "prompt", ContinueFrom: &promptkit.Exchange{}}
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 2, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
callbacks++
|
||||||
|
if callbacks == 1 {
|
||||||
|
if turn.Result != nil || turn.RemainingTurns != 2 {
|
||||||
|
t.Fatalf("initial turn = %#v", turn)
|
||||||
|
}
|
||||||
|
return selected, nil
|
||||||
|
}
|
||||||
|
if turn.Result == nil || turn.Exchange.Details().ParentRunID == "" {
|
||||||
|
t.Fatalf("delivered turn = %#v", turn)
|
||||||
|
}
|
||||||
|
if callbacks == 2 {
|
||||||
|
turn.Result.Exchange = nil
|
||||||
|
*turn.Exchange = promptkit.Exchange{}
|
||||||
|
return selected, nil
|
||||||
|
}
|
||||||
|
if turn.Result.ParentRunID == initial.RunID {
|
||||||
|
t.Fatal("callback mutation redirected branch")
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbacks != 3 || selected.ContinueFrom == nil {
|
||||||
|
t.Fatalf("result=%#v error=%v callbacks=%d", result, err, callbacks)
|
||||||
|
}
|
||||||
|
stop, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, nil }})
|
||||||
|
if err != nil || stop.Outcome != promptkit.CoordinationStopped {
|
||||||
|
t.Fatalf("zero budget = %#v, %v", stop, err)
|
||||||
|
}
|
||||||
|
sentinel := errors.New("callback")
|
||||||
|
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) { return nil, sentinel }})
|
||||||
|
if !errors.Is(err, sentinel) {
|
||||||
|
t.Fatalf("callback error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
86
coordinate_controls_contract_test.go
Normal file
86
coordinate_controls_contract_test.go
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
if req.Prompt.SessionID == "continuation" {
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
return &promptkit.GenerateResponse{Content: "answer", Usage: promptkit.TokenUsage{PromptTokens: 2}}, nil
|
||||||
|
}), 1, capacityInt(0), nil)
|
||||||
|
parent := context.Background()
|
||||||
|
initial, err := engine.Run(parent, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var group promptkit.ExecutionGroup
|
||||||
|
var child context.Context
|
||||||
|
var details []promptkit.PreparedRun
|
||||||
|
var requestEvents, coordinatorEvents []promptkit.ExecutionEvent
|
||||||
|
start := time.Now()
|
||||||
|
result, err := engine.Coordinate(parent, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 2, Observer: promptkit.ComposeObservers(group.Observe, func(e promptkit.ExecutionEvent) { coordinatorEvents = append(coordinatorEvents, e) }), Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
if turn.Result != nil {
|
||||||
|
if ctx.Err() != nil || child == nil || !errors.Is(child.Err(), context.Canceled) {
|
||||||
|
t.Fatal("child was not cleaned up before callback")
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second}); err != nil {
|
||||||
|
t.Fatalf("assessment: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if turn.RemainingTurns == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", SessionID: "continuation", OperationTimeout: 3 * time.Second, Observer: func(e promptkit.ExecutionEvent) { requestEvents = append(requestEvents, e) }, OnPrepared: func(ctx context.Context, prepared promptkit.PreparedRun) {
|
||||||
|
child = ctx
|
||||||
|
details = append(details, prepared)
|
||||||
|
}}, nil
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || time.Since(start) != 4*time.Second || len(details) != 2 || len(details[0].Messages) != 3 || len(details[1].Messages) != 5 {
|
||||||
|
t.Fatalf("result=%+v error=%v details=%v elapsed=%v", result, err, len(details), time.Since(start))
|
||||||
|
}
|
||||||
|
summary := group.Snapshot()
|
||||||
|
if len(requestEvents) != 6 || len(coordinatorEvents) != 6 || len(summary.Members) != 2 || summary.Attempts != 2 || summary.Usage.PromptTokens.Value != 4 || !summary.TerminalCoverageComplete {
|
||||||
|
t.Fatalf("summary=%+v request=%d coordinator=%d", summary, len(requestEvents), len(coordinatorEvents))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateTimeoutLeavesParentAndSiblingUsable(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
engine := newBackendCapacityEngine(t, observationClientFunc(func(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
if req.Prompt.SessionID == "slow" {
|
||||||
|
<-ctx.Done()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
return &promptkit.GenerateResponse{Content: "answer"}, nil
|
||||||
|
}), 1, capacityInt(0), nil)
|
||||||
|
parent := context.Background()
|
||||||
|
initial, err := engine.Run(parent, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var details promptkit.PreparedRun
|
||||||
|
calls := 0
|
||||||
|
result, err := engine.Coordinate(parent, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
calls++
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", SessionID: "slow", OperationTimeout: time.Second, OnPrepared: func(_ context.Context, p promptkit.PreparedRun) { details = p }}, nil
|
||||||
|
}})
|
||||||
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || calls != 1 || result.Outcome != "" || result.Undelivered != nil || details.ParentRunID != initial.RunID || parent.Err() != nil {
|
||||||
|
t.Fatalf("result=%+v error=%v callbacks=%d", result, err, calls)
|
||||||
|
}
|
||||||
|
sibling, err := engine.Run(parent, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||||
|
if err != nil || sibling.Exchange == nil {
|
||||||
|
t.Fatalf("sibling=%v error=%v", sibling, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
170
coordinate_lifecycle_contract_test.go
Normal file
170
coordinate_lifecycle_contract_test.go
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCoordinateReleasesCapacityBeforeCallback(t *testing.T) {
|
||||||
|
client := &preparedRecordingClient{responses: []*promptkit.GenerateResponse{{Content: "initial"}, {Content: ""}, {Content: " "}, {Content: "validator"}}}
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
calls++
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}}, nil
|
||||||
|
}
|
||||||
|
if turn.Result.Validation.Status != promptkit.ValidationFailed || turn.Result.Validation.RepairAttempts != 1 || turn.RemainingTurns != 0 {
|
||||||
|
t.Fatalf("delivered turn = %#v", turn)
|
||||||
|
}
|
||||||
|
_, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
return nil, err
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || calls != 2 || len(client.snapshot()) != 4 {
|
||||||
|
t.Fatalf("outcome=%#v error=%v callbacks=%d calls=%d", result, err, calls, len(client.snapshot()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinatePreservesCapacityAndProviderErrors(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
sourceEngine := newSimpleEngine(t, "source", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
initial, err := sourceEngine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := newCapacityGateClient(release, 1)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||||
|
done := make(chan capacityRunResult, 1)
|
||||||
|
go runCapacityRequest(engine, ctx, promptkit.RunRequest{PromptID: "prompt"}, done)
|
||||||
|
select {
|
||||||
|
case <-client.started:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
}
|
||||||
|
request := promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||||
|
}}
|
||||||
|
_, err = engine.Coordinate(ctx, request)
|
||||||
|
var capacity *promptkit.CapacityError
|
||||||
|
if !errors.As(err, &capacity) || capacity.BackendID != "limited" {
|
||||||
|
t.Fatalf("capacity error = %v", err)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
if finished := awaitCapacityRun(t, done); finished.err != nil {
|
||||||
|
t.Fatal(finished.err)
|
||||||
|
}
|
||||||
|
providerEngine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, `{"error":{"message":"synthetic"}}`)
|
||||||
|
_, err = providerEngine.Coordinate(ctx, request)
|
||||||
|
var provider *promptkit.GenerationError
|
||||||
|
if !errors.As(err, &provider) || provider.StatusCode() != http.StatusUnprocessableEntity {
|
||||||
|
t.Fatalf("provider error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateRetriesAndRepairsDeliverOnce(t *testing.T) {
|
||||||
|
initialEngine := newSimpleEngine(t, "source", promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}))
|
||||||
|
initial, err := initialEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
responses := []*http.Response{observedHTTPResponse(503, `{}`), observedHTTPResponse(200, `{"choices":[{"message":{"content":"invalid"}}]}`), observedHTTPResponse(200, `{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`)}
|
||||||
|
calls := 0
|
||||||
|
one := 1
|
||||||
|
config := simpleConfig("follow-up")
|
||||||
|
config.TransientRetries = &one
|
||||||
|
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
response := responses[calls]
|
||||||
|
calls++
|
||||||
|
return response, nil
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
callbacks := 0
|
||||||
|
result, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
callbacks++
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}}, nil
|
||||||
|
}
|
||||||
|
if !turn.Result.Validation.IsValid || turn.RemainingTurns != 0 {
|
||||||
|
t.Fatalf("turn = %#v", turn)
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}})
|
||||||
|
if err != nil || result.Outcome != promptkit.CoordinationStopped || callbacks != 2 || calls != 3 {
|
||||||
|
t.Fatalf("result=%#v error=%v callbacks=%d attempts=%d", result, err, callbacks, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateCancellationDelivery(t *testing.T) {
|
||||||
|
for _, cancelInside := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "before delivery", true: "inside callback"}[cancelInside], func(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
calls := 0
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
calls++
|
||||||
|
if turn.Result != nil {
|
||||||
|
cancel()
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) {
|
||||||
|
if !cancelInside && event.Kind == promptkit.ExecutionFinished {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}}, nil
|
||||||
|
}})
|
||||||
|
if !errors.Is(err, context.Canceled) || result.Outcome != "" || (result.Undelivered != nil) == cancelInside {
|
||||||
|
t.Fatalf("result=%#v error=%v callbacks=%d", result, err, calls)
|
||||||
|
}
|
||||||
|
if !cancelInside && (calls != 1 || result.Undelivered.Exchange == nil) {
|
||||||
|
t.Fatalf("undelivered result=%#v calls=%d", result, calls)
|
||||||
|
}
|
||||||
|
if cancelInside && calls != 2 {
|
||||||
|
t.Fatalf("callbacks=%d", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCoordinateFailureLeavesSiblingUsable(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sentinel := errors.New("provider")
|
||||||
|
client.err = sentinel
|
||||||
|
result, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
return &promptkit.RunRequest{PromptID: "prompt"}, nil
|
||||||
|
}})
|
||||||
|
if !errors.Is(err, sentinel) || !errors.Is(err, promptkit.ErrLLMGenerate) || result.Outcome != "" || result.Undelivered != nil {
|
||||||
|
t.Fatalf("failure=%#v, %v", result, err)
|
||||||
|
}
|
||||||
|
client.err = nil
|
||||||
|
if sibling, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange}); err != nil || sibling.Exchange == nil {
|
||||||
|
t.Fatalf("sibling=%#v, %v", sibling, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
39
doc.go
39
doc.go
@@ -1,29 +1,33 @@
|
|||||||
// Package promptkit provides an embeddable engine for preparing and executing
|
// Package promptkit provides an embeddable engine for preparing and executing
|
||||||
// prompt-defined LLM workflows.
|
// prompt-defined LLM workflows.
|
||||||
//
|
//
|
||||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
// Applications construct an [Engine] with [NewEngine] or register [Configure]
|
||||||
// sources for prompts, profiles, and backends, and
|
// during their own ConfigKit load and pass its result to [NewFromConfig], then
|
||||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.InspectBackend],
|
||||||
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.ListProfiles], [Engine.ListPrompts], [Engine.Prepare],
|
||||||
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
// [Engine.PrepareExecution], [Engine.Run], [Engine.RunPrepared], or
|
||||||
|
// [Engine.Coordinate]. Captured [Exchange] values support independent
|
||||||
|
// continuations, and [ExecutionGroup] collects explicitly observed runs.
|
||||||
|
// Concrete registries, repositories,
|
||||||
// validators, and the built-in OpenAI-compatible client remain internal
|
// validators, and the built-in OpenAI-compatible client remain internal
|
||||||
// implementation details.
|
// implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||||
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
// InspectRequirements, InspectProfile, InspectBackend, ListProfiles, ListPrompts,
|
||||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
// Prepare, PrepareExecution, Run, RunPrepared, and Coordinate calls. Engine-local backend policies
|
||||||
// bound admitted Run and RunPrepared calls and model generations where
|
// bound admitted Run and RunPrepared calls and model generations where
|
||||||
// configured, while different backend pools and unlimited backends continue
|
// configured, while different backend pools and unlimited backends continue
|
||||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||||
// still receive concurrent calls and must be safe for that use.
|
// still receive concurrent calls and must be safe for that use.
|
||||||
//
|
//
|
||||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
// An Engine captures selected configuration resources during construction.
|
||||||
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
// Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||||
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||||
// InspectRequirements, and InspectProfile return copied inspection values.
|
// InspectRequirements, InspectProfile, and InspectBackend return copied
|
||||||
|
// inspection values.
|
||||||
// ListProfiles and ListPrompts return caller-owned identity slices.
|
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||||
// Returned values and values passed to extension interfaces are likewise
|
// Returned values and values passed to extension interfaces are likewise
|
||||||
// isolated from engine state.
|
// isolated from engine state.
|
||||||
@@ -42,7 +46,7 @@
|
|||||||
// that enforces their filesystem, authorization, and size policies.
|
// that enforces their filesystem, authorization, and size policies.
|
||||||
//
|
//
|
||||||
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
||||||
// and [ValidationResult.Errors] may contain sensitive data. Credential
|
// [ValidationResult.Errors], and [ExchangeDetails.Messages] may contain sensitive data. Credential
|
||||||
// exclusion and redaction do not sanitize those values. Applications and
|
// exclusion and redaction do not sanitize those values. Applications and
|
||||||
// injected collaborators are responsible for access control, retention,
|
// injected collaborators are responsible for access control, retention,
|
||||||
// logging, and secret handling appropriate to their data.
|
// logging, and secret handling appropriate to their data.
|
||||||
@@ -56,17 +60,22 @@
|
|||||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
// used by those values.
|
// used by those values.
|
||||||
//
|
//
|
||||||
// Construction, inspection, handle, and error values, including [Config],
|
// Resource inspection results support direct JSON and YAML encoding.
|
||||||
|
// [PromptInspection], [ProfileInspection], [BackendInspection], and their nested
|
||||||
|
// values define field names, nesting, and omission of absent optional fields.
|
||||||
|
// Formatting belongs to the selected encoder.
|
||||||
|
//
|
||||||
|
// Other construction, inspection, handle, and error values, including [Config],
|
||||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||||
// [ProfileConfig], [RunRequest],
|
// [ProfileConfig], [RunRequest],
|
||||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
// [ArtifactRef], [ExecutionTargetOverride],
|
||||||
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
// [PromptIdentity], [PromptInputDefinition],
|
||||||
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||||
// [ExecutionEvent], do not have stable JSON
|
// [ExecutionEvent], do not have stable JSON
|
||||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||||
// every public value.
|
// every public value.
|
||||||
|
//
|
||||||
// Provider-derived [GenerationError] accessor values are untrusted and can
|
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||||
// contain sensitive request or schema fragments. Applications must apply their
|
// contain sensitive request or schema fragments. Applications must apply their
|
||||||
// own disclosure policy before logging, displaying, or returning them.
|
// own disclosure policy before logging, displaying, or returning them.
|
||||||
|
|||||||
63
docs/adr/0002-shared-configuration-data-model.md
Normal file
63
docs/adr/0002-shared-configuration-data-model.md
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
# 0002: Share Configuration Data Types Across The Public Boundary
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted.
|
||||||
|
|
||||||
|
## Date
|
||||||
|
|
||||||
|
2026-11-21
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
PromptKit is a Go library that applications use to configure and run prompts.
|
||||||
|
Applications use its public package; the code that loads settings and runs
|
||||||
|
prompts lives in internal packages. This separation normally lets the internal
|
||||||
|
code change without changing the application's interface.
|
||||||
|
|
||||||
|
The planned configuration API lets applications load settings once and pass them to
|
||||||
|
PromptKit. Keeping separate public and internal definitions of those same
|
||||||
|
settings would require duplicate declarations and code to copy each field back
|
||||||
|
and forth. That adds maintenance work and opportunities to miss a field without
|
||||||
|
changing what the settings mean.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Define the public configuration settings and their nested data types once in a
|
||||||
|
small, data-only internal package. Expose them through type aliases in the public
|
||||||
|
`promptkit` package. An alias gives the same Go type a public name; consumers
|
||||||
|
continue to use names such as `promptkit.Settings` without importing an internal
|
||||||
|
package. The internal configuration code uses the same definitions directly.
|
||||||
|
|
||||||
|
This is a narrow exception to the rule against exposing internal types. These
|
||||||
|
settings are intentional public contracts, including their fields and
|
||||||
|
configuration tags. Their location does not make them private implementation
|
||||||
|
details that can be changed independently of the public API.
|
||||||
|
|
||||||
|
The rest of the public boundary stays in place. Loading, validation, and
|
||||||
|
execution logic remain internal. Source-building helpers, execution requests,
|
||||||
|
runtime objects, and inspection reports keep separate representations where
|
||||||
|
they serve different purposes. Internal packages still do not import the public
|
||||||
|
package. Sharing declarations does not remove the copies needed to keep a
|
||||||
|
running engine independent of later application changes.
|
||||||
|
|
||||||
|
## Alternatives Considered
|
||||||
|
|
||||||
|
- Keep separate public and internal settings. This allows independent changes,
|
||||||
|
but provides little benefit when both describe the same configuration and
|
||||||
|
must stay synchronized.
|
||||||
|
- Put the shared types in a new public settings package. This also removes
|
||||||
|
duplication, but adds a public package when the existing `promptkit` package
|
||||||
|
already provides a suitable consumer interface.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
Configuration fields have one declaration, and registration and resource
|
||||||
|
capture no longer need conversions between equivalent settings types. The
|
||||||
|
primary benefit is simpler maintenance; performance improvements are secondary.
|
||||||
|
Conversion into execution-ready objects and necessary ownership copies remain.
|
||||||
|
|
||||||
|
This decision leaves the [ConfigKit ownership boundary](0001-configuration-boundary.md)
|
||||||
|
unchanged. The [shared-loading guide](../consumers/pkg-promptkit.md#reuse-an-application-owned-load)
|
||||||
|
describes the implemented consumer workflow; the
|
||||||
|
[internal overview](../internal/overview.md) describes its implementation.
|
||||||
264
docs/consumers/configuration-migration.md
Normal file
264
docs/consumers/configuration-migration.md
Normal file
@@ -0,0 +1,264 @@
|
|||||||
|
# Configuration And API Migration
|
||||||
|
|
||||||
|
This guide covers the current configuration and repair-input migration, as well
|
||||||
|
as earlier module-path and keyed-resource migrations.
|
||||||
|
Promptkit installs a default-on bundled OpenRouter catalog; applications choose configuration
|
||||||
|
inputs explicitly, and execution selects one prompt by its exact ID. This guide
|
||||||
|
explains the shape changes and decisions an application must make; it is not an
|
||||||
|
automatic migration tool. The [framework format reference](../formats.md) owns
|
||||||
|
the current configuration contract, and the root package GoDoc owns exact API
|
||||||
|
signatures and errors.
|
||||||
|
|
||||||
|
## Move Settings Under Promptkit
|
||||||
|
|
||||||
|
Place complete-document resources under `promptkit`, including `schema_root`,
|
||||||
|
`backends`, `profiles`, `prompts`, and `collections`. Old root-level resource
|
||||||
|
keys are application data and are not loaded as Promptkit resources; selecting
|
||||||
|
an unmigrated resource can therefore fail with a not-found error. IDs and
|
||||||
|
references do not acquire a `promptkit` prefix. `SettingsInput` mounts a
|
||||||
|
wrapperless fragment at `/promptkit`, and `CollectionInput` mounts one at
|
||||||
|
`/promptkit/collections/<namespace>`. Discovery helpers still return root-mounted
|
||||||
|
inputs; set destinations explicitly for discovered fragments. Standalone
|
||||||
|
id-first files use native sequence-item placement. The [format reference](../formats.md)
|
||||||
|
describes extensions and strict nested objects.
|
||||||
|
|
||||||
|
Applications can instead embed `promptkit.Settings`, call `Configure` on their
|
||||||
|
ConfigKit definition, load once, and pass the whole result to `NewFromConfig`.
|
||||||
|
The [preparation example](../../examples/go-library/prepare/main.go) shows the
|
||||||
|
workflow. Select `BundledOpenRouterUnderlay` before an application-owned load;
|
||||||
|
the default bundle option applies only to `NewEngine`. The application owns
|
||||||
|
version policy; Promptkit no longer validates resource-version fields.
|
||||||
|
|
||||||
|
Replace `RunRequest.Validation` values of type `*OutputContract` with
|
||||||
|
`*OutputContractInput` and use a pointer for `RepairAttempts`. Nil or null
|
||||||
|
selects zero for `none` and one for `basic`, `json`, and `json_schema`;
|
||||||
|
explicit zero disables correction. Runtime replacements resolve independently
|
||||||
|
of configured budgets. Inspection reports resolved integer budgets, including
|
||||||
|
zero. A failed validation can cause an additional generation and associated
|
||||||
|
latency and usage; a correction is not guaranteed. See the
|
||||||
|
[output contract](../formats.md#output-contract).
|
||||||
|
|
||||||
|
## Update Go Imports
|
||||||
|
|
||||||
|
Promptkit's module and public package import path is now
|
||||||
|
`go.maximumdirect.net/promptkit`, replacing
|
||||||
|
`gitea.maximumdirect.net/eric/promptkit`. Update application, library, test, and
|
||||||
|
generated imports, then select a release using the new declaration:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@VERSION
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace `VERSION` with a release that declares the vanity module path.
|
||||||
|
Earlier tags, including v0.16.0, retain their original module declaration and
|
||||||
|
use the former import path. No permanent `replace` directive is needed for the
|
||||||
|
cutover. Source browser, licensing, and hosted-release links continue to use
|
||||||
|
the Forge repository URLs, independently of Go's module identifier.
|
||||||
|
|
||||||
|
The package name remains `promptkit`. The path change itself does not rename
|
||||||
|
exported APIs or change their behavior, but Go treats packages imported through
|
||||||
|
the two paths as distinct. Update dependent packages that exchange Promptkit
|
||||||
|
types together and review any old-path dependencies retained after tidying.
|
||||||
|
The configuration and API adaptations below are also part of this migration.
|
||||||
|
|
||||||
|
## Select And Compose The Configuration You Need
|
||||||
|
|
||||||
|
Supply each backend, profile, prompt, schema, and collection source that the
|
||||||
|
application intends to use. `NewEngine` selects the bundled OpenRouter underlay
|
||||||
|
by default; callers explicitly select other sources. An application may select an explicit
|
||||||
|
collection input with `CollectionInput`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
input, err := promptkit.CollectionInput(
|
||||||
|
"openrouter",
|
||||||
|
configkit.File("config/openrouter.yaml"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Collection resource IDs are local IDs, while their runtime identities are
|
||||||
|
qualified. For example, a profile with `id: gemini-flash-latest` in collection
|
||||||
|
`openrouter` is selected as `openrouter/gemini-flash-latest`. Stored backend,
|
||||||
|
prompt-default, and other resource references must use their exact intended IDs;
|
||||||
|
Promptkit does not rewrite references when a source is mounted. `extends`
|
||||||
|
selects a sibling ID in the same list, not a cross-collection reference.
|
||||||
|
Review collisions and all references when moving resources into collections.
|
||||||
|
|
||||||
|
A collection's `release_version` remains an optional label. Prompt-version
|
||||||
|
selection has been removed: choose one definition per ID. Application-specific
|
||||||
|
version data may be retained at supported extension points, but is not used by
|
||||||
|
Promptkit for resource selection.
|
||||||
|
|
||||||
|
For example, change a version-keyed definition:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prompts:
|
||||||
|
report:
|
||||||
|
"2.0":
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review this report.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
```
|
||||||
|
|
||||||
|
to one identified wrapperless fragment, mounted with `SettingsInput`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prompts:
|
||||||
|
- id: report
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review this report.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
```
|
||||||
|
|
||||||
|
The same conversion applies to backends and profiles. Sibling inheritance uses
|
||||||
|
identity references, and a later same-ID record replaces the whole lower record.
|
||||||
|
This is another wrapperless fragment to mount with `SettingsInput`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
profiles:
|
||||||
|
- id: base
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: example-model
|
||||||
|
- id: fast
|
||||||
|
extends: {id: base}
|
||||||
|
timeout_seconds: 30
|
||||||
|
```
|
||||||
|
|
||||||
|
To customize a lower resource's fields, add a distinct inherited variant as
|
||||||
|
above; when replacing the same ID, supply the complete intended record. Generic
|
||||||
|
append, replace, imports, and inheritance are described by the
|
||||||
|
[ConfigKit v1 configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md).
|
||||||
|
Source-root controls are now `config:`, not `configkit:`. Imports are optional
|
||||||
|
unless the selected import declares `required: true`.
|
||||||
|
|
||||||
|
Update ConfigKit imports to `go.maximumdirect.net/configkit` at v1.4.0: its Go
|
||||||
|
types are incompatible with the old module path. Promptkit now requires Go
|
||||||
|
1.27.0 or newer. Source builders take ordered slices with explicit IDs:
|
||||||
|
|
||||||
|
```go
|
||||||
|
endpoint, model := "https://example.invalid/v1", "example-model"
|
||||||
|
base := "base"
|
||||||
|
source := promptkit.ProfileSource("application", []promptkit.ProfileConfig{
|
||||||
|
{ID: "base", Endpoint: &endpoint, Model: &model},
|
||||||
|
{ID: "fast", Extends: &base},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace former maps passed to `BackendSource` in the same way, setting each
|
||||||
|
`BackendConfig.ID`. Duplicates are retained for loading to reject. Resource
|
||||||
|
inspection paths now address list indexes, for example
|
||||||
|
`configkit.Keys("promptkit", "profiles").Index(0).Key("model")`. Configuration inspection
|
||||||
|
and diffs expose resolved values, recorded normalization and provenance.
|
||||||
|
`InspectProfile` reports the selected resolved profile without backend completion.
|
||||||
|
Use `Prepare` or `PreparedExecution.Details` for completed request settings,
|
||||||
|
including backend defaults and request overrides; see the
|
||||||
|
[resource inspection guidance](pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
|
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
||||||
|
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
||||||
|
values and may also change because redundant captured fields were removed.
|
||||||
|
Invalidate persisted caches or comparisons that span this change; hashes from
|
||||||
|
inspection and preparation of the same captured definition still agree.
|
||||||
|
|
||||||
|
## Identity Mapping
|
||||||
|
|
||||||
|
The reviewed catalog identities have slash-free profile keys. Their documented
|
||||||
|
mapping is:
|
||||||
|
|
||||||
|
| Previous identity | Namespaced identity or action |
|
||||||
|
| --- | --- |
|
||||||
|
| Backend `openrouter` | `openrouter/default` |
|
||||||
|
| Backend `rakestrawhome` | `rakestrawhome/default` |
|
||||||
|
| Each OpenRouter catalog profile `p` | `openrouter/p`, retaining the exact local key `p` |
|
||||||
|
| Profile `rakestrawhome-gemma-4-31b` | `rakestrawhome/rakestrawhome-gemma-4-31b` |
|
||||||
|
| Built-in `promptkit-openrouter-standard` | No automatic replacement; optionally define caller profile `openrouter/standard` |
|
||||||
|
| Built-in `promptkit-openrouter-auto` | No automatic replacement; optionally define caller profile `openrouter/auto` |
|
||||||
|
|
||||||
|
These are conversion examples, not aliases or bundled resources. Mounting old
|
||||||
|
`/v3` assets does not update backend references, profile inheritance, prompt
|
||||||
|
defaults, or diagnostic defaults inside them. Edit those values to the exact
|
||||||
|
qualified IDs you chose.
|
||||||
|
|
||||||
|
For example, the following wrapperless fragments can be mounted with
|
||||||
|
`SettingsInput`. An optional caller-owned standard profile may extend a selected
|
||||||
|
Gemini profile in the same `openrouter` collection:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
collections:
|
||||||
|
openrouter:
|
||||||
|
profiles:
|
||||||
|
- id: standard
|
||||||
|
extends: {id: gemini-flash-latest}
|
||||||
|
```
|
||||||
|
|
||||||
|
An optional auto profile may instead select the exact backend and model values:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
collections:
|
||||||
|
openrouter:
|
||||||
|
profiles:
|
||||||
|
- id: auto
|
||||||
|
backend: openrouter/default
|
||||||
|
model: openrouter/auto
|
||||||
|
```
|
||||||
|
|
||||||
|
Neither optional profile name is a Promptkit contract. Keep provider model
|
||||||
|
strings and settings as selected by the application, including any meaningful
|
||||||
|
prefixes. If a backend declares diagnostic defaults, update them to exact
|
||||||
|
qualified IDs; an unselected diagnostic default is not automatically made
|
||||||
|
resolvable by migration.
|
||||||
|
|
||||||
|
## Prompt Selection API
|
||||||
|
|
||||||
|
Prompt selection is ID-only. Remove version selectors from calls to `Run`,
|
||||||
|
`Prepare`, `PrepareExecution`, `ValidateRequest`, `InspectRequirements`, and
|
||||||
|
`SmokeTest`. Remove the `PromptVersion` field from `RunRequest`,
|
||||||
|
`RequirementInspectionRequest`, and `SmokeTestRequest`; `InspectPrompt` now
|
||||||
|
has the form `InspectPrompt(ctx, promptID)`. For example, select the prompt
|
||||||
|
explicitly in the request rather than setting a version selector:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "support/report",
|
||||||
|
ProfileID: "support/standard",
|
||||||
|
}
|
||||||
|
_, err = engine.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
If the former format contained multiple versions under one ID, choose the one
|
||||||
|
definition the application should install. Promptkit no longer retains or
|
||||||
|
returns its former descriptive label.
|
||||||
|
|
||||||
|
The public inventory APIs can help an operator inspect a captured effective
|
||||||
|
configuration: `ListProfiles`, `ListPrompts`, and `ListBackends` return exact
|
||||||
|
identities, while `ListNamespaces` and `ListCollections` expose resource-bearing
|
||||||
|
namespaces and effective collection declarations. Inventory does not select
|
||||||
|
which configuration version the application should deploy.
|
||||||
|
|
||||||
|
## Deployment Selection Remains With The Application
|
||||||
|
|
||||||
|
Applications decide which checked-in, embedded, operator-supplied, or
|
||||||
|
programmatic inputs to compose for each deployment. They also decide when to
|
||||||
|
construct a replacement engine after changing those sources. Promptkit does
|
||||||
|
not fetch catalog updates, select provider availability, validate upstream
|
||||||
|
catalog release status, or automatically migrate arbitrary consumer files.
|
||||||
|
The maintained [offline collection example](../../examples/go-library/prepare/main.go)
|
||||||
|
shows explicit source selection, local imports, layering, a scoped view, and
|
||||||
|
operator inventory without a provider call.
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -43,13 +43,13 @@ Before starting work:
|
|||||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||||
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. |
|
| Caller-supplied configuration sources | The [internal source document](internal/sources.md), [framework format reference](formats.md), and [testing policy](policy/testing.md). |
|
||||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||||
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||||
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
||||||
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
| Licensing, dependencies, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
||||||
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
||||||
|
|
||||||
For cross-cutting changes, follow every applicable row. Do not create
|
For cross-cutting changes, follow every applicable row. Do not create
|
||||||
@@ -58,6 +58,14 @@ exist.
|
|||||||
|
|
||||||
## Maintainer Validation
|
## Maintainer Validation
|
||||||
|
|
||||||
|
Use the newest Go installation already available locally for ordinary
|
||||||
|
development checks. Put its `bin` directory first on `PATH` and set
|
||||||
|
`GOTOOLCHAIN=local` to keep that toolchain selected. For repository-isolated
|
||||||
|
checks, set `GOWORK=off`; consumers may use their own workspaces. The selected
|
||||||
|
validation patch does not by itself change the module's consumer minimum. See
|
||||||
|
the [release toolchain policy](policy/release.md#consumer-minimum-and-release-toolchain)
|
||||||
|
and [release procedure](release.md#prepare-the-candidate) for release selection.
|
||||||
|
|
||||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||||
successful checks for unchanged content merely to create a commit. During
|
successful checks for unchanged content merely to create a commit. During
|
||||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||||
@@ -67,7 +75,6 @@ the maintained examples when they are affected:
|
|||||||
go test ./...
|
go test ./...
|
||||||
go vet ./...
|
go vet ./...
|
||||||
gofmt -w path/to/changed.go
|
gofmt -w path/to/changed.go
|
||||||
python3 scripts/check-examples.py
|
|
||||||
git diff --check
|
git diff --check
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -77,12 +84,12 @@ describes its requirements and recovery. For an optional diagnostic of a
|
|||||||
committed candidate without publication, run `./scripts/releasekit.sh check
|
committed candidate without publication, run `./scripts/releasekit.sh check
|
||||||
VERSION`; this is not a required pre-publication step.
|
VERSION`; this is not a required pre-publication step.
|
||||||
|
|
||||||
The standalone `python3 scripts/check-examples.py` command checks semantic
|
The example workflows are covered by the Go suite; use
|
||||||
expectations for the three offline consumer examples. The focused
|
`go test ./examples/go-library/...` to run them separately during development.
|
||||||
`python3 scripts/test-release.py` command exercises the release tooling against
|
The focused `python3 scripts/test-release.py` command exercises the release
|
||||||
disposable local repositories; use it when changing that tooling, not as a
|
tooling against disposable local repositories; use it when changing that
|
||||||
recurring release gate. Tests and examples require no provider credentials or
|
tooling, not as a recurring release gate. Tests and examples require no provider
|
||||||
live inference calls.
|
credentials or live inference calls.
|
||||||
|
|
||||||
Ordinary validation can run on intended uncommitted work. Release validation
|
Ordinary validation can run on intended uncommitted work. Release validation
|
||||||
uses committed candidate contents and the selected local Go installation.
|
uses committed candidate contents and the selected local Go installation.
|
||||||
|
|||||||
382
docs/formats.md
382
docs/formats.md
@@ -6,29 +6,24 @@ This is the canonical reference for Promptkit's backend, profile, prompt, and
|
|||||||
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
||||||
how to select sources and use an engine; the
|
how to select sources and use an engine; the
|
||||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
||||||
owns outbound wire behavior. [ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit)
|
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md)
|
||||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||||
This reference describes what Promptkit does with the final effective values.
|
This reference describes what Promptkit does with the final effective values.
|
||||||
|
|
||||||
A configuration document has keyed top-level `backends`, `profiles`, and
|
A complete configuration document places Promptkit settings beneath `promptkit`.
|
||||||
`prompts` mappings. Their keys are literal IDs, not filename-derived identities.
|
The `backends`, `profiles`, and `prompts` lists contain explicit literal
|
||||||
Prompt versions are a second keyed mapping. Keys and references must be
|
`id` values; `collections` groups namespaced resources. IDs remain exact and
|
||||||
nonblank and remain exact: Promptkit does not trim or re-key them. Old embedded
|
do not include the `promptkit` path. Standalone resource fragments may omit
|
||||||
`id`, `version`, and `base_profile` fields are invalid. Unknown fields are
|
the wrapper when explicitly mounted as described below.
|
||||||
errors.
|
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backends:
|
promptkit:
|
||||||
local:
|
profiles:
|
||||||
|
- id: local-summary
|
||||||
endpoint: http://localhost:8000/v1
|
endpoint: http://localhost:8000/v1
|
||||||
concurrency_limit: 2
|
|
||||||
profiles:
|
|
||||||
local-summary:
|
|
||||||
backend: local
|
|
||||||
model: example-model
|
model: example-model
|
||||||
prompts:
|
prompts:
|
||||||
meeting.summary:
|
- id: meeting.summary
|
||||||
"1.0.0":
|
|
||||||
default_profile: local-summary
|
default_profile: local-summary
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -36,10 +31,63 @@ prompts:
|
|||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
|
collections:
|
||||||
|
support:
|
||||||
|
release_version: "1.4.0"
|
||||||
|
backends:
|
||||||
|
- id: offline
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
- id: fast
|
||||||
|
backend: support/offline
|
||||||
|
model: offline-model
|
||||||
```
|
```
|
||||||
|
|
||||||
Every effective backend, profile, and prompt definition is validated when
|
## Source Headers, Collections, And Resource IDs
|
||||||
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
|
||||||
|
Applications own ConfigKit's version policy and source controls. Promptkit adds
|
||||||
|
no resource-version rule, including for collection, prompt, or profile records.
|
||||||
|
An application's root `version` is distinct from a collection's descriptive
|
||||||
|
`release_version`. Old root-level `prompts` or `profiles` keys are application
|
||||||
|
data, not Promptkit resources, and are neither consumed nor specially rejected.
|
||||||
|
|
||||||
|
The `promptkit` section, each collection, backend, profile, and prompt retain
|
||||||
|
unrecognized fields as application data. Such fields compose normally and are
|
||||||
|
visible to application validation and configuration inspection, but do not
|
||||||
|
become provider parameters or appear in resource reports. Grouping them under
|
||||||
|
an application-specific key reduces collisions; a misspelled optional Promptkit
|
||||||
|
field can also be accepted there. Nested Promptkit-owned objects, including
|
||||||
|
message, input, output, and smoke-test objects, remain strict.
|
||||||
|
|
||||||
|
The root resource lists contain unqualified IDs. Each `collections` entry
|
||||||
|
has a namespace key and may contain `release_version` plus the same three
|
||||||
|
resource lists. Collection resource IDs are local keys; their runtime IDs
|
||||||
|
are `<namespace>/<local-key>`. Root and collection keys may be the same. A
|
||||||
|
namespace and local key must be nonblank and slash-free, and references must
|
||||||
|
name their intended exact ID; mounting a source does not rewrite references.
|
||||||
|
For example, the `support/fast` profile above stores `support/offline` as its
|
||||||
|
backend ID. `extends: {id: base}` refers to a sibling in the same resource
|
||||||
|
list. It is not a qualified or cross-collection reference.
|
||||||
|
|
||||||
|
`release_version` is an optional opaque collection label. Absent or null means
|
||||||
|
no label; an explicitly empty string is retained. It does not select a resource
|
||||||
|
or change execution. Runtime lookups
|
||||||
|
use the captured effective resource indexes. `InspectConfiguration` is an
|
||||||
|
explicit operator view of resolved settings and provenance. Selected field
|
||||||
|
normalizations and the derived queue default are recorded in that tree, with
|
||||||
|
source history retained. Request-time completion remains separate. Source
|
||||||
|
history is not a second runtime resolver, and the
|
||||||
|
retained effective shape is not flattened or rewritten into those indexes.
|
||||||
|
|
||||||
|
For examples of explicitly selecting native collection inputs and inspecting
|
||||||
|
inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
|
||||||
|
[maintained preparation example](../examples/go-library/prepare/main.go). For
|
||||||
|
breaking identity and prompt-selection changes, see the
|
||||||
|
[configuration migration guide](consumers/configuration-migration.md).
|
||||||
|
|
||||||
|
Every effective backend, profile, and prompt definition is validated during
|
||||||
|
the registered configuration load and engine capture, even if never selected.
|
||||||
|
A profile may
|
||||||
be partial: an endpoint and model need only be complete after request overrides
|
be partial: an endpoint and model need only be complete after request overrides
|
||||||
are applied. A prompt's default profile similarly need not be runnable without
|
are applied. A prompt's default profile similarly need not be runnable without
|
||||||
an overriding request. Missing request data, artifacts, schemas, credentials,
|
an overriding request. Missing request data, artifacts, schemas, credentials,
|
||||||
@@ -50,18 +98,19 @@ and static template syntax at construction. It does not reread configuration or
|
|||||||
message files on ordinary runs; construct a new engine to observe edits.
|
message files on ordinary runs; construct a new engine to observe edits.
|
||||||
Schema and request artifact reads still happen during preparation. Use
|
Schema and request artifact reads still happen during preparation. Use
|
||||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
report, resource inspection for resolved prompt, profile, and backend fields,
|
||||||
for request-specific rendered details. For identity discovery before inspection,
|
and preparation for completed request-specific details. For identity discovery before inspection,
|
||||||
see the consumer guide's [profile and prompt listing](consumers/pkg-promptkit.md#discover-available-profiles-and-prompts).
|
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
The `prompts` mapping selects a literal prompt ID, then a string version key:
|
The `prompts` list contains definitions selected by their literal IDs. The
|
||||||
|
following examples in this reference are wrapperless `promptkit` fragments
|
||||||
|
unless explicitly shown as complete documents; mount them with `SettingsInput`.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
prompts:
|
prompts:
|
||||||
meeting.summary:
|
- id: meeting.summary
|
||||||
"1.0.0":
|
|
||||||
default_profile: local-summary
|
default_profile: local-summary
|
||||||
description: Summarize a synthetic meeting note.
|
description: Summarize a synthetic meeting note.
|
||||||
session_id: '{{.session}}'
|
session_id: '{{.session}}'
|
||||||
@@ -88,10 +137,10 @@ prompts:
|
|||||||
`default_profile`, `description`, `session_id`, `inputs`, and cache-control
|
`default_profile`, `description`, `session_id`, `inputs`, and cache-control
|
||||||
fields may be absent or cleared with null where appropriate. A null whole
|
fields may be absent or cleared with null where appropriate. A null whole
|
||||||
definition, required role, output format, or validation mode is invalid.
|
definition, required role, output format, or validation mode is invalid.
|
||||||
`required` and `repair_attempts` default to false and zero, respectively, and
|
`required` defaults to false and is not nullable. An omitted or null
|
||||||
are not nullable. A request that omits a prompt version succeeds only if that
|
`repair_attempts` resolves according to `validation_mode`; see the output
|
||||||
prompt ID has exactly one effective version. `InspectPrompt` selects from the
|
contract below. `InspectPrompt` selects the same captured definition without
|
||||||
same captured definitions without rendering or resolving a profile.
|
rendering or resolving a profile.
|
||||||
|
|
||||||
### Inputs
|
### Inputs
|
||||||
|
|
||||||
@@ -143,49 +192,102 @@ messages; the integration contract defines its wire representation.
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `format` | yes | `text`, `markdown`, or `json` |
|
| `format` | yes | `text`, `markdown`, or `json` |
|
||||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` |
|
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema` |
|
||||||
| `schema_path` | for `json_schema` | Path in the selected schema source |
|
| `schema` | one selection for `json_schema` | Inline JSON Schema value |
|
||||||
| `repair_attempts` | no | Integer 0–3; default 0. Positive values need `basic`, `json`, or `json_schema` |
|
| `schema_file` | one selection for `json_schema` | File relative to the contributing field's source context |
|
||||||
|
| `schema_path` | one selection for `json_schema` | Path within the applicable explicit schema root |
|
||||||
|
| `repair_attempts` | no | Integer 0–3; omitted or null defaults to 0 for `none`, otherwise 1. Positive values need `basic`, `json`, or `json_schema` |
|
||||||
|
|
||||||
`none` skips content validation; `basic` requires nonempty content; `json`
|
`none` skips content validation; `basic` requires nonempty content; `json`
|
||||||
accepts any valid JSON value; `json_schema` checks the selected schema.
|
accepts any valid JSON value; `json_schema` checks the selected schema.
|
||||||
|
In `json_schema` mode exactly one nonnull `schema`, `schema_file`, or
|
||||||
|
`schema_path` must be selected. Null clears a selection; `false`, `true`, and
|
||||||
|
`{}` are values. Blank references are invalid. Other modes ignore these fields
|
||||||
|
for schema selection and loading. ConfigKit composes inline mappings normally;
|
||||||
|
switching forms can require explicitly clearing an inherited alternative.
|
||||||
|
Replacing an identified prompt record replaces its complete earlier record.
|
||||||
`format` determines output artifact metadata. JSON Schema mode also supplies
|
`format` determines output artifact metadata. JSON Schema mode also supplies
|
||||||
structured-output metadata to compatible clients. Plain `json` does not
|
structured-output metadata to compatible clients. Plain `json` does not
|
||||||
request a provider-native JSON-object constraint. A positive repair budget
|
request a provider-native JSON-object constraint. A positive repair budget
|
||||||
counts additional generations after failed validation; exhaustion returns the
|
counts additional generations after failed validation; exhaustion returns the
|
||||||
last candidate and its failed validation result. Operational errors remain
|
last candidate and its failed validation result. Operational errors remain
|
||||||
errors. A request-level `OutputContract` replaces the whole configured one,
|
errors. A request-level `OutputContractInput` replaces the whole configured
|
||||||
not individual fields; an empty request format defaults to `text`.
|
contract, not individual fields; an empty request format defaults to `text`.
|
||||||
|
Its omitted budget resolves from its own validation mode, not the prompt's
|
||||||
|
budget. Explicit zero disables repair. A failed validation can incur one extra
|
||||||
|
generation by default, with corresponding latency and usage; correction is not
|
||||||
|
guaranteed.
|
||||||
|
|
||||||
## Built-In Backends
|
## Backend Definitions
|
||||||
|
|
||||||
The selected catalog modules contribute these backend definitions as the
|
Backend definitions are supplied through caller-selected configuration
|
||||||
earliest underlay. Their settings can be overlaid by caller sources; these IDs
|
sources. The conventional backend IDs are names only; PromptKit does not
|
||||||
are not reserved against configuration changes.
|
install definitions for them.
|
||||||
|
|
||||||
| ID | Catalog endpoint | API-key environment name | Active generation limit |
|
|
||||||
| --- | --- | --- | ---: |
|
|
||||||
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 |
|
|
||||||
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 |
|
|
||||||
|
|
||||||
A backend definition may have `endpoint`, `api_key_env`,
|
A backend definition may have `endpoint`, `api_key_env`,
|
||||||
`concurrency_limit`, `queue_capacity`, and `extra_params`. The final endpoint
|
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
|
||||||
is required and must be an absolute HTTP(S) URL with a host, no user
|
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
|
||||||
information, query, or fragment. An omitted or null API-key environment name
|
user information, query, or fragment. An omitted or null API-key environment
|
||||||
means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
name means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
||||||
negative value is invalid. For positive concurrency, omitted/null queue
|
negative value is invalid. For positive concurrency, omitted/null queue
|
||||||
capacity defaults to 1024. An explicit queue capacity, including zero,
|
capacity defaults to 1024. An explicit queue capacity, including zero,
|
||||||
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
||||||
has no queue. Provider maps contain JSON-compatible values; null entries are
|
has no queue. Provider maps contain JSON-compatible values; null entries are
|
||||||
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
ordinary JSON null. A raw `api_key` is not a Promptkit credential field: an
|
||||||
|
unrecognized backend field is application data and is not automatically redacted
|
||||||
|
by Promptkit. Applications own sensitivity declarations and disclosure policy
|
||||||
|
for their extension fields.
|
||||||
|
|
||||||
|
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
||||||
|
It may contain `default_profile`, `reachability`, and `authentication`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
backends:
|
||||||
|
- id: openrouter
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
smoke_tests:
|
||||||
|
default_profile: application-profile
|
||||||
|
reachability:
|
||||||
|
endpoint: https://status.example/ready?check=1
|
||||||
|
authentication:
|
||||||
|
kind: openrouter_key
|
||||||
|
endpoint: https://openrouter.ai/api/v1/key
|
||||||
|
```
|
||||||
|
|
||||||
|
An omitted or null `smoke_tests` block disables these metadata defaults.
|
||||||
|
`default_profile` is preserved as an exact, nonblank profile ID; its existence
|
||||||
|
is checked only if an inference diagnostic selects that default. Omitted/null
|
||||||
|
`default_profile` means there is no inference default. Omitted/null
|
||||||
|
`reachability` or its endpoint uses the effective backend endpoint. An explicit
|
||||||
|
reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
|
||||||
|
host; unlike the backend base endpoint, its path and query are allowed. Null
|
||||||
|
means fallback, while an explicit blank value is invalid.
|
||||||
|
|
||||||
|
Omitted/null `authentication` means no authentication adapter is declared. A
|
||||||
|
non-null object requires `kind`, which is trimmed and case-normalized; the
|
||||||
|
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
|
||||||
|
uses the adapter's backend-relative destination (`openrouter_key` appends
|
||||||
|
`/key` to the effective backend base); an explicit endpoint follows the same
|
||||||
|
full-URL rules as reachability. `none` declares authentication inapplicable.
|
||||||
|
These values are configuration only: engine construction does not contact a
|
||||||
|
backend.
|
||||||
|
|
||||||
|
These fields are resolved through the ordinary ConfigKit source layers before
|
||||||
|
Promptkit validates the final effective values. Sibling inheritance can overlay
|
||||||
|
individual nested fields; a later same-ID resource replaces the whole record.
|
||||||
|
An explicit null can clear the whole `smoke_tests` block or one of its nested
|
||||||
|
objects. An endpoint set to null uses
|
||||||
|
its documented fallback, while an explicit blank endpoint is invalid. The
|
||||||
|
metadata does not change ordinary generation credential requirements or
|
||||||
|
provider settings.
|
||||||
|
|
||||||
## Profile Definitions
|
## Profile Definitions
|
||||||
|
|
||||||
Profiles are keyed by literal ID and may inherit a named parent with
|
Profiles have literal IDs and may inherit a sibling parent with
|
||||||
`extends`:
|
`extends`:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
profiles:
|
profiles:
|
||||||
local-summary:
|
- id: local-summary
|
||||||
backend: local
|
backend: local
|
||||||
model: example-model
|
model: example-model
|
||||||
temperature: 0.2
|
temperature: 0.2
|
||||||
@@ -193,8 +295,8 @@ profiles:
|
|||||||
timeout_seconds: 90
|
timeout_seconds: 90
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
local-summary-fast:
|
- id: local-summary-fast
|
||||||
extends: local-summary
|
extends: {id: local-summary}
|
||||||
timeout_seconds: 30
|
timeout_seconds: 30
|
||||||
reasoning_effort: low
|
reasoning_effort: low
|
||||||
```
|
```
|
||||||
@@ -206,7 +308,9 @@ Nonblank endpoints follow the backend URL rules; temperature is 0–2, top_p
|
|||||||
is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept
|
is 0–1, max_tokens and timeout_seconds are nonnegative. Provider maps accept
|
||||||
JSON-compatible values with nonempty keys and cannot collide with the built-in
|
JSON-compatible values with nonempty keys and cannot collide with the built-in
|
||||||
client's [standard request fields](integrations/openai-compatible-chat.md#request-body).
|
client's [standard request fields](integrations/openai-compatible-chat.md#request-body).
|
||||||
Credential values do not belong in profiles: raw `api_key` is invalid.
|
A raw `api_key` is not a Promptkit credential setting; an unrecognized profile
|
||||||
|
field is application data. Applications own its sensitivity declarations and
|
||||||
|
disclosure policy, just as for backend extension fields.
|
||||||
|
|
||||||
### Defaults And Overrides
|
### Defaults And Overrides
|
||||||
|
|
||||||
@@ -219,84 +323,158 @@ only a model or backend does not clear unrelated explicitly inherited fields.
|
|||||||
| Profile field | Absent | Explicit value | Explicit null |
|
| Profile field | Absent | Explicit value | Explicit null |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
||||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
|
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Selected backend endpoint |
|
||||||
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
||||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
||||||
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
||||||
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
||||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
|
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Backend lookup name, if any |
|
||||||
| `api_key_required` | False | Boolean, including false | False |
|
| `api_key_required` | False | Boolean, including false | False |
|
||||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
|
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Backend defaults |
|
||||||
|
|
||||||
|
An empty profile mapping does not clear backend map entries. After ConfigKit
|
||||||
|
composition and inheritance, a whole-map null and omission both permit backend
|
||||||
|
defaults. Nested maps merge recursively, while scalar and array values replace;
|
||||||
|
nulls within provider maps or arrays remain literal data.
|
||||||
|
When `api_key_required` is true and the profile source is absent or null,
|
||||||
|
execution suppresses the backend credential source; an explicitly configured
|
||||||
|
profile source remains in use.
|
||||||
|
|
||||||
|
For example, if the backend provides an endpoint, API-key environment name,
|
||||||
|
and provider map, a profile containing `endpoint: null`, `api_key_env: null`,
|
||||||
|
and `extra_params: null` now inherits those backend values after handoff (unless
|
||||||
|
the required-key rule suppresses the credential source). Earlier versions used
|
||||||
|
those nulls to suppress the corresponding fallback. Use `endpoint: ""` or
|
||||||
|
`api_key_env: ""` to clear those individual strings. There is no profile-level
|
||||||
|
whole-map suppression replacement; change the backend map through ordinary
|
||||||
|
ConfigKit composition when different provider defaults are needed. ConfigKit's
|
||||||
|
retained configuration and provenance continue to show the nulls.
|
||||||
|
|
||||||
An empty mapping does not clear inherited map entries; null clears the whole
|
|
||||||
map. Nested maps merge recursively, while scalar and array values replace.
|
|
||||||
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
||||||
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
||||||
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
||||||
data, not a configuration inheritance instruction. An explicit zero timeout
|
data, not a configuration inheritance instruction. An explicit zero timeout
|
||||||
does not disable the caller context or HTTP transport timeout.
|
does not disable the caller context or HTTP transport timeout.
|
||||||
|
|
||||||
|
### Bundled OpenRouter Configuration
|
||||||
|
|
||||||
|
The default bundle installs root backend `openrouter`, endpoint
|
||||||
|
`https://openrouter.ai/api/v1`, optional credential environment name
|
||||||
|
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
|
||||||
|
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
|
||||||
|
applies. No prompt, implicit profile selection, or tuning defaults are installed.
|
||||||
|
Disable the bundle for `NewEngine` with `WithBundledOpenRouter(false)`; an
|
||||||
|
application-owned load selects `BundledOpenRouterUnderlay` explicitly.
|
||||||
|
|
||||||
|
The root profile IDs and their models are:
|
||||||
|
|
||||||
|
| Profile | Model |
|
||||||
|
| --- | --- |
|
||||||
|
| `gpt-astra-latest` | `~openai/gpt-astra-latest` |
|
||||||
|
| `gpt-sol-latest` | `~openai/gpt-sol-latest` |
|
||||||
|
| `gpt-terra-latest` | `~openai/gpt-terra-latest` |
|
||||||
|
| `gpt-luna-latest` | `~openai/gpt-luna-latest` |
|
||||||
|
| `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
||||||
|
| `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
||||||
|
| `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
||||||
|
| `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
||||||
|
| `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
||||||
|
| `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
||||||
|
| `auto` | `openrouter/auto` |
|
||||||
|
| `free` | `openrouter/free` |
|
||||||
|
|
||||||
|
The profiles `auto-low`, `auto-medium`, `auto-high`, `auto-xhigh`, and
|
||||||
|
`auto-max` extend `auto` and add an `auto-router` plugin with the corresponding
|
||||||
|
`cost_tier`. Replacing `auto` changes their inherited backend and model while
|
||||||
|
retaining their plugins. These are root IDs even when a model contains `/`.
|
||||||
|
For example, a caller can replace `auto` as a complete record and extend another
|
||||||
|
bundled profile under a new ID:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
profiles:
|
||||||
|
- id: auto
|
||||||
|
backend: openrouter
|
||||||
|
model: openrouter/free
|
||||||
|
- id: my-luna
|
||||||
|
extends: {id: gpt-luna-latest}
|
||||||
|
timeout_seconds: 60
|
||||||
|
```
|
||||||
|
|
||||||
|
See OpenRouter's [alias examples](https://openrouter.ai/~openai/gpt-luna-latest),
|
||||||
|
[auto router and cost tiers](https://openrouter.ai/docs/guides/routing/routers/auto-router),
|
||||||
|
and [free router](https://openrouter.ai/docs/guides/routing/routers/free-router)
|
||||||
|
for provider behavior and availability.
|
||||||
|
|
||||||
### Source And Profile Precedence
|
### Source And Profile Precedence
|
||||||
|
|
||||||
`NewEngine` composes catalog definitions first, Promptkit-owned definitions
|
`NewEngine` composes the bundled OpenRouter underlay first by default, then
|
||||||
next, then caller underlays in their given order, with an optional main input
|
caller underlays in their given order, followed by an
|
||||||
last. Later tiers overlay earlier ones. Inputs within one underlay group are
|
optional main input. Later layers overlay earlier ones. Inputs within one
|
||||||
strictly composed: conflicting contributions are errors, not priority
|
underlay group are strictly composed: conflicting contributions are errors,
|
||||||
signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
||||||
in sorted order, but that order does not give one file higher precedence.
|
in sorted order, but that order does not give one file higher precedence.
|
||||||
An explicitly selected missing/unreadable root fails; an empty root contributes
|
An explicitly selected missing/unreadable root fails; an empty root contributes
|
||||||
nothing. Imports are disabled by these helpers unless the caller enables them
|
nothing. Imports are disabled by these helpers unless the caller enables them
|
||||||
on an input. Single files can use ConfigKit inputs directly.
|
on an input. Single files can use ConfigKit inputs directly.
|
||||||
|
|
||||||
Backends, profiles, and prompts all follow these same source rules. A later
|
Root backends, profiles, and prompts, as well as collection declarations and
|
||||||
profile or backend can change individual fields of a lower definition. A
|
their resource lists, follow these same source rules. `CollectionInput`
|
||||||
request `ProfileID` takes precedence over a prompt `default_profile`; if
|
mounts one explicitly selected source at `/promptkit/collections/<namespace>` and enables
|
||||||
|
ConfigKit imports for that input; it does not discover or select sources.
|
||||||
|
`SettingsInput` mounts wrapperless settings fragments at `/promptkit`; an
|
||||||
|
id-first resource file uses ConfigKit sequence-item placement at the matching
|
||||||
|
resource list. Directory discovery retains root destinations, so callers mount
|
||||||
|
discovered wrapperless fragments explicitly. No document shape is inferred.
|
||||||
|
A later same-ID record replaces the lower definition in full. Use a distinct
|
||||||
|
inherited sibling when a variant should retain its parent's fields.
|
||||||
|
A request `ProfileID` takes precedence over a prompt `default_profile`; if
|
||||||
neither is selected, preparation fails. Exact profile inspection selects one
|
neither is selected, preparation fails. Exact profile inspection selects one
|
||||||
explicit ID and does not consult a prompt default.
|
explicit ID and does not consult a prompt default.
|
||||||
|
|
||||||
### Profile Inheritance
|
### Resource Inheritance
|
||||||
|
|
||||||
ConfigKit resolves `extends` on effective profile definitions. One profile
|
ConfigKit resolves `extends: {id: parent}` for backends, profiles, and prompts.
|
||||||
may inherit another keyed profile; presence, explicit null, and recursively
|
Each child inherits a surviving sibling in the same root or collection list;
|
||||||
merged maps determine the final fields. References are literal IDs, including
|
collection inheritance cannot cross namespaces. Parents are ordinary validated
|
||||||
IDs containing punctuation. There is no Promptkit-specific depth limit or
|
resources, and profiles may remain partial until request completion. Presence, explicit
|
||||||
second base-chain resolver. Missing references and cycles fail at construction.
|
null, and recursively merged maps determine the final fields. Other resource
|
||||||
For generic inheritance and layer mechanics, consult
|
references are exact IDs and use their qualified form when they select a
|
||||||
[ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit).
|
collection resource. There is no Promptkit-specific depth limit or second
|
||||||
|
base-chain resolver. Missing references and cycles fail at construction. For
|
||||||
## Built-In Profile Catalog
|
generic inheritance and layer mechanics, consult
|
||||||
|
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.4.0/docs/config.md).
|
||||||
Promptkit contributes two opt-in definitions; neither is selected by default:
|
|
||||||
|
|
||||||
| ID | Current definition |
|
|
||||||
| --- | --- |
|
|
||||||
| `promptkit-openrouter-standard` | Extends the selected OpenRouter catalog's `gemini-flash-latest` profile. |
|
|
||||||
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
|
|
||||||
|
|
||||||
Select one through request `ProfileID` or prompt `default_profile`.
|
|
||||||
Caller layers may overlay these definitions or their catalog bases. The
|
|
||||||
[release policy](policy/release.md#catalog-profile-compatibility) owns their
|
|
||||||
cross-version compatibility boundary.
|
|
||||||
|
|
||||||
## External Profile Inventories
|
|
||||||
|
|
||||||
The selected catalog modules, not this document, own their profile inventories:
|
|
||||||
|
|
||||||
- [OpenRouter catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v3.20260930.0/catalog/profiles)
|
|
||||||
- [Rakestrawhome catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v3.20260930.0/catalog/profiles)
|
|
||||||
|
|
||||||
These are tested local data, not a promise of live provider availability.
|
|
||||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns
|
|
||||||
the compatibility boundary.
|
|
||||||
|
|
||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
Schemas can be inline, source-relative files, or files under an explicit root.
|
||||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
An optional `schema_root` may be set at the `promptkit` root and in each
|
||||||
referenced nested schemas resolve relative to the owning schema document.
|
collection. A collection's nonnull root takes precedence over the application
|
||||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
root for its prompts; an absent or null collection root falls back to the
|
||||||
invalid, or unresolvable schema causes an operational validation error.
|
application root. A nonnull blank root is invalid. There is no implicit
|
||||||
|
working-directory root. Roots and `schema_file` references resolve using the
|
||||||
|
effective field's contributing ConfigKit filesystem and base directory;
|
||||||
|
missing usable context is a configuration error. An explicit `.` is valid
|
||||||
|
with usable context. A `schema_path` requires an applicable root. Root choice
|
||||||
|
is made before reading: a missing file never triggers another-root search.
|
||||||
|
Schema selection and context are captured during construction without reading
|
||||||
|
schema files. Files are read and compiled during preparation; a missing,
|
||||||
|
malformed, or unresolvable file is an operational validation error. Inline
|
||||||
|
configuration values follow native ConfigKit composition and null handling.
|
||||||
|
For request-level replacement, inline `Schema` needs no root and `SchemaPath`
|
||||||
|
uses the selected prompt's applicable root. A replacement does not inherit the
|
||||||
|
configured schema selection or `schema_file` context.
|
||||||
|
Runtime inline values are encoded as JSON before preparation; file-backed JSON
|
||||||
|
uses the existing number-preserving decoder. No additional numerical equivalence
|
||||||
|
between input representations is implied.
|
||||||
|
|
||||||
|
Each schema must be self-contained: in-document references, IDs, and anchors are
|
||||||
|
supported, but references requiring another file or network resource fail.
|
||||||
|
References must also work without the source filename as a base; use fragment
|
||||||
|
references or explicit schema IDs rather than relying on that filename.
|
||||||
Valid JSON that fails the schema instead yields a failed validation result.
|
Valid JSON that fails the schema instead yields a failed validation result.
|
||||||
|
Schema documents are detached without applying the provider extra-parameter
|
||||||
|
copy limits; this does not change the schema decoder, compiler, or
|
||||||
|
self-contained-resource rules.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
|
|||||||
@@ -16,14 +16,13 @@ Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
|||||||
Before the client is called, the engine resolves framework, backend, profile,
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
and request values into one execution target. Endpoint configuration is trimmed
|
and request values into one execution target. Endpoint configuration is trimmed
|
||||||
and must be an absolute HTTP or HTTPS URL with a host and without user
|
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||||
information, a query, or a fragment. A non-empty endpoint from the target
|
information, a query, or a fragment. The target's endpoint is validated again
|
||||||
overrides the client's configured base URL. The final selected endpoint is
|
before transport.
|
||||||
validated again before transport.
|
|
||||||
|
|
||||||
The completion URL is composed through parsed URL path operations. Nested base
|
The completion URL is composed through parsed URL path operations. Nested base
|
||||||
paths are retained, repeated trailing slashes are normalized, and the result
|
paths are retained, repeated trailing slashes are normalized, and the result
|
||||||
has exactly one appended `/chat/completions` suffix. Generation fails before
|
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||||
sending when neither source supplies a valid endpoint.
|
sending when the target does not supply a valid endpoint.
|
||||||
|
|
||||||
The target's backend ID is routing metadata for prepared values, results, and
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
injected clients. The built-in client does not derive the URL from that ID and
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
@@ -45,15 +44,57 @@ precedence over the backend default. Only the name reaches prepared metadata;
|
|||||||
the environment value is read just before the provider call and is never added
|
the environment value is read just before the provider call and is never added
|
||||||
to the JSON body.
|
to the JSON body.
|
||||||
|
|
||||||
|
## Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
Promptkit's explicit smoke diagnostics are separate consumer operations; they
|
||||||
|
are not part of ordinary `Run`, the offline examples, or release validation.
|
||||||
|
With the built-in client, inference diagnostics use the same configured client
|
||||||
|
and `/chat/completions` request as generation. Their content cases are private
|
||||||
|
to Promptkit and are not application prompt, schema, or provider protocol
|
||||||
|
extensions. The public
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns request selection,
|
||||||
|
report facts, and injected-client behavior.
|
||||||
|
|
||||||
|
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
|
||||||
|
one unauthenticated `GET` to the selected diagnostic URL; an observed response
|
||||||
|
counts as reachable even when its status is non-2xx. Authentication behavior
|
||||||
|
is selected from backend metadata, not inferred from the host. The
|
||||||
|
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
|
||||||
|
backend base, unless configuration supplies a full diagnostic endpoint (see
|
||||||
|
the [backend format reference](../formats.md#backend-definitions)). It may send
|
||||||
|
the available bearer credential, but can dispatch without one. A 401 is
|
||||||
|
recognized as rejection. A 200 is recognized as acceptance only when its JSON
|
||||||
|
object has a `data` object containing string `label`, Boolean `is_free_tier`,
|
||||||
|
and numeric `usage`; unrelated fields and account limits are ignored. This is
|
||||||
|
only a minimal identifying subset of OpenRouter's documented
|
||||||
|
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||||
|
not validation of the full account document. Other statuses and unrecognized
|
||||||
|
200 responses do not establish acceptance. A 200 response exceeding the bounded
|
||||||
|
recognition allowance reports `authentication_response_limit_exceeded`, leaving
|
||||||
|
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||||
|
RoundTripper boundary and any response associated with it. It includes the
|
||||||
|
selected destination without URL user information, response status, and
|
||||||
|
credential presence when observed. Redirects can change the last observed
|
||||||
|
destination or strip credentials. This evidence does not prove that bytes
|
||||||
|
reached a peer or reveal dispatch hidden inside a custom transport. Diagnostic
|
||||||
|
reports do not retain headers, credentials, provider
|
||||||
|
bodies, or generated text. A consumer-injected `LLMClient` can support inference
|
||||||
|
diagnostics, but does not establish HTTP status, destination, reachability, or
|
||||||
|
upstream credential-supply facts; Promptkit does not fall back to a built-in
|
||||||
|
transport for HTTP-only diagnostics.
|
||||||
|
|
||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
target's model takes precedence over the client's configured model, and one
|
target's model must be available.
|
||||||
must be available.
|
|
||||||
|
|
||||||
Each ordinary message contains its `role` and string `content`. A
|
Each ordinary message contains its `role` and string `content`. A
|
||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||||
|
For a continuation, captured history precedes the new definition and appended
|
||||||
|
messages. Ancestry is local metadata and is not an upstream request field.
|
||||||
|
|
||||||
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
Promptkit sends only `developer`, `system`, `user`, and `assistant` roles and
|
||||||
does so without provider-specific translation. Tool and deprecated function
|
does so without provider-specific translation. Tool and deprecated function
|
||||||
@@ -76,9 +117,10 @@ The client conditionally includes:
|
|||||||
strict flag, and schema document. Plain JSON validation does not add an
|
strict flag, and schema document. Plain JSON validation does not add an
|
||||||
object-only response constraint.
|
object-only response constraint.
|
||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
Configured profile maps recursively overlay backend defaults, while a nonempty
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
runtime override replaces the complete configured map. The [format reference](../formats.md#defaults-and-overrides)
|
||||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
owns these resolution rules. The resulting effective map is merged into the
|
||||||
|
top-level body. Empty keys, values that cannot be encoded, and
|
||||||
collisions with these reserved fields are rejected before any provider call:
|
collisions with these reserved fields are rejected before any provider call:
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
|
|||||||
@@ -68,9 +68,8 @@ Preparation and validation never hold an active permit.
|
|||||||
|
|
||||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||||
returns an idempotent release function. The coordinator uses it around
|
returns an idempotent release function. The coordinator uses it around
|
||||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
individual attempts. There is one queue and one active limit per limited backend
|
||||||
operation for focused internal callers and tests, but engine assembly no
|
pool.
|
||||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
|
||||||
|
|
||||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||||
@@ -88,6 +87,17 @@ release, including canceled acquisition without dispatch. A retry wait holds
|
|||||||
the logical admission lease but no active permit. Observer callbacks must not
|
the logical admission lease but no active permit. Observer callbacks must not
|
||||||
block on work that requires the same run's admission slot.
|
block on work that requires the same run's admission slot.
|
||||||
|
|
||||||
|
## Smoke-Test Capacity
|
||||||
|
|
||||||
|
The engine's smoke tester shares this manager and generation coordinator with
|
||||||
|
the ordinary runner. Inference diagnostics reserve the same logical admission
|
||||||
|
slot and acquire the same backend-scoped active permits for actual model
|
||||||
|
attempts, including retries. They therefore participate in the same configured
|
||||||
|
backend limits as `Run`. Reachability and authentication HTTP probes do not
|
||||||
|
enter generation admission or consume active permits. The public
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns the caller-visible
|
||||||
|
operation and injected-client limits.
|
||||||
|
|
||||||
## Cancellation And Release
|
## Cancellation And Release
|
||||||
|
|
||||||
Admission checks the caller context before reserving a slot. A call canceled
|
Admission checks the caller context before reserving a slot. A call canceled
|
||||||
@@ -110,7 +120,7 @@ validation, bounded admission, idempotent release, context handling, and
|
|||||||
unlimited admission. The
|
unlimited admission. The
|
||||||
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
unlimited calls, passthrough behavior, and panic release.
|
unlimited calls, and panic release using test-only one-shot calls.
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
admission, lease lifetime, failure release, and shared initial/repair
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
@@ -127,4 +137,7 @@ cover shared and independent backend pools during retry and repair. The
|
|||||||
own the public prepared-capacity boundary. The
|
own the public prepared-capacity boundary. The
|
||||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||||
the public generation category and context identity when generation is
|
the public generation category and context identity when generation is
|
||||||
canceled.
|
canceled. The assembled
|
||||||
|
[smoke diagnostic contract tests](../../smoke_integration_contract_test.go)
|
||||||
|
verify that inference diagnostics share the engine's Run admission and recover
|
||||||
|
capacity after cancellation or failure.
|
||||||
|
|||||||
@@ -41,9 +41,22 @@ original error chain. This keeps injected attempt classifications consistent
|
|||||||
with terminal observations without parsing diagnostic text or retrying injected
|
with terminal observations without parsing diagnostic text or retrying injected
|
||||||
clients.
|
clients.
|
||||||
|
|
||||||
|
The diagnostic path uses the same private attempt and retry implementation as
|
||||||
|
ordinary generation. It retains invocation-local transport, response,
|
||||||
|
generation, usage, capacity-wait, and retry-delay evidence before normal
|
||||||
|
result-discard and cancellation checks. Built-in preparation also supplies its
|
||||||
|
frozen request URL to the evidence path before attempts begin; injected
|
||||||
|
generation has no equivalent HTTP destination. This evidence is returned to
|
||||||
|
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
|
||||||
|
observer accounting. The [model-client document](llm.md) describes the
|
||||||
|
transport-observation boundary, and the root
|
||||||
|
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.
|
||||||
|
|
||||||
Focused coordinator tests use controlled transport and wait functions. They
|
Focused coordinator tests use controlled transport and wait functions. They
|
||||||
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
||||||
release, and the injected-client path. The capacity manager tests own permit
|
release, and the injected-client path. The
|
||||||
|
[evidence tests](../../internal/generation/evidence_test.go) own diagnostic
|
||||||
|
attempt retention and retry evidence. The capacity manager tests own permit
|
||||||
acquisition and idempotent release. Public
|
acquisition and idempotent release. Public
|
||||||
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
||||||
repair, observation, frozen prepared execution, and shared-backend behavior
|
repair, observation, frozen prepared execution, and shared-backend behavior
|
||||||
|
|||||||
@@ -26,10 +26,9 @@ client uses its endpoint, credential metadata, generation fields, and extra
|
|||||||
parameters. `BackendID` remains routing metadata
|
parameters. `BackendID` remains routing metadata
|
||||||
for the generation boundary and is not mapped into the provider payload.
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
Construction trims and validates a nonempty configured base URL and clones any
|
Construction clones any supplied `http.Client` so Promptkit can apply its
|
||||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
timeout default without mutating the caller's client. Generation preparation
|
||||||
mutating the caller's client. An empty configured base remains valid because a
|
then:
|
||||||
resolved request target may supply the endpoint. Generation preparation then:
|
|
||||||
|
|
||||||
1. validates shared execution-setting invariants and the final selected base
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
endpoint;
|
endpoint;
|
||||||
@@ -42,6 +41,10 @@ resolved request target may supply the endpoint. Generation preparation then:
|
|||||||
the complete-request failure remains a request failure;
|
the complete-request failure remains a request failure;
|
||||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||||
|
|
||||||
|
Provider values are detached with trusted copying before encoding. Custom JSON
|
||||||
|
encoding methods and per-value encoding-failure diagnostics operate on those
|
||||||
|
private copies rather than retained prepared or diagnostic metadata.
|
||||||
|
|
||||||
Each execution of that prepared request creates a fresh HTTP request, body
|
Each execution of that prepared request creates a fresh HTTP request, body
|
||||||
reader, and attempt timeout context. It closes the response body and returns
|
reader, and attempt timeout context. It closes the response body and returns
|
||||||
the HTTP status, decoded response or error, reported token values and presence,
|
the HTTP status, decoded response or error, reported token values and presence,
|
||||||
@@ -59,6 +62,34 @@ This client does not schedule retries; the [generation coordinator](generation.m
|
|||||||
does so around its prepared attempts. The client has no tool-call support,
|
does so around its prepared attempts. The client has no tool-call support,
|
||||||
provider catalog, inbound HTTP behavior, or durable session store.
|
provider catalog, inbound HTTP behavior, or durable session store.
|
||||||
|
|
||||||
|
## Diagnostic HTTP Evidence
|
||||||
|
|
||||||
|
Built-in diagnostics wrap the selected HTTP client's `RoundTripper` for one
|
||||||
|
invocation using a shallow client copy, preserving its timeout, redirect policy,
|
||||||
|
cookie jar, and underlying transport without mutating the configured client.
|
||||||
|
Evidence distinguishes a diagnostic dispatch invocation from an observed
|
||||||
|
request and response. A newly observed request replaces the previous
|
||||||
|
destination-specific response evidence, so redirects retain only the last
|
||||||
|
observed hop. A response is recorded before envelope decoding or later context
|
||||||
|
checks can discard a generation result.
|
||||||
|
|
||||||
|
The collector keeps only the last observed destination (without URL user
|
||||||
|
information), status, credential-presence, response-availability, decoded
|
||||||
|
generation, and presence-aware usage facts needed by the diagnostic. It does
|
||||||
|
not retain headers, credentials, raw request or response bodies, or a redirect
|
||||||
|
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||||
|
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||||
|
Reachability and named authentication probes use the same configured built-in
|
||||||
|
client. The OpenRouter adapter owns endpoint derivation, bounded account-response
|
||||||
|
recognition, and body closure; the smoke coordinator interprets its evidence. The
|
||||||
|
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||||
|
owns their observable HTTP semantics.
|
||||||
|
|
||||||
|
An injected `LLMClient` does not pass through this HTTP boundary. Its returned
|
||||||
|
response can establish response and inference facts, but not an HTTP
|
||||||
|
destination, status, reachability, or upstream credential-supply fact. The
|
||||||
|
facade does not silently install a built-in transport for those diagnostics.
|
||||||
|
|
||||||
## Prepared Generation
|
## Prepared Generation
|
||||||
|
|
||||||
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||||
@@ -78,14 +109,13 @@ public ownership and redaction semantics belong to the
|
|||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid generation
|
||||||
configuration, invalid generation requests, request execution failures,
|
requests, request execution failures, non-success provider statuses, and
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
malformed successful responses. Provider response bodies are never exposed in
|
||||||
response bodies are never exposed in raw form through non-success errors.
|
raw form through non-success errors.
|
||||||
|
|
||||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
A missing or invalid final selected endpoint is an invalid generation request
|
||||||
invalid final selected endpoint is an invalid generation request and is
|
and is rejected before transport.
|
||||||
rejected before transport.
|
|
||||||
|
|
||||||
Authentication resolves a trimmed direct key before a trimmed configured
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
@@ -129,6 +159,13 @@ caller deadlines, generation deadlines, and whole-request client timeouts
|
|||||||
retain `context.DeadlineExceeded`. The runner adds its generation category
|
retain `context.DeadlineExceeded`. The runner adds its generation category
|
||||||
without discarding those identities or depending on HTTP status mapping.
|
without discarding those identities or depending on HTTP status mapping.
|
||||||
|
|
||||||
|
When a transport failure carries a custom context cancellation cause, the
|
||||||
|
client also retains the standard context identity. It correlates that cause
|
||||||
|
with the returned failure rather than labeling unrelated transport errors from
|
||||||
|
context state alone. Cancellation while reading a successful response body
|
||||||
|
retains the original read failure alongside the context identity within the
|
||||||
|
redacting wrapper.
|
||||||
|
|
||||||
## Test Ownership
|
## Test Ownership
|
||||||
|
|
||||||
The
|
The
|
||||||
@@ -137,7 +174,10 @@ own configuration, client cloning, deterministic deadline precedence,
|
|||||||
authentication, request and response mapping, malformed data, error identity,
|
authentication, request and response mapping, malformed data, error identity,
|
||||||
cancellation, endpoint selection and composition, pre-transport rejection, and
|
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||||
bounded single-document successful-response framing, closure, and
|
bounded single-document successful-response framing, closure, and
|
||||||
response-body suppression. The focused
|
response-body suppression. The [HTTP evidence tests](../../internal/llm/http_evidence_test.go)
|
||||||
|
own invocation-local last-hop request/response facts, while the
|
||||||
|
[prepared-generation evidence tests](../../internal/llm/prepared_generation_evidence_test.go)
|
||||||
|
own retention across decoding and cancellation. The focused
|
||||||
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
||||||
own envelope parsing, normalization, and bounded-reader cases; their
|
own envelope parsing, normalization, and bounded-reader cases; their
|
||||||
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
||||||
|
|||||||
@@ -10,6 +10,13 @@ lifecycle instrumentation spans the facade and runner boundaries.
|
|||||||
Prepared execution starts only after its private payload is atomically claimed;
|
Prepared execution starts only after its private payload is atomically claimed;
|
||||||
the facade receives that invocation's recorder privately and emits the terminal
|
the facade receives that invocation's recorder privately and emits the terminal
|
||||||
event after public error mapping. Failed claims and preparation emit nothing.
|
event after public error mapping. Failed claims and preparation emit nothing.
|
||||||
|
The public projection retains private operation-owner metadata separately from
|
||||||
|
inspectable child causes. Terminal classification uses that owner, preserving
|
||||||
|
context precedence and generation-specific transport/provider details rather
|
||||||
|
than mistaking a collaborator's nested lookup error for the parent's operation.
|
||||||
|
Timeout attribution is finalized and the child scope released before terminal
|
||||||
|
delivery, so observer latency cannot change the already selected outcome.
|
||||||
|
Prepared-content notification is a separate opt-in callback, not an event.
|
||||||
|
|
||||||
The facade masks inherited recorder and prepared-claim state when entering
|
The facade masks inherited recorder and prepared-claim state when entering
|
||||||
preparation or execution. Delegated calls from an injected collaborator retain
|
preparation or execution. Delegated calls from an injected collaborator retain
|
||||||
@@ -23,6 +30,11 @@ callback. The caller supplies normalized failure metadata; the recorder never
|
|||||||
parses an error string or receives provider bodies, credentials, headers, or
|
parses an error string or receives provider bodies, credentials, headers, or
|
||||||
other content. Events are values with no mutable containers. Callback panics
|
other content. Events are values with no mutable containers. Callback panics
|
||||||
propagate to the caller.
|
propagate to the caller.
|
||||||
|
The immediate parent run ID is set before start delivery and is retained as
|
||||||
|
selection metadata changes. It describes ancestry, not implicit group membership.
|
||||||
|
Resolved prompt and profile IDs are the exact runtime identities selected for
|
||||||
|
the invocation, including namespace qualification when supplied by an engine
|
||||||
|
view.
|
||||||
Per-operation callbacks are ordered and synchronous; different operations may
|
Per-operation callbacks are ordered and synchronous; different operations may
|
||||||
invoke the same callback concurrently. The recorder marks each measured phase's
|
invoke the same callback concurrently. The recorder marks each measured phase's
|
||||||
end before delivery, so callback work does not inflate provider duration.
|
end before delivery, so callback work does not inflate provider duration.
|
||||||
@@ -41,6 +53,18 @@ attempts above the first in a generation, and repair generations with at least
|
|||||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||||
attempt events.
|
attempt events.
|
||||||
|
|
||||||
|
An execution group is a zero-ready synchronized collector of explicitly selected
|
||||||
|
events. It enrolls by run ID and aggregates each run's first terminal counters and
|
||||||
|
presence-aware usage exactly once. Snapshots copy members and totals under one
|
||||||
|
lock; terminal coverage and token completeness are distinct. Observer fan-out
|
||||||
|
copies the callback slice and invokes callbacks synchronously without holding a
|
||||||
|
group lock. No group history, registry, or automatic nested membership exists.
|
||||||
|
|
||||||
|
Smoke diagnostics mask inherited execution-recorder state while invoking
|
||||||
|
generation or HTTP collaborators. Their private attempt evidence is assembled
|
||||||
|
into a `SmokeTestReport`; it does not create ordinary run events or enter an
|
||||||
|
enclosing run's observation totals.
|
||||||
|
|
||||||
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
||||||
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
||||||
and callback panic behavior. The [generation coordinator](generation.md) owns
|
and callback panic behavior. The [generation coordinator](generation.md) owns
|
||||||
|
|||||||
@@ -11,28 +11,28 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, resource and requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
| `examples/go-library/observe` | Demonstrates offline retry, observation, prepared notification, and separately budgeted coordinated proposal and assessment runs using two profiles on one finite backend. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||||
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
| `internal/configuration` | Supplies component rules and captures final shared settings and native inspection evidence into owned indexed resources, compiled templates, requirement metadata, identities, and execution profiles. | [Configuration capture](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions, referenced message content, compiled templates, input metadata snapshots, sorted identity-only profile and prompt inventories, backend and presence-aware profile snapshots, and an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
| `internal/settings` | Declares the shared public configuration data model exposed through root aliases under ADR 0002; contains no loading or execution logic. | [Settings declarations](../../internal/settings/settings.go), [ADR 0002](../adr/0002-shared-configuration-data-model.md) |
|
||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
| `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The internal engine assembles these internal components without exposing their
|
The internal engine assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -30,45 +30,98 @@ internal callers and tests.
|
|||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
|
|
||||||
|
Request output-contract replacements carry an optional repair budget. The
|
||||||
|
runner resolves that budget from the replacement's validation mode before
|
||||||
|
building the concrete validation plan; prepared execution retains the resolved
|
||||||
|
integer. Configured budgets are already resolved by the configuration adapter.
|
||||||
|
|
||||||
|
Captured exchanges are immutable parent-linked domain values. Each holds only
|
||||||
|
its new rendered suffix, final assistant message, and run identities; flattening
|
||||||
|
iteratively fills a single detached message buffer for preparation or projects
|
||||||
|
directly into public inspection values. Preparation
|
||||||
|
prepends this history to newly rendered definition messages and appended caller
|
||||||
|
messages before computing the rendered hash. Follow-up selection and credentials
|
||||||
|
remain those of the new request. Prepared execution retains capture and parent
|
||||||
|
references in its single-claim payload, not its durable details; discard clears
|
||||||
|
the payload. Completed runs capture the exact final output, including exhausted
|
||||||
|
structural repair, while operational errors produce no exchange.
|
||||||
|
|
||||||
|
The sequential coordinator uses ordinary facade runs and retains the current
|
||||||
|
immutable exchange independently of callback-owned values. Each run releases its
|
||||||
|
logical admission lease and finishes observation before application callback
|
||||||
|
delivery. Context checks preserve a completed but undelivered result only when
|
||||||
|
cancellation is observed before invoking the callback. Coordinator budgets count
|
||||||
|
started runs, not transport attempts or structural repairs.
|
||||||
|
|
||||||
## Shared Prompt Selection
|
## Shared Prompt Selection
|
||||||
|
|
||||||
|
The internal engine applies each namespace view to slashless caller selections
|
||||||
|
before they reach the runner. The runner therefore selects captured prompts and
|
||||||
|
profiles by the resulting exact runtime IDs. Already-qualified selections pass
|
||||||
|
through, while configured defaults and stored references are used as authored
|
||||||
|
and are not requalified.
|
||||||
|
|
||||||
Ordinary preparation selects the full prompt definition and construction-time
|
Ordinary preparation selects the full prompt definition and construction-time
|
||||||
hash. It retains its early request-ID check before direct-session
|
hash. It retains its early request-ID check before direct-session
|
||||||
normalization. Referenced message content and static template syntax were
|
normalization. Referenced message content and static template syntax were
|
||||||
captured when the engine was built.
|
captured when the engine was built.
|
||||||
|
Rendering uses those captured parses, cloning them per call to bind request
|
||||||
|
input helpers; it does not parse templates during execution.
|
||||||
|
|
||||||
`InspectPrompt` selects captured metadata through an optional repository
|
Resource inspection bypasses the runner: the internal engine selects a captured
|
||||||
capability, falling back to the full definition and existing hash resolution
|
configuration resource, and the facade detaches its values for the caller.
|
||||||
for internal repositories without it. It returns owned metadata without
|
Prompt resource inspection includes captured message text without rendering or
|
||||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
touching profile, artifact, schema, admission, or model collaborators.
|
||||||
admission, or model collaborators. The root
|
|
||||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
|
||||||
exact contract.
|
|
||||||
|
|
||||||
`InspectRequirements` uses the same metadata selection to resolve the
|
`InspectRequirements` uses the same metadata selection to resolve the
|
||||||
request-profile/default identity, then obtains the exact captured requirement
|
request-profile/default identity, then obtains the captured requirement
|
||||||
snapshot by the selected prompt ID and version. The analyzer receives only
|
snapshot by the selected prompt ID. The analyzer receives only
|
||||||
those identity fields rather than an execution definition. It analyzes the
|
those identity fields rather than an execution definition. It analyzes the
|
||||||
snapshot with the supplied caller variables and normalized direct-session
|
snapshot with the supplied caller variables and normalized direct-session
|
||||||
bypass; it does not resolve target readiness or enter the preparation
|
bypass; it does not resolve target readiness or enter the preparation
|
||||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
pipeline. An unavailable captured snapshot is a prompt-load failure. The root
|
||||||
source fallback. The root
|
|
||||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||||
operation's exact contract.
|
operation's exact contract.
|
||||||
|
|
||||||
## Shared Profile Selection
|
## Shared Profile Selection
|
||||||
|
|
||||||
The runner uses one profile-selection and target-resolution boundary for
|
The runner selects a request profile or a prompt default and uses its captured,
|
||||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
presence-aware target constructed from final profile and backend configuration.
|
||||||
request profile or a prompt default; inspection begins with its required
|
Request overrides apply afterward; preparation checks endpoint and model
|
||||||
explicit profile ID. Both use the captured, presence-aware target constructed
|
completeness only after those overrides. Resource profile inspection bypasses
|
||||||
from the final effective profile and backend configuration. Request overrides
|
this path and reports the profile before backend completion. Credential
|
||||||
apply afterward; preparation checks endpoint and model completeness only after
|
availability is checked when execution preparation requires it.
|
||||||
those overrides. Inspection can report a partial profile without requiring a
|
|
||||||
runnable target. Neither path checks credential availability or performs prompt,
|
## Smoke-Test Diagnostics
|
||||||
artifact, schema, rendering, admission, or model-client work. The root
|
|
||||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
The separate `SmokeTester` in `internal/usecase` selects from the same captured
|
||||||
exact contract.
|
configuration snapshot and backend/profile collaborators as the runner. The
|
||||||
|
internal engine applies namespace-view qualification to slashless caller IDs
|
||||||
|
before diagnostic selection; captured defaults and references remain exact.
|
||||||
|
Explicit profile selection takes precedence over prompt-default selection and a
|
||||||
|
backend diagnostic default; HTTP-only selection does not resolve a default
|
||||||
|
inference profile. Exact public selection rules belong to the root
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
Diagnostics do not call `Run`, `Prepare`, or the prompt preparation pipeline.
|
||||||
|
They do not render application messages, load application artifacts, use
|
||||||
|
application schemas, or inspect session templates. Private text and structured
|
||||||
|
cases and the structured case's schema remain internal to the diagnostic
|
||||||
|
operation; they are not entries in prompt/profile listings or inspection.
|
||||||
|
`InspectConfiguration` remains a separate, explicit operator-oriented view of
|
||||||
|
effective settings and source history. A diagnostic report may include selected
|
||||||
|
effective settings and identifiers, so consumers still apply their disclosure
|
||||||
|
policy.
|
||||||
|
|
||||||
|
The engine assembles the smoke tester with the runner's generation coordinator
|
||||||
|
and capacity manager. Inference holds the ordinary logical admission lease and
|
||||||
|
uses the same active permits, retries, timeout, and request evidence path as
|
||||||
|
generation. Reachability and authentication probes bypass generation capacity
|
||||||
|
and do not require a model. An engine with an injected `LLMClient` can run
|
||||||
|
inference diagnostics but has no built-in HTTP probe collaborator; HTTP-only
|
||||||
|
diagnostics report that limitation instead of falling back to another client.
|
||||||
|
The public GoDoc owns the caller-visible timeout, cancellation, report, and
|
||||||
|
transport semantics.
|
||||||
|
|
||||||
## Shared Preparation Pipeline
|
## Shared Preparation Pipeline
|
||||||
|
|
||||||
@@ -78,12 +131,12 @@ performs only the work needed to validate routing and admission:
|
|||||||
|
|
||||||
1. validate the required prompt selection and normalize any direct session ID;
|
1. validate the required prompt selection and normalize any direct session ID;
|
||||||
2. select the captured prompt definition and hash;
|
2. select the captured prompt definition and hash;
|
||||||
3. select the request profile or the prompt's default profile;
|
3. resolve and validate the effective output contract and select its applicable
|
||||||
4. retain the profile's selected backend identity, when present;
|
bound file without loading its schema;
|
||||||
5. apply explicit request overrides to its captured backend-completed target;
|
4. select the request profile or the prompt's default profile;
|
||||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
5. retain the profile's selected backend identity, when present;
|
||||||
7. resolve and validate the effective output contract without loading its
|
6. apply explicit request overrides to its captured backend-completed target;
|
||||||
schema; and
|
7. validate endpoint, model, numeric overrides, and credential requirements; and
|
||||||
8. retain the definition, source identities, effective settings, output
|
8. retain the definition, source identities, effective settings, output
|
||||||
contract, and preparation start time in invocation-local state.
|
contract, and preparation start time in invocation-local state.
|
||||||
|
|
||||||
@@ -94,9 +147,9 @@ profile, or backend:
|
|||||||
schema metadata from it when required;
|
schema metadata from it when required;
|
||||||
2. load and hash input artifacts;
|
2. load and hash input artifacts;
|
||||||
3. render messages and the prompt-defined session;
|
3. render messages and the prompt-defined session;
|
||||||
4. apply any direct session ID, then append already-normalized request messages
|
4. apply any direct session ID, then compose captured history, rendered
|
||||||
after the rendered definition messages;
|
definition messages, and already-normalized request messages in that order;
|
||||||
5. hash the effective rendered prompt; and
|
5. hash the effective rendered prompt and its ordered message content; and
|
||||||
6. construct the prepared value and preparation timing.
|
6. construct the prepared value and preparation timing.
|
||||||
|
|
||||||
`Prepare` runs both phases consecutively and never performs capacity admission.
|
`Prepare` runs both phases consecutively and never performs capacity admission.
|
||||||
@@ -105,11 +158,16 @@ one execution-precedence and error-ordering implementation while allowing a
|
|||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations. `Prepare` discards the plan after returning its public
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
and discards it when the operation ends. Prepared execution transfers the
|
and discards it when the operation ends. Prepared execution retains one
|
||||||
completed preparation's owned message, input-hash, and copied-schema values
|
immutable, redacted metadata snapshot shared by execution and internal details.
|
||||||
into its private payload, while deep-copying mutable provider extras and
|
The facade detaches mutable public details. Claim completion or discard clears
|
||||||
creating an independent details snapshot. Claim completion or discard clears
|
execution-only payload references without modifying the retained metadata.
|
||||||
the execution payload without removing the details snapshot.
|
|
||||||
|
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||||
|
path as `Prepare` and discards the resulting internal prepared value without
|
||||||
|
converting it into a public snapshot. It does not add a runner operation or
|
||||||
|
validator component; request conversion and public error mapping remain at the
|
||||||
|
facade boundary.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
@@ -132,8 +190,14 @@ sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
|
|||||||
encoding of that session, every role and content value, and cache-control
|
encoding of that session, every role and content value, and cache-control
|
||||||
presence and values; its hexadecimal value is opaque.
|
presence and values; its hexadecimal value is opaque.
|
||||||
|
|
||||||
|
The separate message-content hash is computed once after that same composition.
|
||||||
|
It streams a versioned prefix, message count, and length-framed role and text
|
||||||
|
bytes into SHA-256, excluding session and cache metadata. Fixed reference
|
||||||
|
vectors protect the selected v1 scheme across refactors; the public
|
||||||
|
[`PreparedRun` GoDoc](../../types.go) owns its compatibility contract.
|
||||||
|
|
||||||
The configuration snapshot is read-only after engine construction. Concurrent
|
The configuration snapshot is read-only after engine construction. Concurrent
|
||||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
`Prepare` and `Run` calls use detached captured targets and keep invocation
|
||||||
state local.
|
state local.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
@@ -156,6 +220,16 @@ mapping and after the runner's deferred admission release. Preparation and
|
|||||||
prepared execution follows the same terminal mapping after a successful claim;
|
prepared execution follows the same terminal mapping after a successful claim;
|
||||||
invalid or discarded handles do not acquire an observer.
|
invalid or discarded handles do not acquire an observer.
|
||||||
|
|
||||||
|
The facade establishes an optional operation scope for ordinary runs before
|
||||||
|
observation and preparation. A claimed prepared execution establishes its own
|
||||||
|
scope from the retained duration; preparation and idle time do not consume it.
|
||||||
|
The shared scope finalizer marks only a context failure caused by that scope's
|
||||||
|
own deadline, preserving its underlying operation and cause. The scope is
|
||||||
|
released before terminal delivery. At the shared prepared execution boundary,
|
||||||
|
the runner conditionally invokes the adapter for the actual prepared snapshot
|
||||||
|
while admission is held, with context checks before and after notification.
|
||||||
|
The facade adapter projects a detached public value only when selected.
|
||||||
|
|
||||||
Validation preparation and execution honor cancellation at every
|
Validation preparation and execution honor cancellation at every
|
||||||
Promptkit-controlled boundary and do not publish a partial plan or result.
|
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||||
Schema reads are bounded and context-checked between chunks; JSON decoding,
|
Schema reads are bounded and context-checked between chunks; JSON decoding,
|
||||||
@@ -250,4 +324,8 @@ Changes to orchestration should continue to use the existing package
|
|||||||
interfaces, keep request state local to an invocation, and preserve the shared
|
interfaces, keep request state local to an invocation, and preserve the shared
|
||||||
resolution and completion pipeline. Source, renderer, validator, or
|
resolution and completion pipeline. Source, renderer, validator, or
|
||||||
model-client contract changes belong first in their owning package and
|
model-client contract changes belong first in their owning package and
|
||||||
document.
|
document. The
|
||||||
|
[smoke selection tests](../../internal/usecase/smoke_selection_test.go),
|
||||||
|
[diagnostic orchestration tests](../../internal/usecase/smoke_tester_test.go),
|
||||||
|
and [private inference-case tests](../../internal/usecase/smoke_inference_test.go)
|
||||||
|
own diagnostic selection, report assembly, and inference findings.
|
||||||
|
|||||||
@@ -10,43 +10,54 @@ source semantics. Internal packages are not consumer APIs.
|
|||||||
|
|
||||||
## Typed Configuration Loading
|
## Typed Configuration Loading
|
||||||
|
|
||||||
`internal/configuration` is the single adapter from ConfigKit's final
|
The root facade composes Promptkit rules into a ConfigKit definition. An
|
||||||
effective configuration to execution-owned definitions. It performs one
|
application-owned load or the standalone constructor produces final settings
|
||||||
layered load, captures effective presence (including explicit null), records
|
at `/promptkit`; `internal/configuration` captures execution-owned resources
|
||||||
normalization and defaults through resolution, and validates all effective
|
from those settings and a native inspection view without another load.
|
||||||
backend, profile, and prompt values. Replaced lower-layer domain values are
|
`internal/settings` supplies the shared data declarations exposed through
|
||||||
not separately validated. It builds an immutable backend index and
|
root aliases. The adapter records normalization and validates effective values,
|
||||||
presence-aware profile targets, applying final backend defaults only after
|
then uses one metadata snapshot for presence and source origins. Runtime/domain
|
||||||
profile inheritance. It retains ConfigKit's result privately for explicit
|
packages do not depend on ConfigKit.
|
||||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
|
||||||
depend on ConfigKit.
|
|
||||||
|
|
||||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
Capture detaches known settings, builds indexed definitions and profile targets,
|
||||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
and binds source-relative resources from the effective field's source context.
|
||||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
It retains a frozen native focused view and copied diagnostics, not the mutable
|
||||||
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
|
application result. Configuration inspection delegates reporting, redaction,
|
||||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
history and comparison to ConfigKit; explicit queries and metadata retain
|
||||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
whole-application paths and evidence. Runtime never derives values from history.
|
||||||
both results; reports may still contain ordinary application data. See the
|
The [format reference](../formats.md#source-headers-collections-and-resource-ids)
|
||||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
owns the configuration contract and the [consumer guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
|
describes reporting scope.
|
||||||
|
|
||||||
After prompt materialization and effective-profile snapshot construction, the
|
After resource materialization and effective-profile snapshot construction, the
|
||||||
adapter captures lexically ordered, identity-only inventories from the final
|
adapter captures lexically ordered identity inventories for backends, profiles,
|
||||||
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
and prompts, per-kind resource namespace sets, and metadata for effective
|
||||||
of those inventories without consulting ConfigKit or copying definitions.
|
collection declarations. Listing methods return caller-owned copies from those
|
||||||
|
captured values without consulting ConfigKit or copying resource definitions.
|
||||||
|
|
||||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||||
files under an explicit root, provide resource context, and leave imports
|
files under an explicit root, provide resource context, and leave imports
|
||||||
disabled. Each returned slice belongs to one strict-composition group; file
|
disabled. Each returned slice belongs to one strict-composition group; file
|
||||||
order does not confer priority. The engine prepends selected catalog and
|
order does not confer priority. The standalone facade prepends a dedicated
|
||||||
Promptkit-owned definitions, then caller underlays and optional main. The
|
embedded OpenRouter underlay unless disabled, without modifying caller-owned slices.
|
||||||
|
Caller underlays are composed in their given
|
||||||
|
order, followed by the optional main input. The
|
||||||
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
||||||
real loader and final-value handoff.
|
real loader and final-value handoff.
|
||||||
|
|
||||||
|
The public `CollectionInput` helper mounts one caller-selected source at
|
||||||
|
`/promptkit/collections/<namespace>` and enables ConfigKit imports for that input. The
|
||||||
|
helper does not read the source; ConfigKit retains responsibility for loading,
|
||||||
|
composition, imports, and source-local resource context. Imported or inherited
|
||||||
|
`content_file` values continue resolving relative to the source that contributed
|
||||||
|
the effective field, not to a flattened runtime identity. Explicit
|
||||||
|
`InspectConfiguration` views expose the retained effective configuration and
|
||||||
|
provenance for operator review; runtime selection and inventories do not derive
|
||||||
|
from source history.
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
The configuration adapter indexes each prompt by exact ID and version,
|
The configuration adapter indexes each prompt by exact ID,
|
||||||
materializes file-backed message content using the exact effective field's
|
materializes file-backed message content using the exact effective field's
|
||||||
origin and resource context, and checks static message/session template syntax
|
origin and resource context, and checks static message/session template syntax
|
||||||
at construction. It stores captured content, compiled templates, declared
|
at construction. It stores captured content, compiled templates, declared
|
||||||
@@ -58,23 +69,32 @@ declaration; absent and null fields remain distinguishable from explicit
|
|||||||
strings, including empty strings. It closes content-file handles before
|
strings, including empty strings. It closes content-file handles before
|
||||||
returning and retains no source capability for public inspection.
|
returning and retains no source capability for public inspection.
|
||||||
|
|
||||||
Role and cache-control normalization is applied to execution-owned message
|
Prompt materialization shares one metadata snapshot across root and collection
|
||||||
copies. ConfigKit v0.7.0 retains effective input message metadata and its
|
resources during construction, avoiding a full metadata copy per collection.
|
||||||
resource origins; recording a sequence-wide normalization in the retained
|
Input metadata, message content and templates, and output contracts are captured
|
||||||
configuration would replace origins of unchanged content-file fields.
|
separately before the definition, inspection, and hash are assembled. Schema
|
||||||
|
root and source-relative file bindings capture the effective field context
|
||||||
|
without opening schema files; capabilities are excluded from the hash. Inline
|
||||||
|
schema values and file-reference strings participate in the definition hash.
|
||||||
|
|
||||||
The runner selects full definitions from this index for preparation and uses
|
Normalization and defaults are recorded at individual resolved field paths,
|
||||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
including message roles and cache controls. List indexes identify fields for
|
||||||
internal requirement snapshot lookup selects by exact prompt ID and version.
|
presence, validation, sensitivity, and resource context; full IDs select
|
||||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
captured runtime indexes. Unchanged content-file fields retain their origins.
|
||||||
returns declared metadata without rendering or resolving profile, input, or
|
Backend-to-profile and request-time completion remain separate runtime work.
|
||||||
schema resources, and its public shape and prompt hashes do not include the
|
|
||||||
captured presence state. Runtime rendering still checks request variables and
|
The runner selects full definitions from this index for preparation and retains
|
||||||
|
lightweight prompt metadata for `InspectRequirements`; the requirement snapshot
|
||||||
|
lookup uses the selected prompt ID alone. Separate configuration-owned resource
|
||||||
|
inspection projects the resolved record and captured messages without rendering
|
||||||
|
or resolving profile, input, or schema resources. These paths do not rescan
|
||||||
|
configuration or reopen message files. Presence metadata does not change the
|
||||||
|
opaque definition hash. Runtime rendering still checks request variables and
|
||||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||||
cover resource origins, context containment, capture, and selection.
|
cover resource origins, context containment, capture, and selection.
|
||||||
|
|
||||||
The configuration adapter shares exact ID/version selection across its
|
The configuration adapter shares ID-only selection across its internal prompt
|
||||||
internal prompt lookups. The execution snapshot lookup returns an owned prompt
|
lookups. The execution snapshot lookup returns an owned prompt
|
||||||
definition and cached hash; metadata inspection returns owned input metadata
|
definition and cached hash; metadata inspection returns owned input metadata
|
||||||
without copying execution templates. Profile selection likewise has an
|
without copying execution templates. Profile selection likewise has an
|
||||||
identity-only lookup for consumers that do not need the execution target,
|
identity-only lookup for consumers that do not need the execution target,
|
||||||
@@ -112,27 +132,24 @@ dependency-bearing range scopes make discovery incomplete. Uncalled definitions
|
|||||||
and bypassed session templates are not traversed. This internal walker does not
|
and bypassed session templates are not traversed. This internal walker does not
|
||||||
change rendering behavior.
|
change rendering behavior.
|
||||||
|
|
||||||
## Profiles And Built-Ins
|
## Profiles
|
||||||
|
|
||||||
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||||
assets as one ConfigKit underlay and PromptKit-owned profiles as the next.
|
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
||||||
It discovers YAML inputs and supplies passive filesystem context without
|
The configuration adapter validates final effective resource references and
|
||||||
parsing or pre-validating catalog definitions. The configuration loader
|
field values, then captures backend definitions and profile target snapshots
|
||||||
validates only the final effective backend and profile values, including
|
with backend defaults and outbound-control presence. Profiles without provider
|
||||||
consumer overlays of maintained bases or backends. There is no catalog-only
|
overrides share capture-owned immutable backend parameter maps; target lookups
|
||||||
preflight, reserved source namespace, or replay of backend registration.
|
still detach mutable values before runtime use. Null profile endpoint,
|
||||||
|
credential source, and whole provider map permit backend fallback after
|
||||||
ConfigKit resolves profile inheritance before the handoff. The configuration
|
composition, subject to the required-credential rule. A profile can remain
|
||||||
adapter validates effective references and field values, then builds target
|
partial until runtime overrides complete it. Resource inspection selects indexed
|
||||||
snapshots with backend defaults and presence/suppression state. A profile can
|
resolved backend and profile records without execution completion or prompt,
|
||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
artifact, schema, credential lookup, or provider work. Preparation applies runtime overrides and then checks target
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
|
||||||
provider work. Preparation applies runtime overrides and then checks target
|
|
||||||
completeness. Identity-only profile selection can identify a partial captured
|
completeness. Identity-only profile selection can identify a partial captured
|
||||||
profile without copying its target or requiring readiness. The
|
profile without copying its target or requiring readiness. The
|
||||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||||
target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
target interpretation.
|
||||||
cover selected released assets.
|
|
||||||
|
|
||||||
The internal engine delegates listing through a narrow identity-inventory
|
The internal engine delegates listing through a narrow identity-inventory
|
||||||
interface directly to the configuration snapshot. The execution runner has no
|
interface directly to the configuration snapshot. The execution runner has no
|
||||||
@@ -186,20 +203,24 @@ buffer-to-string copy. The
|
|||||||
|
|
||||||
## Schemas And Output Validation
|
## Schemas And Output Validation
|
||||||
|
|
||||||
`internal/validate` provides validators backed by an operating-system
|
`internal/validate` prepares a source-neutral input containing an output contract
|
||||||
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
and an optional bound `fs.FS` file. Invalid generated content is returned as a validation
|
||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error. Basic validation checks body emptiness on the byte slice directly
|
error. Basic validation checks body emptiness on the byte slice directly
|
||||||
instead of first making a response-sized string copy.
|
instead of first making a response-sized string copy.
|
||||||
|
|
||||||
Every preparation operation creates one operation-local validation plan. None,
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
basic, and JSON modes retain the effective output contract without source
|
basic, and JSON modes retain the effective output contract without source
|
||||||
access. JSON Schema mode loads the root document once, resolves and compiles
|
access. JSON Schema mode uses the captured inline document or loads the bound
|
||||||
each transitive reference, and retains the compiled validator. Schema compiler
|
root document once, compiles its declared
|
||||||
resources use canonical escaped file or private-scheme URLs; loaders decode
|
schema locations, and retains the compiled root validator. External resources
|
||||||
their paths once and enforce the configured source boundary. The
|
are rejected without file or network loading; in-document references remain
|
||||||
|
supported. Bound files use canonical escaped private-scheme URLs; inline values
|
||||||
|
use a stable private retrieval URL. The
|
||||||
provider-facing structured-output metadata uses the root document captured by
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
the same plan.
|
the same plan. Its detached schema document is not revalidated against the
|
||||||
|
runtime provider-value copier's traversal limits. Smoke inference uses a
|
||||||
|
separate JSON-only prepared plan before checking the prepared schema plan.
|
||||||
|
|
||||||
Schema preparation and execution remain synchronous. Promptkit checks
|
Schema preparation and execution remain synchronous. Promptkit checks
|
||||||
cancellation before and after source resolution, JSON decoding, compilation,
|
cancellation before and after source resolution, JSON decoding, compilation,
|
||||||
|
|||||||
@@ -26,97 +26,60 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
|||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
|
Selected module: `go.maximumdirect.net/configkit` at `v1.4.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
|
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
|
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/4a7ae664a6287d71cba20f59c7c5b37b6f05b0a3).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog v3
|
## ReleaseKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3`
|
Selected module: `go.maximumdirect.net/releasekit` at `v0.10.0` (maintainer tool).
|
||||||
at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
[LICENSE](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243).
|
||||||
|
|
||||||
## Promptkit Rakestrawhome Catalog v3
|
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3`
|
|
||||||
at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
|
||||||
|
|
||||||
## jsonschema
|
## jsonschema
|
||||||
|
|
||||||
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.2`.
|
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.3`.
|
||||||
|
|
||||||
SPDX-License-Identifier: Apache-2.0
|
SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||||
[LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/29cbed948d24a04700eb94436416b18a07953b71/LICENSE).
|
[LICENSE](https://github.com/santhosh-tekuri/jsonschema/blob/b0fc661f4939578bc429f408c18202573dbafcc4/LICENSE).
|
||||||
|
|
||||||
[Source](https://github.com/santhosh-tekuri/jsonschema/tree/29cbed948d24a04700eb94436416b18a07953b71).
|
[Source](https://github.com/santhosh-tekuri/jsonschema/tree/b0fc661f4939578bc429f408c18202573dbafcc4).
|
||||||
|
|
||||||
## Go Text
|
## Go Text
|
||||||
|
|
||||||
Selected module: `golang.org/x/text` at `v0.14.0` (transitive dependency).
|
Selected module: `golang.org/x/text` at `v0.44.0` (transitive dependency).
|
||||||
|
|
||||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
||||||
[LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) |
|
[LICENSE](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS).
|
[PATENTS](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/text/tree/6c97a165dd661335ff7bce6104a008558123c353).
|
[Source](https://github.com/golang/text/tree/6070ac32abcdd1670599eeb3ee4d513c8184a513).
|
||||||
|
|
||||||
## Go YAML v3
|
|
||||||
|
|
||||||
Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency).
|
|
||||||
|
|
||||||
Copyright (c) 2006-2010 Kirill Simonov
|
|
||||||
|
|
||||||
Copyright (c) 2006-2011 Kirill Simonov
|
|
||||||
|
|
||||||
Copyright (c) 2011-2019 Canonical Ltd
|
|
||||||
|
|
||||||
Copyright 2011-2016 Canonical Ltd.
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT AND Apache-2.0
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
|
||||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
|
||||||
[LICENSE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
|
|
||||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
|
||||||
|
|
||||||
[Source](https://github.com/go-yaml/yaml/tree/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
|
||||||
|
|
||||||
## Go YAML v4
|
## Go YAML v4
|
||||||
|
|
||||||
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0-rc.6` (transitive dependency).
|
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0`.
|
||||||
|
|
||||||
Copyright 2006-2010 Kirill Simonov
|
Copyright 2006-2010 Kirill Simonov
|
||||||
|
|
||||||
@@ -128,7 +91,34 @@ SPDX-License-Identifier: MIT AND Apache-2.0
|
|||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
License and notices: [MIT](licenses/MIT.txt) |
|
||||||
[Apache 2.0](licenses/Apache-2.0.txt) |
|
[Apache 2.0](licenses/Apache-2.0.txt) |
|
||||||
[LICENSE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/LICENSE) |
|
[LICENSE](https://github.com/yaml/go-yaml/blob/f2f9563503f11b7a60da0c532175dcc693ce8d5a/LICENSE) |
|
||||||
[NOTICE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/NOTICE).
|
[NOTICE](https://github.com/yaml/go-yaml/blob/f2f9563503f11b7a60da0c532175dcc693ce8d5a/NOTICE).
|
||||||
|
|
||||||
[Source](https://github.com/yaml/go-yaml/tree/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125).
|
[Source](https://github.com/yaml/go-yaml/tree/f2f9563503f11b7a60da0c532175dcc693ce8d5a).
|
||||||
|
|
||||||
|
## Goldmark
|
||||||
|
|
||||||
|
Selected module: `github.com/yuin/goldmark/v2` at `v2.1.16` (maintainer-tool dependency).
|
||||||
|
|
||||||
|
Copyright (c) 2019 Yusuke Inuzuka
|
||||||
|
|
||||||
|
SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
License and notices: [MIT](licenses/MIT.txt) |
|
||||||
|
[LICENSE](https://github.com/yuin/goldmark/blob/c3f939122ba4dd6f80e05043ad195b82e77fcb41/LICENSE).
|
||||||
|
|
||||||
|
[Source](https://github.com/yuin/goldmark/tree/c3f939122ba4dd6f80e05043ad195b82e77fcb41).
|
||||||
|
|
||||||
|
## Go Modules
|
||||||
|
|
||||||
|
Selected module: `golang.org/x/mod` at `v0.43.0` (maintainer-tool dependency).
|
||||||
|
|
||||||
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
|
||||||
|
[LICENSE](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde/LICENSE) |
|
||||||
|
[PATENTS](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde/PATENTS).
|
||||||
|
|
||||||
|
[Source](https://go.googlesource.com/mod/+/d4f320b9266480df5ed3d791e0c68e64c58c5dde).
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.3
|
**Base Policy Version:** 1.3
|
||||||
|
|
||||||
**Last Updated**: 2026-09-27
|
**Last Updated**: 2026-11-21
|
||||||
|
|
||||||
**Upstream Source:** [architecture.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
**Upstream Source:** [architecture.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
||||||
|
|
||||||
@@ -51,9 +51,10 @@ The [internal component overview](../internal/overview.md) owns the inventory
|
|||||||
of implemented packages and their concrete responsibilities. This policy owns
|
of implemented packages and their concrete responsibilities. This policy owns
|
||||||
the boundaries and dependency rules those components must preserve.
|
the boundaries and dependency rules those components must preserve.
|
||||||
|
|
||||||
The internal engine composes selected catalog and caller ConfigKit inputs once,
|
The facade either registers Promptkit rules in the application's ConfigKit load
|
||||||
then assembles the captured definitions, capacity manager, renderer, validator,
|
or performs one standalone load, then passes captured configuration to the
|
||||||
outbound client, and use-case runner. The configuration adapter interprets and
|
internal engine. The internal engine assembles the captured definitions,
|
||||||
|
capacity manager, renderer, validator, outbound client, and use-case runner. The configuration adapter interprets and
|
||||||
validates only the final effective values, including presence, null, and
|
validates only the final effective values, including presence, null, and
|
||||||
resource context. It retains source history for explicit operator inspection,
|
resource context. It retains source history for explicit operator inspection,
|
||||||
not for validation or a second resolution pass. The root facade translates
|
not for validation or a second resolution pass. The root facade translates
|
||||||
@@ -97,7 +98,8 @@ on consumers or on Scriptorium.
|
|||||||
`internal/domain` owns source-neutral invariants for values shared across
|
`internal/domain` owns source-neutral invariants for values shared across
|
||||||
multiple input and execution boundaries, including execution-setting bounds,
|
multiple input and execution boundaries, including execution-setting bounds,
|
||||||
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
||||||
legality. Callers retain source parsing, required-field rules, other
|
legality, plus shared resource-identity syntax and qualification. Callers retain
|
||||||
|
source parsing, required-field rules, other
|
||||||
source-specific normalization, defaulting, error classification, and policy
|
source-specific normalization, defaulting, error classification, and policy
|
||||||
specific to their own boundary.
|
specific to their own boundary.
|
||||||
|
|
||||||
@@ -127,7 +129,8 @@ consumer contract requires a public package.
|
|||||||
Each package must have one cohesive responsibility and a clear dependency
|
Each package must have one cohesive responsibility and a clear dependency
|
||||||
direction. Internal packages must not expose their types merely to simplify
|
direction. Internal packages must not expose their types merely to simplify
|
||||||
wiring, and the public facade must not leak internal representations through
|
wiring, and the public facade must not leak internal representations through
|
||||||
exported signatures. Production packages under `internal/` must not import the
|
exported signatures, except for the deliberately public configuration data model
|
||||||
|
specified below. Production packages under `internal/` must not import the
|
||||||
module root; the root facade may depend on internal implementation packages.
|
module root; the root facade may depend on internal implementation packages.
|
||||||
New public packages require a durable consumer need that cannot be served
|
New public packages require a durable consumer need that cannot be served
|
||||||
cleanly by the root facade.
|
cleanly by the root facade.
|
||||||
@@ -135,12 +138,34 @@ cleanly by the root facade.
|
|||||||
The [internal component overview](../internal/overview.md) must be updated as
|
The [internal component overview](../internal/overview.md) must be updated as
|
||||||
packages are implemented or their responsibilities change.
|
packages are implemented or their responsibilities change.
|
||||||
|
|
||||||
|
### Shared Configuration Data Exception
|
||||||
|
|
||||||
|
[ADR 0002](../adr/0002-shared-configuration-data-model.md) permits one canonical
|
||||||
|
set of public configuration settings and their nested data types in a small,
|
||||||
|
data-only internal package, exposed through aliases in the root `promptkit`
|
||||||
|
package. The facade and internal configuration adapter may use those same
|
||||||
|
declarations directly. Their fields, tags, and nested types are intentional
|
||||||
|
public contracts and receive the same API review as root-defined types.
|
||||||
|
|
||||||
|
The shared package owns data declarations, not loading, normalization,
|
||||||
|
validation, or execution behavior. This exception does not authorize aliases
|
||||||
|
for arbitrary internal implementation types or merging configuration settings
|
||||||
|
with source-construction helpers, execution requests, runtime objects, or
|
||||||
|
inspection reports that have different meanings. Internal packages still must
|
||||||
|
not import the root facade. Runtime capture still owns the values it retains;
|
||||||
|
shared declarations do not replace necessary ownership copies.
|
||||||
|
|
||||||
|
All other facade, dependency, and ownership rules remain in force. Concrete
|
||||||
|
package placement and implementation status belong in the active plan and,
|
||||||
|
once implemented, the internal component inventory.
|
||||||
|
|
||||||
## Exported API Discipline
|
## Exported API Discipline
|
||||||
|
|
||||||
Export the smallest contract required by real consumers. Exported declarations
|
Export the smallest contract required by real consumers. Exported declarations
|
||||||
must have accurate GoDoc, stable semantics, and tests proportionate to their
|
must have accurate GoDoc, stable semantics, and tests proportionate to their
|
||||||
compatibility risk. Avoid speculative extension points, aliases for internal
|
compatibility risk. Avoid speculative extension points, aliases for internal
|
||||||
types, and public constructors that expose assembly details.
|
types outside the shared configuration data exception, and public constructors
|
||||||
|
that expose assembly details.
|
||||||
|
|
||||||
Once an exported API exists, its Go declaration and GoDoc own its exact public
|
Once an exported API exists, its Go declaration and GoDoc own its exact public
|
||||||
contract. Architecture documentation owns boundary rules, not a duplicate API
|
contract. Architecture documentation owns boundary rules, not a duplicate API
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ secret values.
|
|||||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||||
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
||||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||||
|
|
||||||
Create supporting documents for existing interfaces and responsibilities when
|
Create supporting documents for existing interfaces and responsibilities when
|
||||||
substantive content is available.
|
substantive content is available.
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.8
|
**Base Policy Version:** 1.8
|
||||||
|
|
||||||
**Last Updated**: 2026-09-27
|
**Last Updated**: 2026-11-02
|
||||||
|
|
||||||
**Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
**Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
||||||
|
|
||||||
@@ -221,7 +221,8 @@ Promptkit retains its LGPL-3.0-or-later library grant in
|
|||||||
[COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate
|
[COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate
|
||||||
scope identified there and in [attribution](../oss/attribution.md).
|
scope identified there and in [attribution](../oss/attribution.md).
|
||||||
|
|
||||||
Promptkit publishes source-only Go module tags on Gitea, with no binaries or
|
Promptkit publishes source-only Go module tags and Gitea release pages, with no
|
||||||
separate hosted download page. The [release procedure](../release.md) owns
|
binaries. Each new hosted page reuses the complete checked-in release note,
|
||||||
publication and concrete footer URLs. The hosted-page guidance applies if such
|
including its licensing footer. The [release procedure](../release.md) owns
|
||||||
a channel is deliberately introduced; it does not require creating one.
|
source publication, hosted delivery, and concrete footer URLs. Previously
|
||||||
|
published notes and tags retain their original records.
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.6
|
**Base Policy Version:** 1.6
|
||||||
|
|
||||||
**Last Updated**: 2026-10-08
|
**Last Updated**: 2026-11-02
|
||||||
|
|
||||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
||||||
|
|
||||||
@@ -241,9 +241,13 @@ navigation on 2026-09-26. Apply them prospectively; previously published notes
|
|||||||
and tags retain their original records.
|
and tags retain their original records.
|
||||||
|
|
||||||
Promptkit publishes source-only Go modules from main to origin on Gitea, using
|
Promptkit publishes source-only Go modules from main to origin on Gitea, using
|
||||||
annotated stable SemVer tags. It has no binaries, hosted download pages, or
|
annotated stable SemVer tags. The source commit is pushed through main before
|
||||||
asynchronous deliverables. The source commit is pushed through main before the
|
the tag. After source publication, maintainers use ReleaseKit locally to deliver
|
||||||
tag. The [release procedure](../release.md) owns commands and recovery.
|
a Gitea release page containing the complete checked-in note, including its
|
||||||
|
licensing footer. There are no binaries or other deliverables. Source publication
|
||||||
|
and hosted delivery have separate outcomes; delivery credentials and service
|
||||||
|
availability do not gate source-release completion. The
|
||||||
|
[release procedure](../release.md) owns commands and recovery.
|
||||||
|
|
||||||
Release preparation uses the latest Go installation available in the development
|
Release preparation uses the latest Go installation available in the development
|
||||||
environment to build the maintainer release executable and validate the
|
environment to build the maintainer release executable and validate the
|
||||||
@@ -267,9 +271,11 @@ The [publication command](../../scripts/release.sh) delegates to the released
|
|||||||
ReleaseKit tool selected in `go.mod`. The committed
|
ReleaseKit tool selected in `go.mod`. The committed
|
||||||
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
|
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
|
||||||
committed Go formatting, local Markdown links, workspace and filesystem
|
committed Go formatting, local Markdown links, workspace and filesystem
|
||||||
replacement inspection, and the offline semantic checks for the three
|
replacement inspection. The Go suite includes the offline semantic checks for
|
||||||
maintained consumer examples. These are the selected checks for the final
|
the three maintained consumer examples. It also rejects the configured local
|
||||||
candidate; a separate ordinary test run or standalone build is not an
|
delivery token path if committed; this narrowly scoped check is not a general credential
|
||||||
|
scan. These are the selected checks for the final candidate; a separate
|
||||||
|
ordinary test run or standalone build is not an
|
||||||
additional release gate. The upstream tool owns candidate, remote, and tag
|
additional release gate. The upstream tool owns candidate, remote, and tag
|
||||||
publication checks. It returns native status 0 for confirmed publication, 1
|
publication checks. It returns native status 0 for confirmed publication, 1
|
||||||
for a known validation or publication failure, 2 for invalid input or profile,
|
for a known validation or publication failure, 2 for invalid input or profile,
|
||||||
@@ -279,28 +285,8 @@ captured publication endpoint before retrying; the repository-specific
|
|||||||
instructions. Do not wait for CI or Go module availability. Licensing footer
|
instructions. Do not wait for CI or Go module availability. Licensing footer
|
||||||
URLs are preparation work, not a network gate.
|
URLs are preparation work, not a network gate.
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
Hosted delivery is a separate post-publication step for new releases. It reuses
|
||||||
|
the immutable tag and note without rerunning candidate validation or waiting
|
||||||
External catalog profile data may be added, removed, renamed, or modified
|
for CI. Retry failed delivery against that same version after resolving any
|
||||||
without deprecation or advance notice. This exception is limited to profile
|
uncertain hosted write; preserve published tags and hosted content. Report
|
||||||
data imported from the maintained catalog modules. It does not weaken ordinary
|
delivery status separately from the completed source release.
|
||||||
compatibility for Promptkit's public Go API, configuration and format semantics,
|
|
||||||
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or
|
|
||||||
consumer-owned definitions.
|
|
||||||
|
|
||||||
Within the catalog versions selected and tested by a Promptkit release, and
|
|
||||||
unless a consumer overrides the name or its base, Promptkit keeps the literal
|
|
||||||
profile IDs `promptkit-openrouter-standard` and
|
|
||||||
`promptkit-openrouter-auto` available. The names do not promise a permanent
|
|
||||||
model, backend, routing behavior, settings, cost, output, or upstream
|
|
||||||
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
|
|
||||||
owns their current definitions.
|
|
||||||
|
|
||||||
`v0.10.0` is the initial minor release for this compatibility transition.
|
|
||||||
Afterward, catalog-only updates and adjustments to the current selections may
|
|
||||||
ship in a patch release when all other compatibility obligations remain intact.
|
|
||||||
External profile removals can require consumer action; they are not automatic
|
|
||||||
migrations.
|
|
||||||
|
|
||||||
Catalog adoption and release validation remain local and offline. They do not
|
|
||||||
probe provider endpoints, request model lists, or make inference calls.
|
|
||||||
|
|||||||
@@ -60,10 +60,10 @@ documented validation model.
|
|||||||
|
|
||||||
The selected ReleaseKit profile runs the race-enabled package suite and full
|
The selected ReleaseKit profile runs the race-enabled package suite and full
|
||||||
vet, checks committed Go formatting, local Markdown links, and module
|
vet, checks committed Go formatting, local Markdown links, and module
|
||||||
distribution hygiene, and executes semantic assertions for the three
|
distribution hygiene. The Go suite includes semantic assertions for the three
|
||||||
maintained offline consumer examples. ReleaseKit owns this shared validation
|
maintained offline consumer examples. ReleaseKit owns this shared validation
|
||||||
machinery and publication workflow. The test run builds the packages; the
|
machinery and publication workflow. The test run builds the packages and
|
||||||
examples also exercise ordinary compilation. Separate ordinary tests and a
|
exercises the examples. Separate ordinary tests and a
|
||||||
standalone build are not additional release gates. Ordinary tests remain useful
|
standalone build are not additional release gates. Ordinary tests remain useful
|
||||||
during development. The preparation example protects assembled preparation and
|
during development. The preparation example protects assembled preparation and
|
||||||
inspection behavior. The execution example separately protects assembled
|
inspection behavior. The execution example separately protects assembled
|
||||||
|
|||||||
193
docs/release.md
193
docs/release.md
@@ -1,26 +1,26 @@
|
|||||||
# Release Procedure
|
# Release Procedure
|
||||||
|
|
||||||
This runbook implements the [release policy](policy/release.md). Promptkit
|
This runbook implements the [release policy](policy/release.md). Promptkit
|
||||||
publishes source-only Go module tags from `main` to `origin`. A successful push
|
publishes source-only Go module tags from `main` to `origin`, followed by a
|
||||||
of the selected annotated tag, after its source commit is available on `main`,
|
page-only Gitea release. Successful publication of the intended tag, with its
|
||||||
completes the release. Do not wait for CI or Go module availability.
|
source commit available on `main`, completes the source release. Report hosted
|
||||||
|
delivery separately. Do not wait for CI or Go module availability.
|
||||||
|
|
||||||
## Prepare The Candidate
|
## Prepare The Candidate
|
||||||
|
|
||||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
Required tools are Git, Go with race-detector support, and POSIX shell. Work
|
||||||
3. Work from the repository root with push access to `origin`.
|
from the repository root with push access to `origin`.
|
||||||
|
|
||||||
Use the latest Go installation available locally for the release tool, the
|
Use the latest Go installation available locally for the release tool and its
|
||||||
candidate validation, and the maintained examples. Make it the active `go`
|
child commands. Put it first on `PATH` and keep it selected throughout:
|
||||||
command and record its version:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
export GOTOOLCHAIN=local
|
export GOTOOLCHAIN=local
|
||||||
go version
|
go version
|
||||||
```
|
```
|
||||||
|
|
||||||
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
For a minor release, normally advance the consumer minimum to the initial
|
||||||
release of that Go series before committing the candidate:
|
stable patch of that Go series:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
release_go_version=$(go env GOVERSION)
|
release_go_version=$(go env GOVERSION)
|
||||||
@@ -29,110 +29,131 @@ release_go_series=${release_go_series%.*}
|
|||||||
go mod edit "-go=${release_go_series}.0"
|
go mod edit "-go=${release_go_series}.0"
|
||||||
```
|
```
|
||||||
|
|
||||||
For example, Go 1.26.8 normally results in `go 1.26.0`. Keep a higher patch
|
Retain a higher patch minimum when required by a dependency or identified fix.
|
||||||
minimum if a dependency or identified correctness fix requires it. For a patch
|
For a patch release, preserve the existing minimum. Explain minimum increases
|
||||||
release, leave the consumer minimum unchanged; using a newer validation
|
under the [versioning policy](policy/release.md#versioning-and-compatibility).
|
||||||
toolchain does not itself raise that minimum. If the candidate needs a higher
|
Prepare dependency caches before committing the candidate:
|
||||||
minimum, prepare a minor release instead. See the
|
|
||||||
[consumer-minimum policy](policy/release.md#consumer-minimum-and-release-toolchain).
|
|
||||||
|
|
||||||
Prepare module caches before validation; this may update `go.sum` and belongs
|
|
||||||
before the candidate commit:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
GOTOOLCHAIN=local go mod download
|
go mod download
|
||||||
```
|
```
|
||||||
|
|
||||||
Choose an unused stable version and create `docs/releases/<version>.md` with a
|
Choose an unused stable version and create `docs/releases/<version>.md` with a
|
||||||
title containing `Promptkit` and that exact version, a short summary, and any
|
title containing `Promptkit` and that exact version, a short summary, and any
|
||||||
relevant compatibility impact or upgrade instructions. No fixed section
|
compatibility impact or upgrade instructions. End the note with:
|
||||||
headings are required. Describe a raised Go minimum and its compatibility
|
|
||||||
impact under the [versioning policy](policy/release.md#versioning-and-compatibility).
|
|
||||||
End each new note with the licensing footer below, using the selected version:
|
|
||||||
|
|
||||||
```markdown
|
```markdown
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
|
||||||
```
|
```
|
||||||
|
|
||||||
Check the footer paths against the candidate; the future tag need not resolve
|
Substitute the selected version and check the paths against the candidate;
|
||||||
remotely during preparation. Follow the [OSS policy](policy/oss.md) for these
|
future tag URLs need not resolve during preparation. Follow the
|
||||||
links. Promptkit publishes no binaries, hosted download page, or other
|
[OSS policy](policy/oss.md). Historical notes and tags remain unchanged.
|
||||||
asynchronous deliverables.
|
|
||||||
|
|
||||||
Retire completed roadmaps while preserving deferred work and repairing incoming
|
Supply delivery credentials through `RELEASEKIT_GITEA_TOKEN` or the ignored
|
||||||
links. Review the intended changes through the ordinary development workflow,
|
local file `.releasekit/gitea-token`, with owner-only file permissions. A
|
||||||
commit the candidate on `main`, and push `main` to `origin` before publication.
|
nonblank environment value takes precedence. Source publication neither reads
|
||||||
Preparation may update module metadata; the release bootstrap and validation
|
nor requires this token. The [profile](../releasekit.json) selects the token
|
||||||
preserve the committed candidate.
|
path for committed-file protection; keep credentials out of commits and logs.
|
||||||
|
|
||||||
|
Retire completed roadmaps, preserve unfinished work, and repair affected links.
|
||||||
|
Review and commit the candidate on `main`, including any module updates, then
|
||||||
|
push `main` to `origin` before publishing its tag.
|
||||||
|
|
||||||
## Validate And Publish
|
## Validate And Publish
|
||||||
|
|
||||||
For an optional local diagnostic of a committed candidate, run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
./scripts/releasekit.sh check vMAJOR.MINOR.PATCH
|
|
||||||
```
|
|
||||||
|
|
||||||
This is not a required step before publication. The upstream
|
|
||||||
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
|
|
||||||
describes its requirements.
|
|
||||||
|
|
||||||
Run the single publication command with the chosen version:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||||
```
|
```
|
||||||
|
|
||||||
The familiar wrapper delegates to the native ReleaseKit executable selected in
|
The launcher uses `go tool` to build or reuse the module-selected ReleaseKit
|
||||||
`go.mod`. The committed [profile](../releasekit.json) selects the race-enabled
|
executable in Go's cache, using the selected local toolchain. ReleaseKit runs
|
||||||
Go suite, full vet, committed Go formatting, local Markdown links, workspace
|
the [profile's checks](../releasekit.json) once, verifies publication state, and
|
||||||
and filesystem replacement inspection, and semantic checks for the three
|
publishes only the intended tag. Preserve its native exit status and diagnostic.
|
||||||
maintained offline consumer examples. ReleaseKit also verifies the committed
|
Successful publication needs no separate remote read-back or CI wait.
|
||||||
candidate and publication state, then publishes only the selected tag. Its
|
|
||||||
checks run once for this candidate; do not repeat them as a separate sign-off.
|
|
||||||
|
|
||||||
The native command returns status 0 for confirmed publication, 1 for a known
|
For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`.
|
||||||
validation or publication failure, 2 for invalid input or profile, and 3 when
|
It is not a required step before publication. The upstream
|
||||||
the push outcome remains unknown. Successful publication needs no remote
|
[candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/check.md)
|
||||||
read-back, CI wait, or installation check. Provider probes and inference calls
|
and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/cli.md)
|
||||||
are not part of validation.
|
own shared checks and exit-status definitions.
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
## Deliver The Hosted Release
|
||||||
|
|
||||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns the
|
After source publication, keep a clean checkout at the published tag's commit:
|
||||||
catalog compatibility boundary. This heading preserves older incoming links.
|
|
||||||
|
```sh
|
||||||
|
./scripts/releasekit.sh deliver vMAJOR.MINOR.PATCH
|
||||||
|
```
|
||||||
|
|
||||||
|
The profile selects page-only Gitea delivery for this repository. The hosted
|
||||||
|
body is the complete checked-in note, including its licensing footer. There are
|
||||||
|
no binary assets, checksum manifests, candidate revalidation, or CI waits.
|
||||||
|
ReleaseKit checks delivery credentials during delivery, not as a publication
|
||||||
|
gate. Report the hosted URL on success; delivery failure leaves the source
|
||||||
|
release complete.
|
||||||
|
|
||||||
|
For a later retry, use a clean checkout of the original tag with its recorded
|
||||||
|
module files and supply the token there. In a separate checkout, for example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
(
|
||||||
|
set -e
|
||||||
|
release_version=vMAJOR.MINOR.PATCH
|
||||||
|
release_tag_ref="refs/tags/$release_version"
|
||||||
|
git fetch origin "$release_tag_ref:$release_tag_ref"
|
||||||
|
release_tag_commit=$(git rev-parse --verify "$release_tag_ref^{commit}")
|
||||||
|
git switch --detach "$release_tag_commit"
|
||||||
|
release_worktree_status=$(git status --porcelain --untracked-files=all)
|
||||||
|
if [ -n "$release_worktree_status" ]; then
|
||||||
|
printf '%s\n' 'Delivery retry requires a clean tagged checkout.' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
./scripts/releasekit.sh deliver "$release_version"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Resolve uncertain hosted writes before retrying the same immutable version.
|
||||||
|
Matching content is accepted; conflicts are not overwritten. Do not rerun
|
||||||
|
`publish` or move tags to repair delivery. Follow the upstream
|
||||||
|
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/delivery.md)
|
||||||
|
for result interpretation and recovery. These commands apply to tags containing
|
||||||
|
this delivery configuration; older releases retain their original workflows.
|
||||||
|
|
||||||
## Failure Recovery
|
## Failure Recovery
|
||||||
|
|
||||||
For status 1, use the diagnostic to distinguish a validation failure from a
|
For source-publication failures, use ReleaseKit's status and diagnostic to
|
||||||
known publication conflict. Correct and commit candidate defects, push `main`,
|
distinguish a candidate defect from a publication conflict. Correct and commit
|
||||||
and rerun publication with an unused version when appropriate. Status 2 means
|
candidate defects, push `main`, and rerun publication as appropriate. A failure
|
||||||
the invocation or profile must be corrected.
|
after local tag creation may leave a ref even when its object ID is unavailable.
|
||||||
|
|
||||||
For status 3, do not retry or remove the local tag until the remote outcome is
|
Status 3 means the push outcome remains unknown. Do not retry or remove the local
|
||||||
resolved. ReleaseKit reports the attempted version, tag object, and recovery
|
tag until the authoritative outcome is resolved. ReleaseKit attempts to save
|
||||||
state, including the captured publication endpoint when available. Use that
|
private publication evidence and reports a record path or a separate recording
|
||||||
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
warning. A recording failure does not change the publication outcome.
|
||||||
upstream
|
|
||||||
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
|
Use the recovery command reported for that attempt, or discover local records
|
||||||
|
and explicitly select the matching attempt:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
./scripts/releasekit.sh publication-attempts
|
||||||
releasekit_version='vX.Y.Z'
|
./scripts/releasekit.sh reconcile --record /absolute/path/to/selected-record.json
|
||||||
test "$(git ls-remote --get-url -- "$releasekit_endpoint")" = "$releasekit_endpoint" &&
|
|
||||||
git ls-remote --refs -- "$releasekit_endpoint" "refs/tags/$releasekit_version"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Compare the returned object ID with the reported tag object. An exact match
|
Discovery is local; reconciliation observes the original destination without
|
||||||
confirms publication. A missing ref confirms absence only when the endpoint
|
rerunning validation, changing tags, or delivering hosted content. Neither is
|
||||||
identity check and query both succeed. If either check fails, the outcome
|
a routine gate after successful publication. Follow the upstream
|
||||||
remains unknown; correct the transport configuration and resolve it before
|
[recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/release.md#resolve-publication-failures)
|
||||||
retrying. Only after authoritative absence is confirmed may an unpublished
|
for outcomes, incomplete evidence, and older attempts without records.
|
||||||
local tag be removed before correcting the candidate or retrying. Never move,
|
|
||||||
overwrite, or delete a published tag.
|
|
||||||
|
|
||||||
## Policy Changes
|
Records are stored under Git's common directory by default and can contain
|
||||||
|
credential-bearing endpoints. Keep them private, out of source control and
|
||||||
|
shared logs; retain them while needed for recovery and remove them when that
|
||||||
|
value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
|
||||||
|
for storage and retention details.
|
||||||
|
|
||||||
Update the policy and this procedure together when changing release governance.
|
Current absence does not prove a version was never published. Remove a retained
|
||||||
Consumers own their dependency-availability and release checks; their readiness
|
local tag only after establishing absence and confirming it was not already
|
||||||
does not delay completion of a Promptkit release.
|
published or used by another release channel. Never reuse a known published
|
||||||
|
version or replace a published tag or hosted page. Corrections require a new
|
||||||
|
appropriate version.
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ the existing Go API. This refactor requires no consumer code changes.
|
|||||||
|
|
||||||
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
||||||
`v2.20260922.0`; their exact inventories are linked from the
|
`v2.20260922.0`; their exact inventories are linked from the
|
||||||
[format reference](../formats.md#external-profile-inventories).
|
[migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
|
||||||
|
|
||||||
The maintained names provide continuity only within the catalog versions
|
The maintained names provide continuity only within the catalog versions
|
||||||
selected and tested by a Promptkit release and when consumers do not override
|
selected and tested by a Promptkit release and when consumers do not override
|
||||||
@@ -37,8 +37,8 @@ an available profile or supply their own complete definition; the new
|
|||||||
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
||||||
aliases and need not preserve the previous settings.
|
aliases and need not preserve the previous settings.
|
||||||
|
|
||||||
See the [release policy](../release.md#catalog-profile-compatibility) for the
|
See the [release policy](../policy/release.md#versioning-and-compatibility) for the
|
||||||
compatibility boundary, the [format reference](../formats.md#built-in-profile-catalog)
|
compatibility boundary, the [migration identity mapping](../consumers/configuration-migration.md#identity-mapping)
|
||||||
for current Promptkit-owned definitions and selected external inventories, and
|
for current resource mappings, and
|
||||||
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
||||||
for configuration choices.
|
for configuration choices.
|
||||||
|
|||||||
@@ -72,5 +72,5 @@ for a structured proposal-and-assessment workflow.
|
|||||||
|
|
||||||
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
||||||
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
||||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility)
|
[release compatibility policy](../policy/release.md#versioning-and-compatibility)
|
||||||
and [profile inventories](../formats.md#external-profile-inventories).
|
and [migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ No public API, runtime behavior, or profile changes are introduced.
|
|||||||
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
||||||
See the [project grant](../../COPYRIGHT),
|
See the [project grant](../../COPYRIGHT),
|
||||||
[component attribution](../oss/attribution.md), and
|
[component attribution](../oss/attribution.md), and
|
||||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility).
|
[release compatibility policy](../policy/release.md#versioning-and-compatibility).
|
||||||
|
|
||||||
## Upgrade
|
## Upgrade
|
||||||
|
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ for API and source-selection changes.
|
|||||||
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
||||||
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
||||||
assets directly must adopt the keyed format. The maintained Promptkit profile
|
assets directly must adopt the keyed format. The maintained Promptkit profile
|
||||||
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
|
IDs remain available under the [release compatibility policy](../policy/release.md#versioning-and-compatibility),
|
||||||
which does not promise fixed profile data.
|
which does not promise fixed profile data.
|
||||||
|
|
||||||
## Upgrade
|
## Upgrade
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ increases to 1.26.0.
|
|||||||
engine's captured configuration, are lexically ordered and caller-owned, and
|
engine's captured configuration, are lexically ordered and caller-owned, and
|
||||||
include consumer definitions and maintained profiles that remain effective.
|
include consumer definitions and maintained profiles that remain effective.
|
||||||
Listing performs no provider or execution work. See the
|
Listing performs no provider or execution work. See the
|
||||||
[discovery guide](../consumers/pkg-promptkit.md#discover-available-profiles-and-prompts)
|
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections)
|
||||||
and [GoDoc](../../listing.go).
|
and [GoDoc](../../listing.go).
|
||||||
|
|
||||||
## Efficiency And Tooling
|
## Efficiency And Tooling
|
||||||
|
|||||||
34
docs/releases/v0.15.0.md
Normal file
34
docs/releases/v0.15.0.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Promptkit v0.15.0
|
||||||
|
|
||||||
|
This release adds error-only validation of a concrete prompt request without
|
||||||
|
calling a model. The minimum supported Go version remains 1.26.0.
|
||||||
|
|
||||||
|
## Validate A Concrete Request
|
||||||
|
|
||||||
|
`Engine.ValidateRequest` accepts an ordinary `RunRequest` and returns nil when
|
||||||
|
it passes preparation, or the existing preparation error. Consumers can use it
|
||||||
|
for fail-fast checks, including downstream CLI validation commands, without
|
||||||
|
receiving a prepared snapshot.
|
||||||
|
|
||||||
|
The method shares `Prepare`'s request conversion and preparation path. Unlike
|
||||||
|
requirements inspection, it checks the actual supplied values and bindings.
|
||||||
|
Artifact and schema resource I/O still occurs; success does not certify provider
|
||||||
|
acceptance or later execution success. A subsequent run prepares independently.
|
||||||
|
Validation is optional, not a required step before preparation or execution.
|
||||||
|
|
||||||
|
See the [request validation guide](../consumers/pkg-promptkit.md#validate-a-concrete-request)
|
||||||
|
and [GoDoc](../../engine.go) for usage and the exact contract. Consumers that
|
||||||
|
need execution tied to a checked snapshot can use the existing
|
||||||
|
[prepared execution workflow](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No application-code, configuration, or Go-toolchain migration is required from
|
||||||
|
v0.14.0. This is an optional API addition; existing inspection, preparation, and
|
||||||
|
execution workflows remain available.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.15.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/docs/oss/attribution.md)
|
||||||
86
docs/releases/v0.16.0.md
Normal file
86
docs/releases/v0.16.0.md
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
# Promptkit v0.16.0
|
||||||
|
|
||||||
|
This release adds explicit smoke diagnostics, configuration collections,
|
||||||
|
namespace-scoped engine views, and expanded resource discovery. It also makes
|
||||||
|
intentional breaking changes to configuration ownership and prompt selection.
|
||||||
|
The minimum supported Go version remains 1.26.0.
|
||||||
|
|
||||||
|
## Configuration Collections And Discovery
|
||||||
|
|
||||||
|
Applications can compose caller-selected collections of backends, profiles,
|
||||||
|
and prompts. Collection resources have qualified identities such as
|
||||||
|
`support/answer`; root resources retain unqualified identities. `CollectionInput`
|
||||||
|
provides a convenience for mounting a selected ConfigKit input into a collection.
|
||||||
|
Promptkit now uses ConfigKit v0.8.0, including its optional source specification
|
||||||
|
header support.
|
||||||
|
|
||||||
|
`Engine.WithNamespaces` creates an immutable view with separate prompt, profile,
|
||||||
|
and backend scopes for short caller-supplied IDs. Views share the underlying
|
||||||
|
configuration, clients, capacity pools, and prepared-execution owner. Explicitly
|
||||||
|
qualified IDs and configured references keep their exact identities.
|
||||||
|
|
||||||
|
`ListProfiles` and `ListPrompts` support namespace filters. New `ListBackends`,
|
||||||
|
`ListNamespaces`, and `ListCollections` methods expose captured resource and
|
||||||
|
collection inventories. See the [collection format](../formats.md#source-headers-collections-and-resource-ids),
|
||||||
|
[scoped-view guide](../consumers/pkg-promptkit.md#use-scoped-engine-views), and
|
||||||
|
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
|
## Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
`Engine.SmokeTest` provides backend reachability, authentication, text inference,
|
||||||
|
and structured-output inference diagnostics. Reports provide factual
|
||||||
|
observations, timing, and available usage rather than an overall readiness
|
||||||
|
verdict. Optional Boolean facts distinguish negative observations from unknown
|
||||||
|
or inapplicable checks; matching a JSON schema does not prove that a provider
|
||||||
|
enforced it.
|
||||||
|
|
||||||
|
A concluded diagnostic can report failure with a nil Go error. Invalid requests
|
||||||
|
return ordinary errors, and caller cancellation can return partial observations
|
||||||
|
with a recognizable context error. Diagnostics can contact providers and incur
|
||||||
|
inference costs only when explicitly invoked; they are not automatic execution
|
||||||
|
or release checks. See the [smoke diagnostic guide](../consumers/pkg-promptkit.md#run-explicit-smoke-diagnostics)
|
||||||
|
and [API GoDoc](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
## Breaking Changes And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.15.0 should follow the
|
||||||
|
[configuration and API migration guide](../consumers/configuration-migration.md):
|
||||||
|
|
||||||
|
- Supply backend and profile definitions explicitly. Promptkit no longer bundles
|
||||||
|
provider catalogs or the `promptkit-openrouter-standard` and
|
||||||
|
`promptkit-openrouter-auto` profiles. Conventional backend names do not install
|
||||||
|
definitions automatically. Applications choose and maintain their sources.
|
||||||
|
- Convert version-keyed prompt mappings into one definition per prompt ID.
|
||||||
|
Prompt `version` is now an optional descriptive label, not a selector.
|
||||||
|
Choose deliberately which former definition to install.
|
||||||
|
- Remove `PromptVersion` selectors from `RunRequest`,
|
||||||
|
`RequirementInspectionRequest`, and `SmokeTestRequest`, where applicable.
|
||||||
|
Change prompt inspection calls to `InspectPrompt(ctx, promptID)`. Update code
|
||||||
|
consuming version-bearing inspection, identity, result, or observation fields
|
||||||
|
to handle optional labels, including nil and explicitly empty labels.
|
||||||
|
- When moving resources into collections, update stored backend, profile, and
|
||||||
|
diagnostic-default references to their intended qualified IDs. Mounting a
|
||||||
|
source does not rewrite references; profile inheritance remains within its
|
||||||
|
sibling mapping. Root configuration `version`, when supplied, is now a source
|
||||||
|
specification header, distinct from descriptive resource labels.
|
||||||
|
|
||||||
|
The migration guide includes identity mappings and before/after prompt examples.
|
||||||
|
The [maintained offline example](../../examples/go-library/prepare/main.go)
|
||||||
|
demonstrates explicit composition, source-relative content, scoped preparation,
|
||||||
|
and inventory without contacting a provider.
|
||||||
|
|
||||||
|
After adapting configuration and API calls:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.16.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Correctness And Efficiency
|
||||||
|
|
||||||
|
Collection prompt materialization shares one lazy origin-metadata snapshot,
|
||||||
|
avoiding repeated full metadata copies across collections. Internal prepared
|
||||||
|
snapshots detach optional prompt labels. Smoke diagnostics preserve caller
|
||||||
|
cancellation and report actual inference destinations and known-zero usage when
|
||||||
|
no generation request was dispatched.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/docs/oss/attribution.md)
|
||||||
68
docs/releases/v0.17.0.md
Normal file
68
docs/releases/v0.17.0.md
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
# Promptkit v0.17.0
|
||||||
|
|
||||||
|
This release adopts the vanity module path `go.maximumdirect.net/promptkit`
|
||||||
|
and ConfigKit v1.0.0, standardizes configuration resources as identified lists,
|
||||||
|
and removes descriptive prompt-version metadata. These are intentional breaking
|
||||||
|
changes before v1. The minimum supported Go version increases from 1.26.0 to
|
||||||
|
1.27.0 for the ConfigKit v1 migration.
|
||||||
|
|
||||||
|
## Breaking Changes And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.16.0 should follow the
|
||||||
|
[configuration and API migration guide](../consumers/configuration-migration.md):
|
||||||
|
|
||||||
|
- Change Promptkit imports from `gitea.maximumdirect.net/eric/promptkit` to
|
||||||
|
`go.maximumdirect.net/promptkit`. Update packages that exchange Promptkit
|
||||||
|
types together; old-path and vanity-path types are distinct.
|
||||||
|
- Change ConfigKit imports to `go.maximumdirect.net/configkit` at v1.0.0 and
|
||||||
|
adopt its renamed public inspection types and fields where used.
|
||||||
|
- Convert root and collection `prompts`, `profiles`, and `backends` from keyed
|
||||||
|
mappings to lists whose records contain an explicit `id`. Collections remain
|
||||||
|
keyed by namespace. Sibling inheritance now uses `extends: {id: parent}`;
|
||||||
|
later same-ID records replace lower records in full.
|
||||||
|
- Change `ProfileSource` and `BackendSource` calls to pass slices of
|
||||||
|
`ProfileConfig` and `BackendConfig`, setting each record's `ID`.
|
||||||
|
- Change source-root `configkit:` controls to `config:`. Imports are optional
|
||||||
|
unless marked `required: true`. Optional Promptkit specification headers now
|
||||||
|
use `version: 2` or `version: "2"`; headerless sources remain valid.
|
||||||
|
- Remove descriptive prompt `version` values and code consuming `PromptVersion`,
|
||||||
|
`PromptIdentity.Version`, or corresponding `prompt_version` output members.
|
||||||
|
Collection `release_version` labels remain available. Prompt hashes may change
|
||||||
|
across this migration.
|
||||||
|
|
||||||
|
After adapting Go imports, configuration, and API calls, use Go 1.27.0 or newer:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.17.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Earlier tags retain their original module declarations. No permanent local
|
||||||
|
replacement is needed for this cutover. See the
|
||||||
|
[framework format reference](../formats.md) for current configuration semantics
|
||||||
|
and the [maintained preparation example](../../examples/go-library/prepare/main.go)
|
||||||
|
for explicit composition, source-relative imports, and scoped discovery.
|
||||||
|
|
||||||
|
## Configuration Inspection And Execution
|
||||||
|
|
||||||
|
Configuration inspection and diffs expose ConfigKit's composed values and
|
||||||
|
provenance, including list-index paths. Promptkit applies domain normalization
|
||||||
|
and execution defaults to owned interpretations without replacing the source
|
||||||
|
origins needed for relative resources. Profile inspection and preparation expose
|
||||||
|
their interpreted or effective settings. See the
|
||||||
|
[migration guide](../consumers/configuration-migration.md#select-and-compose-the-configuration-you-need)
|
||||||
|
for the distinction and inspection-path updates.
|
||||||
|
|
||||||
|
## Maintainer Tooling
|
||||||
|
|
||||||
|
Release tooling now uses the module-selected ReleaseKit v0.8.0 through `go tool`,
|
||||||
|
with page-only Gitea delivery and saved-publication-attempt recovery. The
|
||||||
|
[release procedure](../release.md) describes publication and recovery.
|
||||||
|
|
||||||
|
The three maintained offline examples now have ordinary Go workflow tests,
|
||||||
|
included in the standard suite. Their separate Python runner and custom release
|
||||||
|
hook have been removed; Python is no longer a normal release prerequisite.
|
||||||
|
The optional release-wiring diagnostic remains Python. See the
|
||||||
|
[development guide](../development.md#maintainer-validation).
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/docs/oss/attribution.md)
|
||||||
65
docs/releases/v0.18.0.md
Normal file
65
docs/releases/v0.18.0.md
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
# Promptkit v0.18.0
|
||||||
|
|
||||||
|
Configuration inspection now exposes Promptkit's recorded field normalization
|
||||||
|
and derived queue-capacity defaults through ConfigKit v1.1.0. Engines also
|
||||||
|
include a default-on OpenRouter configuration. The minimum supported Go
|
||||||
|
version remains 1.27.0.
|
||||||
|
|
||||||
|
## Bundled OpenRouter Backend And Profiles
|
||||||
|
|
||||||
|
New engines include an embedded OpenRouter backend and seventeen root profiles:
|
||||||
|
ten latest-family aliases, the auto and free routers, and five auto-router
|
||||||
|
cost-tier variants. Select a profile explicitly in a prompt or request; the
|
||||||
|
bundle does not supply an implicit execution profile or contact providers
|
||||||
|
during construction. The backend uses `OPENROUTER_API_KEY` when available under
|
||||||
|
the existing optional-credential rules.
|
||||||
|
|
||||||
|
The bundle is the lowest-priority underlay. Applications can replace its records
|
||||||
|
or derive named variants through ordinary configuration, and external collections
|
||||||
|
remain supported. Cost-tier variants inherit the effective `auto` profile, so
|
||||||
|
replacing that parent changes their inherited target while retaining plugins.
|
||||||
|
The [bundle reference](../formats.md#bundled-openrouter-configuration) owns the
|
||||||
|
inventory and composition details.
|
||||||
|
|
||||||
|
Explicit smoke diagnostics now have OpenRouter authentication metadata and an
|
||||||
|
inference fallback profile. They remain opt-in and may incur provider costs.
|
||||||
|
Latest aliases and routers can change upstream; configured slugs do not pin
|
||||||
|
the concrete model, pricing, or availability. See the
|
||||||
|
[consumer guide](../consumers/pkg-promptkit.md#construct-an-engine) for selection
|
||||||
|
and opt-out examples.
|
||||||
|
|
||||||
|
## Recorded Configuration Normalization
|
||||||
|
|
||||||
|
Inspection shows normalized backend and profile settings, prompt metadata, and
|
||||||
|
configured message roles and cache controls, with field-level resolution
|
||||||
|
history. Existing source evidence remains available, and unchanged content-file
|
||||||
|
origins continue to select their resource context.
|
||||||
|
|
||||||
|
Configuration diffs compare normalized effective values, so spelling-only
|
||||||
|
changes that normalize identically may no longer appear. Lists remain atomic in
|
||||||
|
diffs. Inspection does not include backend-to-profile completion or request
|
||||||
|
overrides; see the [configuration inspection guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
New engines install a bundled OpenRouter backend and root profiles by default.
|
||||||
|
Inventory listings and inspection sources therefore change, and references to
|
||||||
|
bundled IDs that previously failed may now resolve. Pass
|
||||||
|
`promptkit.WithBundledOpenRouter(false)` to retain application-only behavior.
|
||||||
|
Caller records can replace bundled definitions through ordinary composition;
|
||||||
|
same-ID replacements replace complete records rather than merging selected fields.
|
||||||
|
No configuration-format migration is required from v0.17.0.
|
||||||
|
Consumers that display or compare configuration inspection results should account
|
||||||
|
for normalized values and derived defaults replacing their earlier authored
|
||||||
|
representations. Request-time overrides and caller-appended messages retain
|
||||||
|
their existing processing.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.18.0
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [framework format reference](../formats.md) for current configuration
|
||||||
|
semantics. Consumers upgrading from versions before v0.17.0 should also follow
|
||||||
|
the [configuration and API migration guide](../consumers/configuration-migration.md).
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/docs/oss/attribution.md)
|
||||||
80
docs/releases/v0.19.0.md
Normal file
80
docs/releases/v0.19.0.md
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
# Promptkit v0.19.0
|
||||||
|
|
||||||
|
This release completes two rounds of audit remediation, correcting error and
|
||||||
|
diagnostic reporting and simplifying configuration, validation, and value
|
||||||
|
ownership. It includes intentional compatibility changes for custom artifact
|
||||||
|
readers, programmatic configuration sources, and schemas. The minimum supported
|
||||||
|
Go version remains 1.27.0; ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.18.0 should review these changes:
|
||||||
|
|
||||||
|
- A successful `ArtifactReader.Read` transfers ownership of the returned
|
||||||
|
artifact and its mutable body storage to Promptkit. Custom readers must stop
|
||||||
|
mutating returned values through retained aliases. Readers that reuse scratch
|
||||||
|
buffers must return independent storage. Promptkit retains its defensive
|
||||||
|
body copy, but that copy does not make concurrent buffer reuse safe. See the
|
||||||
|
[extension guidance](../consumers/pkg-promptkit.md#extension-interfaces).
|
||||||
|
- Provider maps passed through `ProfileSource` and `BackendSource` now use
|
||||||
|
ConfigKit's native Mapping input forms directly: built-in scalars, `[]any`,
|
||||||
|
and `map[string]any` recursively. Convert `json.Number` and typed containers
|
||||||
|
explicitly to supported values, choosing numeric precision appropriate to
|
||||||
|
the application. The former recursive compatibility conversion is removed.
|
||||||
|
Runtime execution overrides retain their separate accepted-value behavior.
|
||||||
|
- Schemas used for local validation and native structured output must be
|
||||||
|
self-contained. Move externally referenced definitions into the root document
|
||||||
|
and use internal references or explicit schema IDs. References requiring
|
||||||
|
another file or network resource, or relying on the source filename as a base,
|
||||||
|
fail during preparation. Promptkit does not bundle external schemas. See the
|
||||||
|
[schema reference](../formats.md#schemas).
|
||||||
|
|
||||||
|
Prepared schemas no longer inherit the depth and node limits used when copying
|
||||||
|
runtime provider values. Schemas previously rejected solely by those incidental
|
||||||
|
limits may now be accepted; decoder/compiler checks and the self-contained
|
||||||
|
schema requirement still apply.
|
||||||
|
|
||||||
|
No configuration-format migration is required from v0.18.0. After addressing
|
||||||
|
any affected integrations:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.19.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Errors And Diagnostics
|
||||||
|
|
||||||
|
Public errors and terminal observations now preserve the category of the
|
||||||
|
operation that failed even when an injected collaborator returns an error from
|
||||||
|
another Promptkit operation. Underlying causes remain available through
|
||||||
|
`errors.Is` and `errors.As`, without repeatedly duplicating nested cause trees.
|
||||||
|
|
||||||
|
Built-in generation preserves response-body read failures separately from
|
||||||
|
malformed response content, and failures during request preparation are
|
||||||
|
distinguished from dispatched attempts.
|
||||||
|
|
||||||
|
OpenRouter authentication diagnostics bound response recognition to 1 MiB.
|
||||||
|
An oversized HTTP 200 response reports
|
||||||
|
`SmokeTestReasonAuthenticationResponseLimitExceeded`, leaving credential
|
||||||
|
acceptance unknown. Cancellation during the read remains an interruption.
|
||||||
|
Consumers that enumerate diagnostic reasons should handle the new reason;
|
||||||
|
see the [diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||||
|
|
||||||
|
## Internal Simplification And Ownership
|
||||||
|
|
||||||
|
Prompt lookup and rendering now consistently use captured configuration, and
|
||||||
|
smoke syntax and schema checks share the prepared output-validation lifecycle.
|
||||||
|
Unused routing defaults, alternate private workflows, and duplicate wire
|
||||||
|
representations have been removed.
|
||||||
|
|
||||||
|
Provider-setting resolution, inspection, prepared details, and smoke reports
|
||||||
|
avoid redundant validation and copying of already accepted values. Prepared
|
||||||
|
execution shares one private redacted metadata snapshot while preserving
|
||||||
|
independent caller-visible values. Provider values are detached before custom
|
||||||
|
JSON encoders run, protecting retained metadata from encoder-side mutation.
|
||||||
|
|
||||||
|
Requirement analysis, namespace-filtered listings, and repair-feedback
|
||||||
|
construction also avoid unnecessary traversal and temporary allocation. These
|
||||||
|
changes preserve existing request isolation and execution behavior without
|
||||||
|
introducing new performance guarantees.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/docs/oss/attribution.md)
|
||||||
55
docs/releases/v0.20.0.md
Normal file
55
docs/releases/v0.20.0.md
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Promptkit v0.20.0
|
||||||
|
|
||||||
|
This minor release adds generically renderable resource inspection and typed
|
||||||
|
prompt source construction. The Go consumer minimum remains 1.27.0 and
|
||||||
|
ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Migration
|
||||||
|
|
||||||
|
`InspectProfile` now returns the resolved profile resource instead of the
|
||||||
|
backend-completed `EffectiveModelParams`. Replace accesses to that field with
|
||||||
|
the optional profile fields; use `Prepare` or `PreparedExecution.Details` for
|
||||||
|
completed request settings. `InspectPrompt` now includes captured messages and
|
||||||
|
uses optional pointers for presence-sensitive fields, including default profile,
|
||||||
|
inputs, input metadata, and output settings. Check pointers before dereferencing.
|
||||||
|
All three resource reports support ordinary JSON and YAML rendering without
|
||||||
|
per-field adapters. See the [consumer guidance](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
|
After configuration composition and inheritance, profile null endpoint,
|
||||||
|
API-key source, and whole provider map now permit backend defaults during
|
||||||
|
execution as well as appearing absent in resource inspection. Use explicit
|
||||||
|
empty strings to clear endpoint or API-key source. There is no replacement for
|
||||||
|
profile-level suppression of an entire backend provider map; edit backend
|
||||||
|
defaults through composition instead. The required-key rule still applies.
|
||||||
|
See [defaults and overrides](../formats.md#defaults-and-overrides) for the
|
||||||
|
migration example and retained ConfigKit null provenance.
|
||||||
|
|
||||||
|
## Resource Inspection And Sources
|
||||||
|
|
||||||
|
`InspectBackend` reports the resolved backend and configured diagnostics without
|
||||||
|
contacting a provider or deriving diagnostic destinations. Prompt and profile
|
||||||
|
inspection likewise report selected resources independently of request readiness;
|
||||||
|
configuration provenance, requirement analysis, and preparation remain separate.
|
||||||
|
`PromptSource` adds typed programmatic prompt definitions, and `BackendConfig`
|
||||||
|
supports sibling inheritance. ConfigKit continues to own source composition,
|
||||||
|
snapshotting, and relative message-file contexts.
|
||||||
|
|
||||||
|
The source helpers' [GoDoc](../../prompts.go) and
|
||||||
|
[backend configuration API](../../backends.go) describe programmatic construction.
|
||||||
|
|
||||||
|
## Refinements
|
||||||
|
|
||||||
|
Package GoDoc and migration guidance now consistently describe the resource
|
||||||
|
inspection contract, and the full package overview is visible in generated
|
||||||
|
documentation. Prompt inspection avoids repeating the same metadata lookup for
|
||||||
|
each declared input.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
After adapting affected inspection consumers and reviewing profile null usage:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.20.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/docs/oss/attribution.md)
|
||||||
34
docs/releases/v0.21.0.md
Normal file
34
docs/releases/v0.21.0.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Promptkit v0.21.0
|
||||||
|
|
||||||
|
This minor release updates ConfigKit from v1.1.0 to v1.2.0, making main-relative
|
||||||
|
companion imports available through Promptkit's existing configuration-source
|
||||||
|
API. The Go consumer minimum remains 1.27.0.
|
||||||
|
|
||||||
|
## Main-Relative Companion Imports
|
||||||
|
|
||||||
|
Applications can bind a non-file underlay's declared imports to the selected
|
||||||
|
main configuration's filesystem context. For example, an application can load
|
||||||
|
prompt definitions from an adjacent `prompts/` directory without calculating
|
||||||
|
that directory before loading the main configuration.
|
||||||
|
|
||||||
|
Imports remain explicitly selected by the caller; the upgrade does not enable
|
||||||
|
automatic discovery. Imported prompt message files retain their source-local
|
||||||
|
context. The [consumer example](../consumers/pkg-promptkit.md#load-companion-prompts-relative-to-the-main-configuration)
|
||||||
|
shows collection mounting and YAML filename filtering. ConfigKit's
|
||||||
|
[layered-loading contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md#layered-loading)
|
||||||
|
defines context selection and fallback behavior.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Promptkit's exported methods and request shapes are unchanged. No configuration
|
||||||
|
format migration is required from v0.20.0. Applications constructing ConfigKit
|
||||||
|
`Input` or `LayeredOptions` with positional struct literals must switch to
|
||||||
|
field-named literals because those upstream types gained fields. See the
|
||||||
|
[ConfigKit release notes](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/releases/v1.2.0.md)
|
||||||
|
for upstream changes.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.21.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/docs/oss/attribution.md)
|
||||||
67
docs/releases/v0.22.0.md
Normal file
67
docs/releases/v0.22.0.md
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
# Promptkit v0.22.0
|
||||||
|
|
||||||
|
This minor release adds captured conversation exchanges, optional sequential
|
||||||
|
coordination, and explicit execution-group summaries. Applications can continue
|
||||||
|
or branch a conversation without reconstructing rendered history, while keeping
|
||||||
|
response interpretation and workflow decisions in application code.
|
||||||
|
|
||||||
|
## Capture And Continue Conversations
|
||||||
|
|
||||||
|
Set `RunRequest.CaptureExchange` to capture a completed run's effective messages
|
||||||
|
and final assistant content. Supply its `RunResult.Exchange` as `ContinueFrom`
|
||||||
|
on a new request to continue it. Exchanges are reusable, support independent
|
||||||
|
branches and different engines, and expose detached history through `Details()`.
|
||||||
|
Prepared execution also supports capture of the exact snapshot it executes.
|
||||||
|
|
||||||
|
Internal structural-repair instructions and intermediate candidates are not
|
||||||
|
conversation history. A completed result with failed structural validation can
|
||||||
|
still be captured; an operational error returning no result creates no exchange.
|
||||||
|
Follow-ups select their own profiles, output contracts, credentials, and sessions
|
||||||
|
under ordinary request rules. See [capture and continuation guidance](../consumers/pkg-promptkit.md#capture-and-continue-a-conversation)
|
||||||
|
and the [public exchange contract](../../exchange.go).
|
||||||
|
|
||||||
|
## Coordinate Application-Directed Turns
|
||||||
|
|
||||||
|
`Engine.Coordinate` advances one conversation branch using an application callback
|
||||||
|
and an additional-turn budget. The application selects the next request or stops;
|
||||||
|
Promptkit handles turn progression and cancellation. Each completed run releases
|
||||||
|
backend admission before the callback, allowing assessment calls to use the same
|
||||||
|
backend. Stop and budget exhaustion are control-flow outcomes, not judgments of
|
||||||
|
response acceptance. Transport retries and structural repairs remain separate
|
||||||
|
from the turn budget.
|
||||||
|
|
||||||
|
See the [coordination walkthrough](../consumers/pkg-promptkit.md#coordinate-application-directed-turns)
|
||||||
|
and [coordinator GoDoc](../../coordinate.go) for callback and cancellation behavior.
|
||||||
|
|
||||||
|
## Collect Selected Execution Facts
|
||||||
|
|
||||||
|
`ExecutionGroup` collects explicitly selected execution observations, while
|
||||||
|
`ComposeObservers` combines groups with existing diagnostic observers. Detached
|
||||||
|
group snapshots report member identities, terminal coverage, attempt/retry/repair
|
||||||
|
counts, and available token totals without counting a run more than once.
|
||||||
|
Terminal coverage remains distinct from token-usage completeness; membership is
|
||||||
|
not inferred from ancestry or nested calls.
|
||||||
|
|
||||||
|
See [execution-group guidance](../consumers/pkg-promptkit.md#collect-selected-execution-facts),
|
||||||
|
the [group GoDoc](../../execution_group.go), and the updated
|
||||||
|
[offline observation example](../../examples/go-library/observe/main.go).
|
||||||
|
|
||||||
|
## Efficiency And Compatibility
|
||||||
|
|
||||||
|
History assembly and inspection avoid intermediate full-history copies. Observer
|
||||||
|
composition retains storage only for selected callbacks and avoids fan-out
|
||||||
|
closures for zero or one selected callback.
|
||||||
|
|
||||||
|
Capture and coordination are opt-in; existing execution workflows and
|
||||||
|
configuration formats need no migration from v0.21.0. The Go consumer minimum
|
||||||
|
remains 1.27.0, and dependency versions are unchanged. Exported request, prepared,
|
||||||
|
result, and observation structs gained fields; consumers using positional struct
|
||||||
|
literals must switch to keyed literals. Continuation metadata adds `parent_run_id`
|
||||||
|
to prepared/result JSON when applicable; the opaque exchange itself is excluded.
|
||||||
|
Exchanges are in-process capabilities, not serialized restartable sessions.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.22.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.22.0/docs/oss/attribution.md)
|
||||||
62
docs/releases/v0.23.0.md
Normal file
62
docs/releases/v0.23.0.md
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
# Promptkit v0.23.0
|
||||||
|
|
||||||
|
This minor release adds optional whole-operation timeouts and prepared-request
|
||||||
|
notifications to ordinary and coordinated execution. Applications can bound
|
||||||
|
each continuation independently and retain the request selected for execution
|
||||||
|
without replacing Promptkit's turn loop or preparing the same request twice.
|
||||||
|
|
||||||
|
## Bound Individual Operations
|
||||||
|
|
||||||
|
`RunRequest.OperationTimeout` selects a fresh cooperative budget covering
|
||||||
|
preparation and execution, including capacity waiting, retries, validation,
|
||||||
|
structural repair, and prepared-request notification. It is independent of the
|
||||||
|
provider generation timeout. Zero or negative values select no additional
|
||||||
|
timeout; existing parent cancellation and earlier deadlines remain effective.
|
||||||
|
|
||||||
|
For split execution, `PrepareExecution` retains the duration without starting
|
||||||
|
a timer. `RunPrepared` starts that budget after claiming the handle, so prior
|
||||||
|
preparation and idle time do not consume it. Callers can still supply one shared
|
||||||
|
deadline context to bound both calls together.
|
||||||
|
|
||||||
|
Local budget failures match both `ErrOperationTimeout` and
|
||||||
|
`context.DeadlineExceeded`. Each coordinated turn uses its selected budget;
|
||||||
|
expiry stops that coordination through the existing operational-error path
|
||||||
|
without canceling its parent or sibling work. Completed outcomes are fixed
|
||||||
|
before terminal observation and are not changed by a slow terminal observer.
|
||||||
|
Cancellation does not forcibly interrupt callbacks or collaborators that ignore
|
||||||
|
their context.
|
||||||
|
|
||||||
|
See the [execution-controls guide](../consumers/pkg-promptkit.md#bound-execution-and-retain-prepared-details)
|
||||||
|
and the [public request contract](../../types.go) and
|
||||||
|
[timeout error contract](../../engine.go) for exact behavior.
|
||||||
|
|
||||||
|
## Retain Prepared Details Before Generation
|
||||||
|
|
||||||
|
`RunRequest.OnPrepared` optionally receives a detached, credential-redacted
|
||||||
|
`PreparedRun` and the operation context before first generation. It uses the
|
||||||
|
actual prepared snapshot, including continuation history, without rereading
|
||||||
|
inputs or rerendering. Applications may retain these details even if generation
|
||||||
|
later fails without returning a result.
|
||||||
|
|
||||||
|
Notification is synchronous, at most once per logical execution, and runs while
|
||||||
|
logical admission is held. It is not repeated for retries or structural repair.
|
||||||
|
Preparation or admission failure can prevent delivery. The application owns
|
||||||
|
diagnostic storage and disclosure decisions; ordinary observers remain
|
||||||
|
metadata-only. See the [OnPrepared GoDoc](../../types.go) and updated
|
||||||
|
[offline observation example](../../examples/go-library/observe/main.go).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Both controls are opt-in. Existing keyed requests and configuration files need
|
||||||
|
no migration from v0.22.0. `Prepare` and `ValidateRequest` continue using only
|
||||||
|
their supplied contexts and do not invoke `OnPrepared`. `RunRequest` gained
|
||||||
|
fields; consumers using positional struct literals must switch to keyed
|
||||||
|
literals. Prepared/result JSON representations are unchanged.
|
||||||
|
|
||||||
|
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.23.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/docs/oss/attribution.md)
|
||||||
33
docs/releases/v0.23.1.md
Normal file
33
docs/releases/v0.23.1.md
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
# Promptkit v0.23.1
|
||||||
|
|
||||||
|
This patch release fixes construction-error wrapping so downstream consumers
|
||||||
|
can classify failures from their underlying causes without treating the
|
||||||
|
`ErrInvalidConfig` category as an independent failure.
|
||||||
|
|
||||||
|
## Fixed
|
||||||
|
|
||||||
|
`NewEngine` now uses single-cause wrapping for both configuration-loading and
|
||||||
|
option-application failures. `errors.Is` still recognizes `ErrInvalidConfig`,
|
||||||
|
and `errors.As` still exposes ConfigKit loading errors with their diagnostics
|
||||||
|
and source context. Genuine joined causes, including cancellation, remain
|
||||||
|
inspectable. Consumers retain responsibility for mapping those failures to
|
||||||
|
application outcomes or exit statuses.
|
||||||
|
|
||||||
|
See the [constructor GoDoc](../../engine.go) and
|
||||||
|
[error-handling guide](../consumers/pkg-promptkit.md#handle-errors) for the
|
||||||
|
public error contracts and usage.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No configuration or request changes are required when upgrading from v0.23.0.
|
||||||
|
Consumers that inspect constructor error trees directly should account for
|
||||||
|
the category no longer appearing as a separate sibling branch; use
|
||||||
|
`errors.Is` and `errors.As` for category and typed-cause inspection.
|
||||||
|
|
||||||
|
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.23.1
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.1/docs/oss/attribution.md)
|
||||||
35
docs/releases/v0.23.2.md
Normal file
35
docs/releases/v0.23.2.md
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
# Promptkit v0.23.2
|
||||||
|
|
||||||
|
This patch release refreshes configuration, validation, and maintainer-tool
|
||||||
|
dependencies. No Promptkit configuration or request API migration is required.
|
||||||
|
|
||||||
|
## Dependency Updates
|
||||||
|
|
||||||
|
- ConfigKit v1.2.0 → v1.2.1 and go-yaml v4.0.0-rc.6 → v4.0.0 move
|
||||||
|
configuration loading to stable go-yaml, with ConfigKit's corresponding
|
||||||
|
loader-plugin adaptation.
|
||||||
|
- jsonschema v6.0.2 → v6.0.3 includes fixes to validation-location reporting,
|
||||||
|
numeric edge cases, and email validation.
|
||||||
|
- Go Text (`golang.org/x/text`) v0.14.0 → v0.44.0 refreshes the transitive
|
||||||
|
dependency used by jsonschema.
|
||||||
|
- ReleaseKit v0.9.0 → v0.10.0 updates maintainer tooling, selecting Goldmark
|
||||||
|
v2.1.16 and Go Modules (`golang.org/x/mod`) v0.43.0. ReleaseKit's Go 1.27.0
|
||||||
|
minimum is already satisfied by Promptkit's existing minimum.
|
||||||
|
|
||||||
|
Dependency notices and revision links are maintained in the
|
||||||
|
[attribution document](../oss/attribution.md). Maintainers should use the
|
||||||
|
updated [release runbook](../release.md); publication and hosted delivery
|
||||||
|
remain separate operations.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
The Go consumer minimum remains 1.27.0. Existing configuration, requests,
|
||||||
|
and release commands require no changes from v0.23.1. Upstream validator
|
||||||
|
fixes may affect validation results or diagnostic locations for edge cases.
|
||||||
|
See the [consumer guide](../consumers/pkg-promptkit.md) for current usage.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.23.2
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.2/docs/oss/attribution.md)
|
||||||
36
docs/releases/v0.23.3.md
Normal file
36
docs/releases/v0.23.3.md
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
# Promptkit v0.23.3
|
||||||
|
|
||||||
|
This patch release corrects timeout attribution and preserves error identities
|
||||||
|
in the built-in OpenAI-compatible HTTP client.
|
||||||
|
|
||||||
|
## Fixed
|
||||||
|
|
||||||
|
- Operation budgets expiring during an in-flight HTTP generation request now
|
||||||
|
match both `ErrOperationTimeout` and `context.DeadlineExceeded`. Terminal
|
||||||
|
observations report a deadline failure instead of a transport failure.
|
||||||
|
- HTTP failures carrying custom caller cancellation or deadline causes retain
|
||||||
|
both the custom cause and the standard context identity. Unrelated transport
|
||||||
|
failures are not relabeled solely because a context has been canceled.
|
||||||
|
- Cancellation while reading a successful response body preserves the original
|
||||||
|
reader error alongside the context identity, without exposing its diagnostic
|
||||||
|
text through the redacted error message.
|
||||||
|
|
||||||
|
Regression coverage exercises `Run` and `RunPrepared`, public error identities,
|
||||||
|
and attempt and terminal classifications. See the [request and timeout
|
||||||
|
contracts](../../types.go), [public error GoDoc](../../engine.go), and
|
||||||
|
[error-handling guide](../consumers/pkg-promptkit.md#handle-errors).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No configuration or request API migration is required from v0.23.2. Consumers
|
||||||
|
using `errors.Is` for cancellation or deadline handling now receive the
|
||||||
|
correct identities on the affected paths; applications continue to own their
|
||||||
|
status mapping and diagnostic presentation.
|
||||||
|
|
||||||
|
The Go consumer minimum remains 1.27.0, and dependency versions are unchanged.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.23.3
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.3/docs/oss/attribution.md)
|
||||||
83
docs/releases/v0.24.0.md
Normal file
83
docs/releases/v0.24.0.md
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
# Promptkit v0.24.0
|
||||||
|
|
||||||
|
This minor release adds application-owned configuration loading, prompt-owned
|
||||||
|
schemas, and a prepared message-content hash. It also enables one repair attempt
|
||||||
|
by default for enabled output validation. This is a breaking pre-v1 release;
|
||||||
|
consumers upgrading from v0.23.x should review the migration steps below.
|
||||||
|
|
||||||
|
## Added And Changed
|
||||||
|
|
||||||
|
- Applications can embed `promptkit.Settings`, compose Promptkit's rules with
|
||||||
|
`Configure`, load configuration once through ConfigKit, and pass the entire
|
||||||
|
result to `NewFromConfig`. `NewEngine` remains the standalone convenience
|
||||||
|
constructor. See the [shared-loading guide](../consumers/pkg-promptkit.md#reuse-an-application-owned-load)
|
||||||
|
and [offline preparation example](../../examples/go-library/prepare/main.go).
|
||||||
|
- Prompts can select inline, source-relative, or explicit-root JSON Schemas.
|
||||||
|
Collections can supply their own schema root. Schema resources are loaded
|
||||||
|
during preparation and retained by prepared execution. See the
|
||||||
|
[schema reference](../formats.md#schemas).
|
||||||
|
- Prepared results include `MessageContentHash` (`message_content_hash` in
|
||||||
|
JSON), an opaque equality value for ordered message roles and exact text,
|
||||||
|
independent of session identity, cache hints, and other metadata. The existing
|
||||||
|
rendered-prompt hash algorithm is unchanged. See the
|
||||||
|
[prepared-result GoDoc](../../types.go) and
|
||||||
|
[preparation guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
||||||
|
- Configuration inspection uses a native ConfigKit view focused on `/promptkit`
|
||||||
|
and adds `HistoricalPaths`. It retains whole-load evidence and supports
|
||||||
|
explicit queries outside that focus; comparison can include ancestor changes.
|
||||||
|
See the [inspection guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration).
|
||||||
|
|
||||||
|
## Fixes And Efficiency
|
||||||
|
|
||||||
|
- Prompt-input inspection and requirements now retain effective metadata supplied
|
||||||
|
by application defaults, including inherited input requirements.
|
||||||
|
- Configuration capture avoids expanding inheritance provenance just to determine
|
||||||
|
input-field presence, and shares immutable backend defaults across profiles
|
||||||
|
without overrides while preserving defensive copies at public lookups.
|
||||||
|
- Schema handling avoids copying inactive runtime schemas and unused slice
|
||||||
|
capacity. Inline schemas are separate from provider extra-parameter copy limits.
|
||||||
|
See the [schema reference](../formats.md#schemas) and
|
||||||
|
[request GoDoc](../../types.go).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
The [configuration migration guide](../consumers/configuration-migration.md#move-settings-under-promptkit)
|
||||||
|
owns the full migration walkthrough. In particular:
|
||||||
|
|
||||||
|
- Move complete-document resources and `schema_root` under `promptkit`.
|
||||||
|
Previous root-level resources are now application data, not Promptkit
|
||||||
|
resources; IDs and references keep their existing names. `SettingsInput`
|
||||||
|
mounts wrapperless fragments at `/promptkit`; `CollectionInput` mounts them
|
||||||
|
at `/promptkit/collections/<namespace>`. Discovery helpers still produce
|
||||||
|
root-mounted inputs, so adjust fragment destinations explicitly.
|
||||||
|
- For an application-owned load, select `BundledOpenRouterUnderlay` before
|
||||||
|
loading if you want the bundled catalog. Bundling remains default-on for
|
||||||
|
`NewEngine`; it is not implicitly added by `NewFromConfig`.
|
||||||
|
- Replace `RunRequest.Validation` values of type `*OutputContract` with
|
||||||
|
`*OutputContractInput`; its `RepairAttempts` field is a pointer. Omitted or
|
||||||
|
null budgets now resolve to one for `basic`, `json`, and `json_schema`, or
|
||||||
|
zero for `none`. Set an explicit zero to retain single-pass behavior.
|
||||||
|
Runtime replacements still replace the whole contract and resolve their
|
||||||
|
defaults independently. A failed check may now incur another generation call,
|
||||||
|
latency, and usage, without guaranteeing a valid result. See the
|
||||||
|
[output contract](../formats.md#output-contract).
|
||||||
|
- `PromptOutputInspection.RepairAttempts` is now a resolved integer rather than
|
||||||
|
an optional pointer, and its JSON includes zero. `OutputContract` remains the
|
||||||
|
resolved contract type used by prepared details. See the
|
||||||
|
[inspection GoDoc](../../resource_inspection.go) and [contract GoDoc](../../types.go).
|
||||||
|
- Schema paths require an explicitly configured applicable root; there is no
|
||||||
|
implicit working-directory root. Review file references and make schemas
|
||||||
|
self-contained as described in the [schema reference](../formats.md#schemas).
|
||||||
|
- Promptkit no longer validates resource-version fields. Applications own version
|
||||||
|
policy and sensitivity rules for their extension fields; see the
|
||||||
|
[format reference](../formats.md).
|
||||||
|
|
||||||
|
Older prepared-result JSON without `message_content_hash` decodes with an empty
|
||||||
|
value. Callers using positional `PreparedRun` literals may need to adjust them.
|
||||||
|
ConfigKit is updated to v1.4.0; the Go consumer minimum remains 1.27.0.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.24.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.24.0/docs/oss/attribution.md)
|
||||||
@@ -63,8 +63,8 @@ Every engine now includes the reserved `rakestrawhome` backend, identified by
|
|||||||
selects that backend. Consumers can use the maintained endpoint, credential,
|
selects that backend. Consumers can use the maintained endpoint, credential,
|
||||||
capacity, and model defaults without registering either definition themselves.
|
capacity, and model defaults without registering either definition themselves.
|
||||||
|
|
||||||
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
|
See the [backend format reference](../formats.md#backend-definitions)
|
||||||
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
|
and the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping)
|
||||||
for the current contracts.
|
for the current contracts.
|
||||||
|
|
||||||
## Structured Generation Errors
|
## Structured Generation Errors
|
||||||
@@ -126,8 +126,8 @@ cycle-safe, and resolved afresh for ordinary operations. Prepared execution
|
|||||||
freezes the fully resolved target. The selected leaf ID remains public while
|
freezes the fully resolved target. The selected leaf ID remains public while
|
||||||
effective execution settings reflect the resolved chain.
|
effective execution settings reflect the resolved chain.
|
||||||
|
|
||||||
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
See the [profile inheritance format reference](../formats.md#resource-inheritance),
|
||||||
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
|
the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping),
|
||||||
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
||||||
behavior.
|
behavior.
|
||||||
|
|
||||||
|
|||||||
@@ -96,9 +96,8 @@ precedence rules. Promptkit no longer keeps duplicate embedded profile assets
|
|||||||
or hard-coded definitions for those maintained backends.
|
or hard-coded definitions for those maintained backends.
|
||||||
|
|
||||||
The module versions in Promptkit's `go.mod` identify the catalog releases
|
The module versions in Promptkit's `go.mod` identify the catalog releases
|
||||||
tested with this release. The [built-in backend and profile format
|
tested with this release. The [backend format reference](../formats.md#backend-definitions)
|
||||||
reference](../formats.md#built-in-backends) remains the canonical consumer
|
remains the canonical consumer contract, while the [internal source documentation](../internal/sources.md#profiles)
|
||||||
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
|
|
||||||
describes the dependency boundary.
|
describes the dependency boundary.
|
||||||
|
|
||||||
## Public API Changes
|
## Public API Changes
|
||||||
|
|||||||
@@ -38,8 +38,17 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
No ideas are currently awaiting selection. Active feature work belongs in its
|
### Schema Bundling
|
||||||
focused roadmap rather than this catalog.
|
|
||||||
|
Allow consumers to author reusable schemas across multiple files and package
|
||||||
|
them into a single self-contained schema for local validation and provider
|
||||||
|
submission. This could restore multi-file authoring convenience while retaining
|
||||||
|
the [self-contained schema requirement](../formats.md#schemas).
|
||||||
|
|
||||||
|
Bundling would need to preserve reference semantics, identifiers, anchors, and
|
||||||
|
recursion. A complete bundle would not guarantee that a provider supports every
|
||||||
|
schema feature. This is a possible future enhancement; implementation is not
|
||||||
|
currently approved.
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
372
engine.go
372
engine.go
@@ -4,15 +4,17 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"reflect"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine"
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
internalengine "go.maximumdirect.net/promptkit/internal/engine"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||||
@@ -24,13 +26,14 @@ var (
|
|||||||
// ErrInvalidRequest identifies a request whose required values, overrides,
|
// ErrInvalidRequest identifies a request whose required values, overrides,
|
||||||
// credentials, or effective settings are invalid.
|
// credentials, or effective settings are invalid.
|
||||||
ErrInvalidRequest = errors.New("invalid run request")
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
// ErrPromptNotFound identifies a requested prompt ID or version that is not
|
// ErrPromptNotFound identifies a requested prompt ID that is not present in
|
||||||
// present in the selected prompt source. It does not also match
|
// the captured configuration. It does not also match ErrPromptLoad.
|
||||||
// ErrPromptLoad.
|
|
||||||
ErrPromptNotFound = errors.New("prompt not found")
|
ErrPromptNotFound = errors.New("prompt not found")
|
||||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||||
// every configured profile source. It does not also match ErrProfileLoad.
|
// every configured profile source. It does not also match ErrProfileLoad.
|
||||||
ErrProfileNotFound = errors.New("profile not found")
|
ErrProfileNotFound = errors.New("profile not found")
|
||||||
|
// ErrBackendNotFound identifies a backend absent from resource inspection.
|
||||||
|
ErrBackendNotFound = errors.New("backend not found")
|
||||||
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||||
// nor the selected prompt's default profile is present. Such an error also
|
// nor the selected prompt's default profile is present. Such an error also
|
||||||
// matches ErrInvalidRequest.
|
// matches ErrInvalidRequest.
|
||||||
@@ -58,6 +61,13 @@ var (
|
|||||||
// A [CapacityError] reports the selected backend ID. It is not an invalid
|
// A [CapacityError] reports the selected backend ID. It is not an invalid
|
||||||
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
||||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||||
|
// ErrOperationTimeout identifies expiration of the timeout selected by
|
||||||
|
// RunRequest.OperationTimeout. Such errors also match context.DeadlineExceeded
|
||||||
|
// and retain their underlying operation and collaborator identities.
|
||||||
|
// Earlier parent cancellation or deadlines and provider-only deadlines do
|
||||||
|
// not acquire this identity. A completed result remains completed even if
|
||||||
|
// a terminal observer outlasts the selected budget.
|
||||||
|
ErrOperationTimeout = errors.New("operation timeout exceeded")
|
||||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||||
// response. A built-in OpenAI-compatible non-2xx response is available as a
|
// response. A built-in OpenAI-compatible non-2xx response is available as a
|
||||||
// [GenerationError]. Errors returned by an injected LLMClient remain
|
// [GenerationError]. Errors returned by an injected LLMClient remain
|
||||||
@@ -82,23 +92,29 @@ var (
|
|||||||
ErrValidation = errors.New("failed to validate output")
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
// Engine inspects resources, enumerates prompt requirements, and
|
||||||
// prepares and runs Promptkit prompt requests.
|
// prepares and runs Promptkit prompt requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
||||||
// [Engine.InspectPrompt],
|
// [Engine.InspectConfiguration],
|
||||||
|
// [Engine.InspectPrompt], [Engine.InspectBackend],
|
||||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
||||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
||||||
// model generation. Injected collaborators may still be invoked concurrently
|
// [Engine.ListCollections]. Engines constructed separately by [NewEngine] own
|
||||||
// across different backend pools or for unlimited backends.
|
// independent backend-capacity pools that coordinate Run, RunPrepared, and
|
||||||
|
// inference SmokeTest admission and model generation. Namespace views returned
|
||||||
|
// by [Engine.WithNamespaces] share their underlying engine's pools, clients,
|
||||||
|
// and prepared-execution owner. Injected collaborators may still be invoked
|
||||||
|
// concurrently across different backend pools or for unlimited backends.
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
internal *internalengine.Engine
|
internal *internalengine.Engine
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
||||||
// highest-priority main input. Each underlay's inputs strictly compose within
|
// highest-priority main input. By default a bundled OpenRouter underlay precedes
|
||||||
// their group; later underlays overlay earlier ones. A nil Main uses
|
// the caller underlays; [WithBundledOpenRouter] can disable it. Inputs within
|
||||||
|
// each underlay strictly compose; later underlays overlay earlier ones. A nil Main uses
|
||||||
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
||||||
// may fail construction if invalid.
|
// may fail construction if invalid.
|
||||||
type ConfigurationSources struct {
|
type ConfigurationSources struct {
|
||||||
@@ -111,9 +127,6 @@ type ConfigurationSources struct {
|
|||||||
type Config struct {
|
type Config struct {
|
||||||
// Sources combines maintained definitions with caller-selected inputs.
|
// Sources combines maintained definitions with caller-selected inputs.
|
||||||
Sources ConfigurationSources
|
Sources ConfigurationSources
|
||||||
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
|
||||||
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
|
||||||
SchemaDir string
|
|
||||||
// Timeout is the transport-wide safety cap for the built-in LLM client
|
// Timeout is the transport-wide safety cap for the built-in LLM client
|
||||||
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
|
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
|
||||||
// value selects the 10-minute default.
|
// value selects the 10-minute default.
|
||||||
@@ -133,7 +146,7 @@ type Config struct {
|
|||||||
// Option customizes engine construction.
|
// Option customizes engine construction.
|
||||||
//
|
//
|
||||||
// NewEngine applies options in argument order and ignores nil options. Within
|
// NewEngine applies options in argument order and ignores nil options. Within
|
||||||
// each schema-source, model-client, and artifact-reader category, the last
|
// each model-client and artifact-reader category, the last
|
||||||
// non-nil valid option replaces earlier options. An invalid option fails
|
// non-nil valid option replaces earlier options. An invalid option fails
|
||||||
// construction even if a later option would replace it.
|
// construction even if a later option would replace it.
|
||||||
type Option interface {
|
type Option interface {
|
||||||
@@ -148,6 +161,17 @@ func (f optionFunc) apply(options *engineOptions) error {
|
|||||||
|
|
||||||
type engineOptions struct {
|
type engineOptions struct {
|
||||||
internal internalengine.Options
|
internal internalengine.Options
|
||||||
|
disableBundledOpenRouter bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithBundledOpenRouter enables or disables the built-in OpenRouter underlay.
|
||||||
|
// It is enabled by default for NewEngine; when repeated, the last option wins.
|
||||||
|
// It has no effect on an already resolved NewFromConfig handoff.
|
||||||
|
func WithBundledOpenRouter(enabled bool) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
options.disableBundledOpenRouter = !enabled
|
||||||
|
return nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||||
@@ -183,123 +207,200 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
|
||||||
//
|
|
||||||
// Prompt schema_path values resolve within this source when schema validation
|
|
||||||
// or structured output is requested. fsys must be non-nil and root must be
|
|
||||||
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
|
|
||||||
// replaces Config.SchemaDir and earlier schema-source options.
|
|
||||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
return options.internal.SetSchemaFS(fsys, root)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
|
||||||
//
|
|
||||||
// Prompt schema_path values refer to the file's base name. path must name an
|
|
||||||
// existing non-directory file when NewEngine applies the option. This option
|
|
||||||
// replaces Config.SchemaDir and earlier schema-source options.
|
|
||||||
func WithSchemaFile(path string) Option {
|
|
||||||
return optionFunc(func(options *engineOptions) error {
|
|
||||||
return options.internal.SetSchemaFile(path)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewEngine constructs an Engine from configuration and options.
|
// NewEngine constructs an Engine from configuration and options.
|
||||||
//
|
//
|
||||||
// ConfigKit composes maintained catalog inputs, PromptKit-owned definitions,
|
// ConfigKit composes caller-selected underlays in their given order, followed
|
||||||
// caller underlays, and the optional main input in that order. Construction
|
// by the optional main input, after the default-on bundled OpenRouter underlay.
|
||||||
// validates the final effective backend, profile, and prompt definitions and
|
// The bundle adds a root backend and profiles to inventories, but no prompts
|
||||||
// captures referenced prompt content. A missing prompt collection is allowed.
|
// or implicit profile selection. Construction validates the final effective
|
||||||
// Schema documents and request artifacts remain runtime resources.
|
// backend, profile, and prompt definitions and captures referenced prompt
|
||||||
|
// content. A missing prompt collection is allowed. Schema documents and
|
||||||
|
// request artifacts remain runtime resources.
|
||||||
//
|
//
|
||||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
// NewEngine returns an error matching ErrInvalidConfig through errors.Is for
|
||||||
// configuration or options, preserving a ConfigKit LoadError when loading
|
// invalid configuration or options. The error unwraps to the original
|
||||||
// fails. Each constructed
|
// construction cause, preserving a ConfigKit LoadError when loading fails.
|
||||||
// Engine has independent backend-capacity pools. Construction does not perform
|
// Each constructed Engine has independent backend-capacity pools. Construction
|
||||||
// model requests or require credentials. A negative Config.TransientRetries
|
// does not perform model requests or require credentials. A negative
|
||||||
// is invalid.
|
// Config.TransientRetries is invalid.
|
||||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||||
|
options, err := applyEngineOptions(opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
runtime := RuntimeConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: cfg.TransientRetries}
|
||||||
|
if _, err := domain.ResolveTransientRetries(2, runtime.TransientRetries); err != nil {
|
||||||
|
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
underlays := append([]configkit.Underlay(nil), cfg.Sources.Underlays...)
|
||||||
|
if !options.disableBundledOpenRouter {
|
||||||
|
underlays = append([]configkit.Underlay{BundledOpenRouterUnderlay()}, underlays...)
|
||||||
|
}
|
||||||
|
main := configkit.Input{Source: configkit.NoSource()}
|
||||||
|
if cfg.Sources.Main != nil {
|
||||||
|
main = *cfg.Sources.Main
|
||||||
|
}
|
||||||
|
definition := Configure(configkit.Definition[standaloneConfiguration]{}, func(value *standaloneConfiguration) *Settings { return &value.PromptKit })
|
||||||
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[standaloneConfiguration]{Underlays: underlays, Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
return newFromConfig(result, runtime, options)
|
||||||
|
}
|
||||||
|
|
||||||
|
type standaloneConfiguration struct {
|
||||||
|
PromptKit Settings `yaml:"promptkit"`
|
||||||
|
Application map[string]any `yaml:",inline"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Configuration is the completed application-owned configuration evidence
|
||||||
|
// needed to capture PromptKit resources without loading configuration again.
|
||||||
|
type Configuration interface {
|
||||||
|
Value(configkit.Path) (any, bool)
|
||||||
|
InspectSubtree(configkit.Path) (configkit.Inspection, bool)
|
||||||
|
Diagnostics() []configkit.Diagnostic
|
||||||
|
}
|
||||||
|
|
||||||
|
// RuntimeConfig selects runtime-only client and retry settings for NewFromConfig.
|
||||||
|
// Its fields have the same semantics as the corresponding Config fields.
|
||||||
|
type RuntimeConfig struct {
|
||||||
|
// Timeout is the built-in client's transport timeout fallback.
|
||||||
|
Timeout time.Duration
|
||||||
|
// HTTPClient is cloned for the built-in client and ignored by injected clients.
|
||||||
|
HTTPClient *http.Client
|
||||||
|
// TransientRetries selects additional built-in attempts; nil defaults to two.
|
||||||
|
TransientRetries *int
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFromConfig captures resources from an application-owned completed load.
|
||||||
|
// The application must register PromptKit rules during that load, using Configure
|
||||||
|
// or equivalent explicit composition. Missing or nil selected settings contribute
|
||||||
|
// no resources; Settings and *Settings are accepted, while other nonnil types,
|
||||||
|
// nil results, and unavailable inspection views fail with ErrInvalidConfig.
|
||||||
|
// Construction consumes resolved settings while inspection retains the load's
|
||||||
|
// frozen evidence. Callers should keep those consistent at handoff. After
|
||||||
|
// construction, later settings mutations do not alter the engine. This method
|
||||||
|
// does not load sources, rerun callbacks, or select bundled underlays.
|
||||||
|
func NewFromConfig(result Configuration, cfg RuntimeConfig, opts ...Option) (*Engine, error) {
|
||||||
|
options, err := applyEngineOptions(opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return newFromConfig(result, cfg, options)
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyEngineOptions(opts []Option) (engineOptions, error) {
|
||||||
var options engineOptions
|
var options engineOptions
|
||||||
for _, opt := range opts {
|
for _, opt := range opts {
|
||||||
if opt == nil {
|
if opt == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := opt.apply(&options); err != nil {
|
if err := opt.apply(&options); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
return engineOptions{}, newPublicOperationError(ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return options, nil
|
||||||
|
}
|
||||||
|
|
||||||
assembled, err := internalengine.New(
|
func newFromConfig(result Configuration, cfg RuntimeConfig, options engineOptions) (*Engine, error) {
|
||||||
internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
|
if result == nil || isNilConfiguration(result) {
|
||||||
options.internal,
|
return nil, newPublicOperationError(ErrInvalidConfig, errors.New("configuration result is nil"))
|
||||||
)
|
}
|
||||||
|
if _, err := domain.ResolveTransientRetries(2, cfg.TransientRetries); err != nil {
|
||||||
|
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
value, available := result.Value(configkit.Keys("promptkit"))
|
||||||
|
var selected *Settings
|
||||||
|
if available && value != nil {
|
||||||
|
switch section := value.(type) {
|
||||||
|
case Settings:
|
||||||
|
selected = §ion
|
||||||
|
case *Settings:
|
||||||
|
selected = section
|
||||||
|
default:
|
||||||
|
return nil, newPublicOperationError(ErrInvalidConfig, fmt.Errorf("promptkit section has incompatible type %T", value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
|
||||||
|
if !ok {
|
||||||
|
return nil, newPublicOperationError(ErrInvalidConfig, errors.New("configuration inspection is unavailable"))
|
||||||
|
}
|
||||||
|
snapshot, err := configuration.Capture(selected, view, result.Diagnostics())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
|
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
assembled, err := internalengine.New(snapshot, internalengine.Config{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)}, options.internal)
|
||||||
|
if err != nil {
|
||||||
|
return nil, newPublicOperationError(ErrInvalidConfig, err)
|
||||||
}
|
}
|
||||||
return &Engine{internal: assembled}, nil
|
return &Engine{internal: assembled}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt definition without selecting a
|
func isNilConfiguration(result Configuration) bool {
|
||||||
// profile or starting execution work.
|
value := reflect.ValueOf(result)
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
|
||||||
|
return value.IsNil()
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectPrompt resolves one prompt ID without selecting a profile or starting
|
||||||
|
// execution work.
|
||||||
//
|
//
|
||||||
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
// InspectPrompt requires a nonblank promptID and performs an exact,
|
||||||
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
// case-sensitive lookup in the captured configuration snapshot. A
|
||||||
// prompt selection. An empty version succeeds only when that ID has one
|
// namespace-scoped view qualifies slashless caller IDs before lookup; already
|
||||||
// version; a nonempty version selects one exact ID/version pair from the
|
// qualified IDs pass through.
|
||||||
// captured configuration snapshot.
|
|
||||||
//
|
//
|
||||||
// A successful result proves that the selected definition and any referenced
|
// A successful result proves that the selected definition and any referenced
|
||||||
// message content files were structurally loaded. Inputs are returned in
|
// message content files were structurally loaded. Inputs and captured,
|
||||||
// definition order. DefaultProfileID is declared metadata only and is not
|
// unrendered messages are returned in definition order. DefaultProfileID is
|
||||||
|
// declared metadata only and is not
|
||||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||||
// Schema path when declared but without loading or compiling that schema.
|
// Schema selection when declared but without loading or compiling schema files.
|
||||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||||
// the selected captured definition; its spelling, length,
|
// the selected captured definition; its spelling, length,
|
||||||
// encoding, algorithm, and security properties are not contracts.
|
// encoding, algorithm, and security properties are not contracts.
|
||||||
//
|
//
|
||||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
// This method does not return parsed templates, filesystem capabilities,
|
||||||
// rendered messages, or execution settings. It does not resolve a profile or
|
// schema file contents, rendered messages, or execution settings. It does not resolve a profile or
|
||||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||||
// contact a provider, or generate model output. The returned PromptInspection
|
// contact a provider, or generate model output. The returned PromptInspection
|
||||||
// and its input slice are caller-owned. Inspection does not reload configuration
|
// and its input slice are caller-owned. Inspection does not reload configuration
|
||||||
// or content files; construct another Engine to observe source changes.
|
// or content files; construct another Engine to observe source changes.
|
||||||
//
|
//
|
||||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||||
// matches ErrInvalidRequest. An absent exact ID or version matches
|
// matches ErrInvalidRequest. An absent exact ID matches ErrPromptNotFound and
|
||||||
// ErrPromptNotFound and not ErrPromptLoad. An ambiguous omitted version matches
|
// not ErrPromptLoad. Invalid definitions and resource failures reject
|
||||||
// ErrPromptLoad. Invalid definitions and resource failures reject construction
|
// construction instead. Cancellation during lookup preserves the context
|
||||||
// instead. Cancellation during lookup preserves the context error. InspectPrompt
|
// error. InspectPrompt returns no partial result on error.
|
||||||
// returns no partial result on error.
|
|
||||||
func (e *Engine) InspectPrompt(
|
func (e *Engine) InspectPrompt(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
promptID string,
|
promptID string,
|
||||||
promptVersion string,
|
|
||||||
) (*PromptInspection, error) {
|
) (*PromptInspection, error) {
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
}
|
}
|
||||||
|
|
||||||
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion)
|
inspection, err := e.internal.InspectPrompt(ctx, promptID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainPromptInspection(inspection), nil
|
return fromPromptResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||||
// starting execution work.
|
// starting execution work.
|
||||||
//
|
//
|
||||||
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
||||||
// in the captured effective configuration. It applies the framework timeout
|
// in the captured effective configuration. It reports the resolved profile
|
||||||
// baseline, selected
|
// resource without backend completion, execution defaults, or request overrides.
|
||||||
// backend, and then selected profile to EffectiveModelParams without a request
|
|
||||||
// override. BackendID is empty for an endpoint-only profile.
|
|
||||||
//
|
//
|
||||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
// APIKeyEnv in the returned resource is an environment-variable name, never its
|
||||||
// value. APIKeyRequired instead reports a direct credential requirement and is
|
// value. APIKeyRequired, when present and true, reports that a credential is
|
||||||
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
// required; an explicit profile environment source can also be present. InspectProfile neither derives
|
||||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
// absent or blank named environment variable is not an error.
|
// absent or blank named environment variable is not an error.
|
||||||
//
|
//
|
||||||
@@ -323,7 +424,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainProfileInspection(inspection), nil
|
return fromProfileResource(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectBackend selects a resolved backend by exact ID from the captured
|
||||||
|
// configuration, qualifying a slashless ID with the backend namespace. It
|
||||||
|
// reports configured resource fields and diagnostics, not credential values,
|
||||||
|
// derived diagnostic destinations, or runtime capacity occupancy. References
|
||||||
|
// are not resolved and no network request is made. The result and its mutable
|
||||||
|
// values are caller-owned. Source changes require a new engine.
|
||||||
|
//
|
||||||
|
// An invalid engine matches ErrInvalidConfig, an invalid ID matches
|
||||||
|
// ErrInvalidRequest, and an absent backend matches ErrBackendNotFound (not
|
||||||
|
// ErrInvalidRequest). Cancellation remains detectable via errors.Is. No
|
||||||
|
// partial result is returned on error.
|
||||||
|
func (e *Engine) InspectBackend(ctx context.Context, backendID string) (*BackendInspection, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
inspection, err := e.internal.InspectBackend(ctx, backendID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrBackendNotFound, err)
|
||||||
|
}
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromBackendResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
@@ -345,7 +471,35 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
// reported in the applicable operation category; no general errors.Is
|
// reported in the applicable operation category; no general errors.Is
|
||||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||||
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
||||||
|
// RunRequest.OperationTimeout and RunRequest.OnPrepared have no effect here.
|
||||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||||
|
prepared, err := e.prepareRequest(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return fromDomainPreparedRun(prepared), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateRequest reports whether req passes the same preparation path used by
|
||||||
|
// [Engine.Prepare] at the time of this call. A nil error means this concrete
|
||||||
|
// request passed preparation. It checks only this invocation and returns errors
|
||||||
|
// as encountered rather than aggregating them. All supplied inputs follow the
|
||||||
|
// same loading behavior as [Engine.Prepare], including inputs unused by the
|
||||||
|
// selected template branch. The method does not call an LLM or retain a
|
||||||
|
// prepared snapshot or executable handle, but preparation may still perform
|
||||||
|
// resource I/O and incur its associated time and memory costs.
|
||||||
|
//
|
||||||
|
// Its errors and context handling follow [Engine.Prepare], including an error
|
||||||
|
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
|
||||||
|
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
|
||||||
|
// when execution must use the snapshot produced by preparation.
|
||||||
|
// Execution-only controls do not start a timer or invoke a notification here.
|
||||||
|
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
|
||||||
|
_, err := e.prepareRequest(ctx, req)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.PreparedRun, error) {
|
||||||
ctx = observation.WithoutExecutionState(ctx)
|
ctx = observation.WithoutExecutionState(ctx)
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
@@ -360,15 +514,16 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainPreparedRun(prepared), nil
|
return prepared, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrepareExecution completely prepares a prompt request without calling the
|
// PrepareExecution completely prepares a prompt request without calling the
|
||||||
// configured LLMClient or reserving backend admission capacity. Validated
|
// configured LLMClient or reserving backend admission capacity. Validated
|
||||||
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||||
//
|
//
|
||||||
// The returned opaque handle is bound to this Engine and permits one
|
// The returned opaque handle is bound to this Engine's underlying execution
|
||||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
// owner, shared with its namespace views, and permits one [Engine.RunPrepared]
|
||||||
|
// invocation in total. Preparation freezes the selected sources,
|
||||||
// rendered messages, effective settings, inputs, provider structured-output
|
// rendered messages, effective settings, inputs, provider structured-output
|
||||||
// metadata, and validation resources needed by that invocation. The handle
|
// metadata, and validation resources needed by that invocation. The handle
|
||||||
// retains a direct RunRequest.APIKey only in private execution state;
|
// retains a direct RunRequest.APIKey only in private execution state;
|
||||||
@@ -381,6 +536,8 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
||||||
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
||||||
// privately for RunPrepared; preparation itself emits no events.
|
// privately for RunPrepared; preparation itself emits no events.
|
||||||
|
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
|
||||||
|
// starting a timer or invoking the notification during preparation.
|
||||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||||
ctx = observation.WithoutExecutionState(ctx)
|
ctx = observation.WithoutExecutionState(ctx)
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
@@ -411,6 +568,9 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
|||||||
// returns the final failed validation result with cumulative usage and actual
|
// returns the final failed validation result with cumulative usage and actual
|
||||||
// repair attempts. An inability to generate or validate returns an error and
|
// repair attempts. An inability to generate or validate returns an error and
|
||||||
// no partial result.
|
// no partial result.
|
||||||
|
// RunRequest.OperationTimeout selects a cooperative whole-operation deadline;
|
||||||
|
// RunRequest.OnPrepared selects conditional notification before generation.
|
||||||
|
// The outcome and timeout attribution are fixed before terminal observation.
|
||||||
//
|
//
|
||||||
// The built-in client retries eligible provider statuses using the effective
|
// The built-in client retries eligible provider statuses using the effective
|
||||||
// Config or request budget. With R additional transient attempts and P repair
|
// Config or request budget. With R additional transient attempts and P repair
|
||||||
@@ -442,18 +602,30 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
|||||||
// result on error.
|
// result on error.
|
||||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||||
ctx = observation.WithoutExecutionState(ctx)
|
ctx = observation.WithoutExecutionState(ctx)
|
||||||
|
scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
|
||||||
|
defer scope.Close()
|
||||||
|
ctx = scope.Context
|
||||||
if req.Observer == nil {
|
if req.Observer == nil {
|
||||||
return e.run(ctx, req)
|
result, err := e.run(ctx, req)
|
||||||
|
err = mapPublicError(scope.Finalize(err))
|
||||||
|
scope.Close()
|
||||||
|
return result, err
|
||||||
}
|
}
|
||||||
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
||||||
|
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
|
||||||
|
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
|
||||||
|
}
|
||||||
recorder.Start()
|
recorder.Start()
|
||||||
if idErr != nil {
|
if idErr != nil {
|
||||||
err := fmt.Errorf("failed to create run id: %w", idErr)
|
err := mapPublicError(scope.Finalize(fmt.Errorf("failed to create run id: %w", idErr)))
|
||||||
|
scope.Close()
|
||||||
terminal := observedFinish(nil, err)
|
terminal := observedFinish(nil, err)
|
||||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure})
|
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure})
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
result, err := e.run(observation.WithRecorder(ctx, recorder), req)
|
result, err := e.run(observation.WithRecorder(ctx, recorder), req)
|
||||||
|
err = mapPublicError(scope.Finalize(err))
|
||||||
|
scope.Close()
|
||||||
terminal := observedFinish(result, err)
|
terminal := observedFinish(result, err)
|
||||||
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure,
|
recorder.Finish(observation.FinishData{Outcome: terminal.Outcome, Failure: terminal.Failure,
|
||||||
StatusCode: terminal.StatusCode, RetriesExhausted: terminal.RetriesExhausted})
|
StatusCode: terminal.StatusCode, RetriesExhausted: terminal.RetriesExhausted})
|
||||||
@@ -472,7 +644,7 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
|
|
||||||
result, err := e.internal.Run(ctx, domainReq)
|
result, err := e.internal.Run(ctx, domainReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, err
|
||||||
}
|
}
|
||||||
return fromDomainRunResult(result), nil
|
return fromDomainRunResult(result), nil
|
||||||
}
|
}
|
||||||
@@ -480,6 +652,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
// RunPrepared atomically claims and executes a handle created by
|
// RunPrepared atomically claims and executes a handle created by
|
||||||
// [Engine.PrepareExecution].
|
// [Engine.PrepareExecution].
|
||||||
//
|
//
|
||||||
|
// The originating Engine and namespace views sharing its underlying owner may
|
||||||
|
// claim the handle. Execution uses its captured selections without applying the
|
||||||
|
// invoking view's namespaces. An Engine constructed separately by [NewEngine]
|
||||||
|
// is a foreign owner.
|
||||||
|
//
|
||||||
// A valid owning-Engine invocation consumes the handle's one attempt before
|
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||||
// credential revalidation, backend admission, generation, or validation.
|
// credential revalidation, backend admission, generation, or validation.
|
||||||
// Cancellation, capacity rejection, generation failure, operational
|
// Cancellation, capacity rejection, generation failure, operational
|
||||||
@@ -511,6 +688,9 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
|
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
|
||||||
// Callbacks run outside the handle and capacity locks; panics propagate and
|
// Callbacks run outside the handle and capacity locks; panics propagate and
|
||||||
// do not promise a terminal event.
|
// do not promise a terminal event.
|
||||||
|
// The retained RunRequest.OperationTimeout starts after claim; the retained
|
||||||
|
// RunRequest.OnPrepared is notified only if execution reaches its prepared
|
||||||
|
// point. Final timeout attribution precedes terminal observation.
|
||||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||||
ctx = observation.WithoutExecutionState(ctx)
|
ctx = observation.WithoutExecutionState(ctx)
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
|
|||||||
52
errors.go
52
errors.go
@@ -5,18 +5,56 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// publicOperationError keeps classification separate from inspectable causes.
|
||||||
|
// Cache wording so formatting nested operations does not repeatedly rebuild it.
|
||||||
|
type publicOperationError struct {
|
||||||
|
owner error
|
||||||
|
cause error
|
||||||
|
message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPublicOperationError(owner, cause error) *publicOperationError {
|
||||||
|
return &publicOperationError{owner: owner, cause: cause, message: fmt.Sprintf("%v: %v", owner, cause)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *publicOperationError) Error() string { return e.message }
|
||||||
|
func (e *publicOperationError) Unwrap() error { return e.cause }
|
||||||
|
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
|
||||||
|
|
||||||
func mapPublicError(err error) error {
|
func mapPublicError(err error) error {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var operation *usecase.OperationError
|
||||||
|
if errors.As(err, &operation) && operation != nil {
|
||||||
|
owner := publicErrorFor(operation.Category)
|
||||||
|
if owner != nil {
|
||||||
|
// Project the underlying specialized error separately, retaining both
|
||||||
|
// the outer operation and the original cause for Is/As callers.
|
||||||
|
mapped := mapPublicError(operation.Cause)
|
||||||
|
if errors.Is(operation.Cause, llm.ErrTransport) {
|
||||||
|
mapped = errors.Join(ErrTransport, mapped)
|
||||||
|
}
|
||||||
|
if errors.Is(operation.Cause, llm.ErrMalformedResponse) {
|
||||||
|
mapped = errors.Join(ErrMalformedResponse, mapped)
|
||||||
|
}
|
||||||
|
// The projection already retains ordinary causes; adding the original
|
||||||
|
// again would duplicate its entire subtree at each enclosing operation.
|
||||||
|
return newPublicOperationError(owner, mapped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mapUnownedPublicError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapUnownedPublicError(err error) error {
|
||||||
var internalCapacityError *usecase.CapacityError
|
var internalCapacityError *usecase.CapacityError
|
||||||
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
@@ -62,6 +100,8 @@ func mapPublicError(err error) error {
|
|||||||
|
|
||||||
func publicErrorFor(err error) error {
|
func publicErrorFor(err error) error {
|
||||||
switch {
|
switch {
|
||||||
|
case errors.Is(err, usecase.ErrOperationTimeout):
|
||||||
|
return ErrOperationTimeout
|
||||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
||||||
return ErrPromptNotFound
|
return ErrPromptNotFound
|
||||||
case errors.Is(err, profile.ErrProfileNotFound):
|
case errors.Is(err, profile.ErrProfileNotFound):
|
||||||
|
|||||||
@@ -6,10 +6,29 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestConstructionErrorPreservesJoinedCause(t *testing.T) {
|
||||||
|
operational := errors.New("independent operational failure")
|
||||||
|
cause := errors.Join(operational, context.Canceled)
|
||||||
|
engine, err := NewEngine(Config{}, optionFunc(func(*engineOptions) error {
|
||||||
|
return cause
|
||||||
|
}))
|
||||||
|
if engine != nil || err == nil {
|
||||||
|
t.Fatalf("construction = (%#v, %v), want failure", engine, err)
|
||||||
|
}
|
||||||
|
if unwrapped := errors.Unwrap(err); unwrapped != cause {
|
||||||
|
t.Fatalf("construction cause = %v, want the unchanged joined cause", unwrapped)
|
||||||
|
}
|
||||||
|
for _, target := range []error{ErrInvalidConfig, operational, context.Canceled} {
|
||||||
|
if !errors.Is(err, target) {
|
||||||
|
t.Errorf("construction error = %v, want identity %v", err, target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
||||||
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)
|
internalErr := fmt.Errorf("%w: %w", usecase.ErrLLMGenerate, context.Canceled)
|
||||||
|
|
||||||
@@ -80,3 +99,12 @@ func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
|||||||
t.Fatalf("request encoding category = %v", err)
|
t.Fatalf("request encoding category = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorRetainsOwnerWithSpecializedCapacity(t *testing.T) {
|
||||||
|
internal := &usecase.CapacityError{BackendID: "limited"}
|
||||||
|
err := mapPublicError(&usecase.OperationError{Category: usecase.ErrLLMGenerate, Cause: internal})
|
||||||
|
var capacity *CapacityError
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) || !errors.As(err, &capacity) || capacity.BackendID != "limited" || !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("mapped owned capacity = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -9,9 +9,10 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed prompts/*.yaml schemas/*.json
|
//go:embed prompts/*.yaml schemas/*.json
|
||||||
@@ -75,6 +76,12 @@ type summary struct {
|
|||||||
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
AssessmentAttempts uint64 `json:"assessment_attempts"`
|
||||||
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
AssessmentTransientRetries uint64 `json:"assessment_transient_retries"`
|
||||||
AssessmentOutcome string `json:"assessment_outcome"`
|
AssessmentOutcome string `json:"assessment_outcome"`
|
||||||
|
CoordinationOutcome promptkit.CoordinationOutcome `json:"coordination_outcome"`
|
||||||
|
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
|
||||||
|
GroupMembers int `json:"group_members"`
|
||||||
|
GroupAttempts uint64 `json:"group_attempts"`
|
||||||
|
PreparedMessages int `json:"prepared_messages"`
|
||||||
|
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func run() (summary, error) {
|
func run() (summary, error) {
|
||||||
@@ -83,27 +90,35 @@ func run() (summary, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
|
for i := range inputs {
|
||||||
|
inputs[i].Destination = configkit.Keys("promptkit")
|
||||||
|
}
|
||||||
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
||||||
|
filesystem := configkit.NewFilesystem("example schemas", assets)
|
||||||
|
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||||
|
backend.ID = backendID
|
||||||
inputs = append(inputs,
|
inputs = append(inputs,
|
||||||
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
|
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
|
||||||
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
|
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||||
})},
|
backend,
|
||||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
|
})),
|
||||||
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
|
promptkit.SettingsInput(promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
|
||||||
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
|
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
|
||||||
})},
|
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
|
||||||
|
})),
|
||||||
)
|
)
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
||||||
HTTPClient: &http.Client{Transport: transport},
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
}, promptkit.WithSchemaFS(assets, "schemas"))
|
}, promptkit.WithBundledOpenRouter(false)) // Keep the example's configuration self-contained.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var proposalLog eventLog
|
var proposalLog eventLog
|
||||||
|
var executions promptkit.ExecutionGroup
|
||||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, Observer: proposalLog.record,
|
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return summary{}, err
|
||||||
@@ -116,18 +131,23 @@ func run() (summary, error) {
|
|||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
var assessmentLog eventLog
|
var assessmentLog eventLog
|
||||||
assessment, err := engine.Run(context.Background(), promptkit.RunRequest{
|
var preparedDetails promptkit.PreparedRun
|
||||||
PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(proposal.RawOutput)}, Observer: assessmentLog.record,
|
decision := "rejected"
|
||||||
})
|
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
|
||||||
|
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
|
||||||
|
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
if turn.Result == nil {
|
||||||
|
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
|
||||||
|
}
|
||||||
|
if !turn.Result.Validation.IsValid {
|
||||||
|
return nil, fmt.Errorf("follow-up failed structural validation")
|
||||||
|
}
|
||||||
|
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !assessment.Validation.IsValid {
|
if !assessment.Validation.IsValid {
|
||||||
return summary{}, fmt.Errorf("assessment failed structural validation")
|
return nil, fmt.Errorf("assessment failed structural validation")
|
||||||
}
|
|
||||||
assessmentTerminal, err := assessmentLog.terminal()
|
|
||||||
if err != nil {
|
|
||||||
return summary{}, err
|
|
||||||
}
|
}
|
||||||
var proposalValue struct {
|
var proposalValue struct {
|
||||||
Risk string `json:"risk"`
|
Risk string `json:"risk"`
|
||||||
@@ -135,16 +155,32 @@ func run() (summary, error) {
|
|||||||
var assessmentValue struct {
|
var assessmentValue struct {
|
||||||
Recommendation string `json:"recommendation"`
|
Recommendation string `json:"recommendation"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal([]byte(proposal.RawOutput), &proposalValue); err != nil {
|
if err := json.Unmarshal([]byte(turn.Result.RawOutput), &proposalValue); err != nil {
|
||||||
return summary{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
if err := json.Unmarshal([]byte(assessment.RawOutput), &assessmentValue); err != nil {
|
||||||
return summary{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
decision := "rejected"
|
|
||||||
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
if proposalValue.Risk == "low" && assessmentValue.Recommendation == "accept" {
|
||||||
decision = "accepted"
|
decision = "accepted"
|
||||||
}
|
}
|
||||||
|
return nil, nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return summary{}, err
|
||||||
|
}
|
||||||
|
exhausted, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{Start: proposal.Exchange, Next: func(context.Context, promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
|
return &promptkit.RunRequest{PromptID: "example.proposal"}, nil
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
return summary{}, err
|
||||||
|
}
|
||||||
|
assessmentTerminal, err := assessmentLog.terminal()
|
||||||
|
if err != nil {
|
||||||
|
return summary{}, err
|
||||||
|
}
|
||||||
|
group := executions.Snapshot()
|
||||||
return summary{
|
return summary{
|
||||||
ProposalAttempts: proposalTerminal.Attempts,
|
ProposalAttempts: proposalTerminal.Attempts,
|
||||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||||
@@ -152,17 +188,26 @@ func run() (summary, error) {
|
|||||||
AssessmentAttempts: assessmentTerminal.Attempts,
|
AssessmentAttempts: assessmentTerminal.Attempts,
|
||||||
AssessmentTransientRetries: assessmentTerminal.TransientRetries,
|
AssessmentTransientRetries: assessmentTerminal.TransientRetries,
|
||||||
AssessmentOutcome: decision,
|
AssessmentOutcome: decision,
|
||||||
|
CoordinationOutcome: coordination.Outcome,
|
||||||
|
ExhaustedOutcome: exhausted.Outcome,
|
||||||
|
GroupMembers: len(group.Members),
|
||||||
|
GroupAttempts: group.Attempts,
|
||||||
|
PreparedMessages: len(preparedDetails.Messages),
|
||||||
|
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
value, err := run()
|
if err := execute(os.Stdout); err != nil {
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if err := json.NewEncoder(os.Stdout).Encode(value); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func execute(out io.Writer) error {
|
||||||
|
value, err := run()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return json.NewEncoder(out).Encode(value)
|
||||||
|
}
|
||||||
|
|||||||
36
examples/go-library/observe/main_test.go
Normal file
36
examples/go-library/observe/main_test.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestObserveExample(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := execute(&output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got struct {
|
||||||
|
ProposalAttempts *int `json:"proposal_attempts"`
|
||||||
|
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||||
|
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||||
|
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||||
|
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||||
|
Outcome string `json:"assessment_outcome"`
|
||||||
|
CoordinationOutcome string `json:"coordination_outcome"`
|
||||||
|
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||||
|
GroupMembers int `json:"group_members"`
|
||||||
|
GroupAttempts int `json:"group_attempts"`
|
||||||
|
PreparedMessages int `json:"prepared_messages"`
|
||||||
|
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.ProposalAttempts == nil || *got.ProposalAttempts != 2 || got.ProposalRetries == nil || *got.ProposalRetries != 1 ||
|
||||||
|
got.UsageComplete == nil || *got.UsageComplete || got.AssessmentAttempts == nil || *got.AssessmentAttempts != 1 ||
|
||||||
|
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
|
||||||
|
t.Fatalf("unexpected observation summary: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.assessment:
|
- id: example.assessment
|
||||||
"1.0.0":
|
|
||||||
default_profile: assessment-profile
|
default_profile: assessment-profile
|
||||||
description: Assess a structured proposal independently of schema validity.
|
description: Assess a structured proposal independently of schema validity.
|
||||||
inputs:
|
inputs:
|
||||||
@@ -15,4 +14,12 @@ prompts:
|
|||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
schema_path: assessment.schema.json
|
repair_attempts: 0
|
||||||
|
schema:
|
||||||
|
$schema: https://json-schema.org/draft/2020-12/schema
|
||||||
|
type: object
|
||||||
|
required: [recommendation, confidence]
|
||||||
|
properties:
|
||||||
|
recommendation: {type: string}
|
||||||
|
confidence: {type: number}
|
||||||
|
additionalProperties: false
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.proposal:
|
- id: example.proposal
|
||||||
"1.0.0":
|
|
||||||
default_profile: proposal-profile
|
default_profile: proposal-profile
|
||||||
description: Propose a bounded improvement from a synthetic idea.
|
description: Propose a bounded improvement from a synthetic idea.
|
||||||
inputs:
|
inputs:
|
||||||
@@ -15,4 +14,5 @@ prompts:
|
|||||||
output:
|
output:
|
||||||
format: json
|
format: json
|
||||||
validation_mode: json_schema
|
validation_mode: json_schema
|
||||||
|
repair_attempts: 0
|
||||||
schema_path: proposal.schema.json
|
schema_path: proposal.schema.json
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
backends:
|
||||||
|
- id: offline
|
||||||
|
endpoint: https://review.example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
- id: audit
|
||||||
|
backend: review/offline
|
||||||
|
model: review-model
|
||||||
|
prompts:
|
||||||
|
- id: check
|
||||||
|
default_profile: review/audit
|
||||||
|
description: Inspect a review item without contacting a model provider.
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review the supplied synthetic example.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: fragments/resources.yaml
|
||||||
|
required: true
|
||||||
|
- path: fragments/prepare.yaml
|
||||||
|
destination: [prompts]
|
||||||
|
placement: sequence-item
|
||||||
|
required: true
|
||||||
|
release_version: "1.4.0"
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
id: prepare
|
||||||
|
default_profile: support/fast
|
||||||
|
description: Prepare a prompt without contacting a model provider.
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: true
|
||||||
|
content_type: text/plain
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: Summarize the note in one sentence.
|
||||||
|
- role: user
|
||||||
|
content_file: ../messages/note.md
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: basic
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
backends:
|
||||||
|
- id: offline
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
- id: base
|
||||||
|
backend: support/offline
|
||||||
|
model: offline-model
|
||||||
|
timeout_seconds: 90
|
||||||
|
- id: fast
|
||||||
|
extends: {id: base}
|
||||||
|
timeout_seconds: 60
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{{input "note"}}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
# A later input replaces the profile with an inherited variant.
|
||||||
|
profiles:
|
||||||
|
- id: fast
|
||||||
|
extends: {id: base}
|
||||||
|
timeout_seconds: 30
|
||||||
@@ -4,74 +4,174 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type summary struct {
|
type summary struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version"`
|
|
||||||
SelectedProfile string `json:"selected_profile"`
|
SelectedProfile string `json:"selected_profile"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
MessageCount int `json:"message_count"`
|
MessageCount int `json:"message_count"`
|
||||||
|
MessageContentHash string `json:"message_content_hash"`
|
||||||
|
ProfileTimeout int `json:"profile_timeout_seconds"`
|
||||||
|
Namespaces []string `json:"namespaces"`
|
||||||
|
Collections []collectionSummary `json:"collections"`
|
||||||
|
ScopedPrompts []string `json:"scoped_prompts"`
|
||||||
|
ReviewPrompts []string `json:"review_prompts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type collectionSummary struct {
|
||||||
|
Namespace string `json:"namespace"`
|
||||||
|
ReleaseVersion *string `json:"release_version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type applicationSettings struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
if err := run(context.Background(), "examples/go-library/prepare", os.Stdout); err != nil {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
fmt.Fprintln(os.Stderr, err)
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
os.Exit(1)
|
||||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
}
|
||||||
})}}}},
|
}
|
||||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/prepare/prompt.yaml")},
|
|
||||||
}})
|
func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||||
|
support, err := promptkit.CollectionInput(
|
||||||
|
"support",
|
||||||
|
configkit.File(filepath.Join(exampleDir, "collections", "support", "collection.yaml")),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
return err
|
||||||
|
}
|
||||||
|
review, err := promptkit.CollectionInput(
|
||||||
|
"review",
|
||||||
|
configkit.File(filepath.Join(exampleDir, "collections", "review", "collection.yaml")),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
override, err := promptkit.CollectionInput(
|
||||||
|
"support",
|
||||||
|
configkit.File(filepath.Join(exampleDir, "collections", "support", "operator-overrides.yaml")),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[applicationSettings]{}, func(value *applicationSettings) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[applicationSettings]{
|
||||||
|
Definition: definition,
|
||||||
|
Underlays: []configkit.Underlay{
|
||||||
|
{Inputs: []configkit.Input{support, review}},
|
||||||
|
{Inputs: []configkit.Input{override}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||||
|
Prompt: "support", Profile: "support", Backend: "support",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
return err
|
||||||
}
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
if _, found := view.Explain(configkit.Keys("promptkit", "collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
|
||||||
exit(fmt.Errorf("configured profile model was not found"))
|
return fmt.Errorf("collection profile override was not found")
|
||||||
}
|
}
|
||||||
requirements, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
|
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||||
PromptID: "example.prepare",
|
PromptID: "prepare",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
return err
|
||||||
}
|
}
|
||||||
if !requirements.Complete || len(requirements.Inputs) != 1 ||
|
if !requirements.Complete || len(requirements.Inputs) != 1 ||
|
||||||
requirements.Inputs[0].Name != "note" ||
|
requirements.Inputs[0].Name != "note" ||
|
||||||
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
|
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
|
||||||
exit(fmt.Errorf("required note input was not discovered before preparation"))
|
return fmt.Errorf("required note input was not discovered before preparation")
|
||||||
}
|
}
|
||||||
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
request := promptkit.RunRequest{
|
||||||
PromptID: "example.prepare",
|
PromptID: "prepare",
|
||||||
|
ProfileID: "fast",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"note": promptkit.Inline("Ada finished the migration review."),
|
"note": promptkit.Inline("Ada finished the migration review."),
|
||||||
},
|
},
|
||||||
|
}
|
||||||
|
if err := scoped.ValidateRequest(ctx, request); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
prepared, err := scoped.Prepare(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
profile, err := scoped.InspectProfile(ctx, "fast")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 30 {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"later profile override was not applied: timeout is %v",
|
||||||
|
profile.TimeoutSeconds,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
namespaces, err := scoped.ListNamespaces(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
collections, err := scoped.ListCollections(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
scopedPrompts, err := scoped.ListPrompts(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reviewPrompts, err := scoped.ListPrompts(ctx, promptkit.NamespaceFilter{
|
||||||
|
Mode: promptkit.NamespaceNamed, Name: "review",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
return err
|
||||||
|
}
|
||||||
|
scopedPromptIDs := make([]string, len(scopedPrompts))
|
||||||
|
for index := range scopedPrompts {
|
||||||
|
scopedPromptIDs[index] = scopedPrompts[index].ID
|
||||||
|
}
|
||||||
|
reviewPromptIDs := make([]string, len(reviewPrompts))
|
||||||
|
for index := range reviewPrompts {
|
||||||
|
reviewPromptIDs[index] = reviewPrompts[index].ID
|
||||||
|
}
|
||||||
|
collectionSummaries := make([]collectionSummary, len(collections))
|
||||||
|
for index := range collections {
|
||||||
|
collectionSummaries[index] = collectionSummary{
|
||||||
|
Namespace: collections[index].Namespace, ReleaseVersion: collections[index].ReleaseVersion,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
encoder := json.NewEncoder(out)
|
||||||
encoder.SetIndent("", " ")
|
encoder.SetIndent("", " ")
|
||||||
if err := encoder.Encode(summary{
|
return encoder.Encode(summary{
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
|
||||||
SelectedProfile: prepared.SelectedProfileID,
|
SelectedProfile: prepared.SelectedProfileID,
|
||||||
Model: prepared.EffectiveModelParams.Model,
|
Model: prepared.EffectiveModelParams.Model,
|
||||||
MessageCount: len(prepared.Messages),
|
MessageCount: len(prepared.Messages),
|
||||||
}); err != nil {
|
MessageContentHash: prepared.MessageContentHash,
|
||||||
exit(err)
|
ProfileTimeout: *profile.TimeoutSeconds,
|
||||||
}
|
Namespaces: namespaces,
|
||||||
}
|
Collections: collectionSummaries,
|
||||||
|
ScopedPrompts: scopedPromptIDs,
|
||||||
func exit(err error) {
|
ReviewPrompts: reviewPromptIDs,
|
||||||
fmt.Fprintln(os.Stderr, err)
|
})
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
|||||||
46
examples/go-library/prepare/main_test.go
Normal file
46
examples/go-library/prepare/main_test.go
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPrepareExample(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := run(t.Context(), ".", &output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got struct {
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
Profile string `json:"selected_profile"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
MessageCount int `json:"message_count"`
|
||||||
|
MessageContentHash string `json:"message_content_hash"`
|
||||||
|
Timeout int `json:"profile_timeout_seconds"`
|
||||||
|
Namespaces []string `json:"namespaces"`
|
||||||
|
Collections []map[string]json.RawMessage `json:"collections"`
|
||||||
|
ScopedPrompts []string `json:"scoped_prompts"`
|
||||||
|
ReviewPrompts []string `json:"review_prompts"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.PromptID != "support/prepare" || got.Profile != "support/fast" || got.Model != "offline-model" ||
|
||||||
|
got.MessageCount <= 0 || got.MessageContentHash == "" || got.Timeout != 30 || !reflect.DeepEqual(got.Namespaces, []string{"review", "support"}) ||
|
||||||
|
!reflect.DeepEqual(got.ScopedPrompts, []string{"support/prepare"}) || !reflect.DeepEqual(got.ReviewPrompts, []string{"review/check"}) || len(got.Collections) != 2 {
|
||||||
|
t.Fatalf("unexpected preparation summary: %+v", got)
|
||||||
|
}
|
||||||
|
for index, want := range []struct{ namespace, release string }{{"review", ""}, {"support", "1.4.0"}} {
|
||||||
|
var namespace string
|
||||||
|
if err := json.Unmarshal(got.Collections[index]["namespace"], &namespace); err != nil || namespace != want.namespace {
|
||||||
|
t.Fatalf("collection namespace = %q, %v", namespace, err)
|
||||||
|
}
|
||||||
|
value, present := got.Collections[index]["release_version"]
|
||||||
|
var release *string
|
||||||
|
if !present || json.Unmarshal(value, &release) != nil || (index == 0 && release != nil) || (index == 1 && (release == nil || *release != want.release)) {
|
||||||
|
t.Fatalf("collection release = %s, present %t; want %q", value, present, want.release)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
prompts:
|
|
||||||
example.prepare:
|
|
||||||
"1.0.0":
|
|
||||||
default_profile: offline-example
|
|
||||||
description: Prepare a prompt without contacting a model provider.
|
|
||||||
inputs:
|
|
||||||
- name: note
|
|
||||||
required: true
|
|
||||||
content_type: text/plain
|
|
||||||
messages:
|
|
||||||
- role: system
|
|
||||||
content: Summarize the note in one sentence.
|
|
||||||
- role: user
|
|
||||||
content: '{{input "note"}}'
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: basic
|
|
||||||
@@ -4,10 +4,12 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type deterministicClient struct{}
|
type deterministicClient struct{}
|
||||||
@@ -39,41 +41,41 @@ type summary struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
if err := run(context.Background(), "examples/go-library/run", os.Stdout); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
|
||||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
|
||||||
})}}}},
|
}))}}},
|
||||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
|
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml")), Destination: configkit.Keys("promptkit")},
|
||||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(deterministicClient{})) // Use only this example's offline configuration.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||||
PromptID: "example.run",
|
PromptID: "example.run",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"note": promptkit.Inline("Ada finished the migration review."),
|
"note": promptkit.Inline("Ada finished the migration review."),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
encoder := json.NewEncoder(out)
|
||||||
encoder.SetIndent("", " ")
|
encoder.SetIndent("", " ")
|
||||||
if err := encoder.Encode(summary{
|
return encoder.Encode(summary{
|
||||||
Output: result.RawOutput,
|
Output: result.RawOutput,
|
||||||
ValidationStatus: result.Validation.Status,
|
ValidationStatus: result.Validation.Status,
|
||||||
IsValid: result.Validation.IsValid,
|
IsValid: result.Validation.IsValid,
|
||||||
Model: result.ModelName,
|
Model: result.ModelName,
|
||||||
TotalTokens: result.Usage.TotalTokens,
|
TotalTokens: result.Usage.TotalTokens,
|
||||||
}); err != nil {
|
})
|
||||||
exit(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func exit(err error) {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
|||||||
28
examples/go-library/run/main_test.go
Normal file
28
examples/go-library/run/main_test.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunExample(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := run(t.Context(), ".", &output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got struct {
|
||||||
|
Output string `json:"output"`
|
||||||
|
Valid *bool `json:"is_valid"`
|
||||||
|
Status string `json:"validation_status"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Tokens *int `json:"total_tokens"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Output != "Ada finished the migration review." || got.Valid == nil || !*got.Valid ||
|
||||||
|
got.Status != "passed" || got.Model != "offline-model" || got.Tokens == nil || *got.Tokens != 18 {
|
||||||
|
t.Fatalf("unexpected execution summary: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.run:
|
- id: example.run
|
||||||
"1.0.0":
|
|
||||||
default_profile: offline-example
|
default_profile: offline-example
|
||||||
description: Run a prompt with a deterministic injected model client.
|
description: Run a prompt with a deterministic injected model client.
|
||||||
inputs:
|
inputs:
|
||||||
|
|||||||
47
exchange.go
Normal file
47
exchange.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import "go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
|
||||||
|
const exchangeString = "promptkit.Exchange{opaque}"
|
||||||
|
|
||||||
|
// Exchange is an immutable, reusable in-process snapshot of a completed
|
||||||
|
// conversation. It is not a persistence or restart format. Different engines
|
||||||
|
// can continue it, and independent branches do not alter one another.
|
||||||
|
//
|
||||||
|
// History preserves the effective request's message content, roles, and cache
|
||||||
|
// metadata, followed by the exact final [RunResult.RawOutput] as an assistant
|
||||||
|
// message. Intermediate structural-repair candidates and corrective instructions
|
||||||
|
// are excluded. A continuation extends that logical history according to
|
||||||
|
// [RunRequest.ContinueFrom], without changing its source exchange.
|
||||||
|
type Exchange struct{ internal *domain.Exchange }
|
||||||
|
|
||||||
|
// ExchangeDetails contains a detached copy of an exchange's logical history.
|
||||||
|
type ExchangeDetails struct {
|
||||||
|
// RunID identifies the execution that produced this exchange.
|
||||||
|
RunID string `json:"run_id"`
|
||||||
|
// ParentRunID identifies the immediate source execution, or is empty.
|
||||||
|
ParentRunID string `json:"parent_run_id,omitempty"`
|
||||||
|
Messages []RenderedMessage `json:"messages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details returns detached history, or zero details for a nil or empty handle.
|
||||||
|
// Messages include caller and model content and require appropriate handling.
|
||||||
|
// Inspection uses only retained state, without rendering, filesystem reads, or
|
||||||
|
// provider calls. Mutating returned messages or cache metadata cannot affect
|
||||||
|
// this exchange, another branch, or subsequent Details calls.
|
||||||
|
func (e *Exchange) Details() ExchangeDetails {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return ExchangeDetails{}
|
||||||
|
}
|
||||||
|
messages := make([]RenderedMessage, e.internal.MessageCount())
|
||||||
|
e.internal.VisitMessages(func(index int, message domain.RenderedMessage) {
|
||||||
|
messages[index] = RenderedMessage{Role: message.Role, Content: message.Content, CacheControl: fromDomainCacheControl(message.CacheControl)}
|
||||||
|
})
|
||||||
|
return ExchangeDetails{RunID: e.internal.RunID(), ParentRunID: e.internal.ParentRunID(), Messages: messages}
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns an opaque representation without conversation content.
|
||||||
|
func (e Exchange) String() string { return exchangeString }
|
||||||
|
|
||||||
|
// GoString returns an opaque representation without conversation content.
|
||||||
|
func (e Exchange) GoString() string { return exchangeString }
|
||||||
113
exchange_capture_contract_test.go
Normal file
113
exchange_capture_contract_test.go
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExchangeCaptureAndPreparedContinuation(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
ctx := context.Background()
|
||||||
|
plain, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil || plain.Exchange != nil {
|
||||||
|
t.Fatalf("plain run = %#v, %v", plain, err)
|
||||||
|
}
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil || initial.Exchange == nil {
|
||||||
|
t.Fatalf("captured run = %#v, %v", initial, err)
|
||||||
|
}
|
||||||
|
want := []promptkit.RenderedMessage{{Role: promptkit.RoleUser, Content: "question"}, {Role: promptkit.RoleAssistant, Content: "answer"}}
|
||||||
|
if details := initial.Exchange.Details(); details.RunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) {
|
||||||
|
t.Fatalf("details = %#v", details)
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(initial)
|
||||||
|
if err != nil || strings.Contains(string(encoded), "exchange") || strings.Contains(string(encoded), "question") {
|
||||||
|
t.Fatalf("result JSON = %s, %v", encoded, err)
|
||||||
|
}
|
||||||
|
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if handle.Details().ParentRunID != initial.RunID {
|
||||||
|
t.Fatal("prepared ancestry missing")
|
||||||
|
}
|
||||||
|
continued, err := engine.RunPrepared(ctx, handle)
|
||||||
|
if err != nil || continued.Exchange == nil || continued.ParentRunID != initial.RunID {
|
||||||
|
t.Fatalf("continued = %#v, %v", continued, err)
|
||||||
|
}
|
||||||
|
want = append(want, promptkit.RenderedMessage{Role: promptkit.RoleUser, Content: "question"}, promptkit.RenderedMessage{Role: promptkit.RoleAssistant, Content: "answer"})
|
||||||
|
if details := continued.Exchange.Details(); details.ParentRunID != initial.RunID || !reflect.DeepEqual(details.Messages, want) {
|
||||||
|
t.Fatalf("continued details = %#v", details)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExchangeCapturesInvalidFinalOutput(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{{Content: ""}, {Content: " "}}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true,
|
||||||
|
Validation: &promptkit.OutputContractInput{Format: promptkit.FormatText, ValidationMode: promptkit.ValidationBasic, RepairAttempts: new(1)}})
|
||||||
|
if err != nil || result == nil || result.Validation.Status != promptkit.ValidationFailed || result.Exchange == nil {
|
||||||
|
t.Fatalf("invalid result = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
if messages := result.Exchange.Details().Messages; len(messages) != 2 || messages[1].Content != " " {
|
||||||
|
t.Fatalf("history = %#v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExchangeFailuresLeaveSourceReusable(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
original := initial.Exchange.Details()
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange}
|
||||||
|
handle, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
handle.Discard()
|
||||||
|
if result, err := engine.RunPrepared(ctx, handle); result != nil || !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("discarded = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
client.err = errors.New("provider failure")
|
||||||
|
if result, err := engine.Run(ctx, request); result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
t.Fatalf("failure = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
client.err = nil
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
if err != nil || result.Exchange == nil || !reflect.DeepEqual(initial.Exchange.Details(), original) {
|
||||||
|
t.Fatalf("reused source = %#v, %v", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyExchangeRejectedBeforePreparation(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
|
||||||
|
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
|
||||||
|
request := promptkit.RunRequest{PromptID: "missing", ContinueFrom: &promptkit.Exchange{}}
|
||||||
|
ctx := context.Background()
|
||||||
|
if _, err := engine.Prepare(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("prepare error = %v", err)
|
||||||
|
}
|
||||||
|
if err := engine.ValidateRequest(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("validate error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.PrepareExecution(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("prepare execution error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(ctx, request); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("run error = %v", err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatal("invalid exchange reached the model")
|
||||||
|
}
|
||||||
|
}
|
||||||
38
exchange_test.go
Normal file
38
exchange_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExchangeDetailsAndRequestConversion(t *testing.T) {
|
||||||
|
var nilExchange *Exchange
|
||||||
|
if got := nilExchange.Details(); got.RunID != "" || got.Messages != nil {
|
||||||
|
t.Fatalf("nil details = %#v", got)
|
||||||
|
}
|
||||||
|
if _, err := toDomainRunRequest(RunRequest{ContinueFrom: &Exchange{}}); err == nil {
|
||||||
|
t.Fatal("empty exchange accepted")
|
||||||
|
}
|
||||||
|
history := domain.CaptureExchange(nil, "source", []domain.RenderedMessage{{Role: domain.RoleUser, Content: "private", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}}}, "answer")
|
||||||
|
handle := &Exchange{internal: history}
|
||||||
|
request, err := toDomainRunRequest(RunRequest{ContinueFrom: handle})
|
||||||
|
if err != nil || request.ContinueFrom != history || !request.CaptureExchange {
|
||||||
|
t.Fatalf("request = %#v, error = %v", request, err)
|
||||||
|
}
|
||||||
|
details := handle.Details()
|
||||||
|
details.Messages[0].CacheControl.TTL = "changed"
|
||||||
|
if handle.Details().Messages[0].CacheControl.TTL != "1h" {
|
||||||
|
t.Fatal("details changed exchange")
|
||||||
|
}
|
||||||
|
if strings.Contains(fmt.Sprintf("%v %#v", handle, handle), "private") {
|
||||||
|
t.Fatal("format exposed history")
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(PreparedRun{ParentRunID: "source"})
|
||||||
|
if err != nil || !strings.Contains(string(encoded), `"parent_run_id":"source"`) {
|
||||||
|
t.Fatalf("prepared JSON = %s, %v", encoded, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
192
execution_budget_contract_test.go
Normal file
192
execution_budget_contract_test.go
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOperationBudgetCoversPreparationAndReleasesAdmission(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
reader := &capacityArtifactReader{entered: make(chan struct{}, 2), release: release}
|
||||||
|
client := newCapacityGateClient(closedCapacityChannel(), 2)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), reader)
|
||||||
|
parent := context.Background()
|
||||||
|
req := capacityInputRequest("http://example.invalid/v1")
|
||||||
|
req.OperationTimeout = time.Second
|
||||||
|
_, err := engine.Run(parent, req)
|
||||||
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || !errors.Is(err, promptkit.ErrArtifactLoad) || parent.Err() != nil {
|
||||||
|
t.Fatalf("preparation error=%v parent=%v", err, parent.Err())
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
if result, err := engine.Run(parent, req); err != nil || result == nil {
|
||||||
|
t.Fatalf("capacity not released: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperationBudgetCoversGenerationCapacityWait(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := newCapacityGateClient(release, 2)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(1), nil)
|
||||||
|
results := make(chan capacityRunResult, 1)
|
||||||
|
go runCapacityRequest(engine, context.Background(), promptkit.RunRequest{PromptID: "prompt"}, results)
|
||||||
|
<-client.started
|
||||||
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second})
|
||||||
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("capacity wait error=%v", err)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
if result := <-results; result.err != nil {
|
||||||
|
t.Fatal(result.err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperationBudgetInterruptsRetryAndRepair(t *testing.T) {
|
||||||
|
for _, repair := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "retry", true: "repair"}[repair], func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
config := simpleConfig("message")
|
||||||
|
one := 1
|
||||||
|
config.TransientRetries = &one
|
||||||
|
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
if repair && calls == 1 {
|
||||||
|
return retryHTTPResponse(200, "not json"), nil
|
||||||
|
}
|
||||||
|
if repair {
|
||||||
|
<-r.Context().Done()
|
||||||
|
return nil, r.Context().Err()
|
||||||
|
}
|
||||||
|
response := retryHTTPResponse(503, "")
|
||||||
|
response.Header = make(http.Header)
|
||||||
|
response.Header.Set("Retry-After", "5")
|
||||||
|
return response, nil
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req := simpleGenerationRequest()
|
||||||
|
req.OperationTimeout = time.Second
|
||||||
|
if repair {
|
||||||
|
req.Validation = &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}
|
||||||
|
}
|
||||||
|
_, err = engine.Run(context.Background(), req)
|
||||||
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("error=%v calls=%d", err, calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperationOutcomeFrozenBeforeTerminalObserver(t *testing.T) {
|
||||||
|
for _, split := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "run", true: "prepared"}[split], func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
engine := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||||
|
}))
|
||||||
|
var terminal promptkit.ExecutionEvent
|
||||||
|
req := promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, Observer: func(event promptkit.ExecutionEvent) {
|
||||||
|
if event.Kind == promptkit.ExecutionFinished {
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
terminal = event
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
var result *promptkit.RunResult
|
||||||
|
var err error
|
||||||
|
if split {
|
||||||
|
var handle *promptkit.PreparedExecution
|
||||||
|
handle, err = engine.PrepareExecution(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
result, err = engine.RunPrepared(context.Background(), handle)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result, err = engine.Run(context.Background(), req)
|
||||||
|
}
|
||||||
|
if err != nil || result == nil || terminal.Outcome != promptkit.ExecutionOutcomeSuccess {
|
||||||
|
t.Fatalf("result=%v error=%v terminal=%+v", result, err, terminal)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotificationConsumesOperationBudget(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
engine := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
t.Fatal("generation ran")
|
||||||
|
return nil, nil
|
||||||
|
}))
|
||||||
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) { <-ctx.Done() }})
|
||||||
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperationTimeoutDoesNotRelabelOtherFailures(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
failure := errors.New("independent failure")
|
||||||
|
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
<-ctx.Done()
|
||||||
|
return nil, failure
|
||||||
|
}))
|
||||||
|
var terminal promptkit.ExecutionEvent
|
||||||
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, Observer: func(event promptkit.ExecutionEvent) {
|
||||||
|
if event.Kind == promptkit.ExecutionFinished {
|
||||||
|
terminal = event
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
if !errors.Is(err, failure) || errors.Is(err, promptkit.ErrOperationTimeout) || terminal.Failure != promptkit.ExecutionFailureGeneration {
|
||||||
|
t.Fatalf("error=%v terminal=%+v", err, terminal)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperationTimeoutTerminalObserverSeesFinalFailure(t *testing.T) {
|
||||||
|
for _, split := range []bool{false, true} {
|
||||||
|
t.Run(map[bool]string{false: "run", true: "prepared"}[split], func(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
engine := observationEngine(t, observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
<-ctx.Done()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}))
|
||||||
|
var terminal promptkit.ExecutionEvent
|
||||||
|
req := promptkit.RunRequest{PromptID: "prompt", OperationTimeout: time.Second, Observer: func(event promptkit.ExecutionEvent) {
|
||||||
|
if event.Kind == promptkit.ExecutionFinished {
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
terminal = event
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
var err error
|
||||||
|
if split {
|
||||||
|
var handle *promptkit.PreparedExecution
|
||||||
|
handle, err = engine.PrepareExecution(context.Background(), req)
|
||||||
|
if err == nil {
|
||||||
|
_, err = engine.RunPrepared(context.Background(), handle)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
_, err = engine.Run(context.Background(), req)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || terminal.Failure != promptkit.ExecutionFailureDeadline {
|
||||||
|
t.Fatalf("error=%v terminal=%+v", err, terminal)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
38
execution_controls_conversion_test.go
Normal file
38
execution_controls_conversion_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExecutionControlsConversion(t *testing.T) {
|
||||||
|
ctx := context.WithValue(context.Background(), struct{}{}, "selected")
|
||||||
|
called := false
|
||||||
|
req := RunRequest{OperationTimeout: 3 * time.Second, OnPrepared: func(got context.Context, details PreparedRun) {
|
||||||
|
called = true
|
||||||
|
if got != ctx || details.PromptID != "prompt" {
|
||||||
|
t.Fatalf("unexpected notification: %v, %+v", got, details)
|
||||||
|
}
|
||||||
|
details.InputHashes["input"] = "changed"
|
||||||
|
}}
|
||||||
|
converted, err := toDomainRunRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if converted.OperationTimeout != req.OperationTimeout || converted.OnPrepared == nil || called {
|
||||||
|
t.Fatal("execution controls were not retained without invocation")
|
||||||
|
}
|
||||||
|
req.OnPrepared = func(context.Context, PreparedRun) { t.Fatal("callback selection changed") }
|
||||||
|
prepared := &domain.PreparedRun{PromptID: "prompt", InputHashes: map[string]string{"input": "original"}}
|
||||||
|
converted.OnPrepared(ctx, prepared)
|
||||||
|
if !called || prepared.InputHashes["input"] != "original" {
|
||||||
|
t.Fatal("notification did not deliver detached details")
|
||||||
|
}
|
||||||
|
withoutCallback, err := toDomainRunRequest(RunRequest{})
|
||||||
|
if err != nil || withoutCallback.OnPrepared != nil {
|
||||||
|
t.Fatalf("nil callback was adapted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user