Compare commits
357 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8ff9aa3efd | |||
| 78b78ea00b | |||
| 63e9f0b731 | |||
| 20b0f78af2 | |||
| 101a2840d4 | |||
| f083ad5309 | |||
| 19a0604874 | |||
| ea31e71280 | |||
| 27966c1686 | |||
| ea63a7b0ff | |||
| b5b9b1f209 | |||
| 129959b24c | |||
| dfe9d6566f | |||
| bf3d2e19bd | |||
| 4cbd4e2da0 | |||
| b0a16ccce8 | |||
| 8add7c49e6 | |||
| 8563379216 | |||
| 951739eda0 | |||
| a0f696181f | |||
| 3951dcf1f3 | |||
| 363920deb2 | |||
| 06315062b3 | |||
| c669f14610 | |||
| 52287ab2d1 | |||
| a43e0f17ee | |||
| ffaaf552c8 | |||
| d7007eb072 | |||
| 4f3212bfb6 | |||
| 4faf43cc29 | |||
| c9663ec63c | |||
| d8c98ac575 | |||
| 5ee602cc4b | |||
| 96c619f7d4 | |||
| b3b5ec3820 | |||
| 5dd1418953 | |||
| f5c900cb0a | |||
| c5f3de13f3 | |||
| 88725e79ab | |||
| 6c015a17fa | |||
| 9463c9463a | |||
| 1fbd175e53 | |||
| 4b4ce43796 | |||
| d78474a3bf | |||
| acaf2ccd8b | |||
| 97e5739bfb | |||
| 90f796aef7 | |||
| 0dec3dcec0 | |||
| 448e64dc40 | |||
| de46935a0e | |||
| c69703b2b6 | |||
| f660b6e0fc | |||
| 5f56c65427 | |||
| cfa3a85308 | |||
| 1ff6d778af | |||
| 3f93763c9c | |||
| 4aeb58f913 | |||
| ac1e150168 | |||
| ff57dbf8b3 | |||
| beb18cc9d6 | |||
| 7203e5d6a1 | |||
| 356563a6e5 | |||
| 6673284312 | |||
| 6f42e1e007 | |||
| cee9bbc7b2 | |||
| 7db5572b62 | |||
| a91bc1608b | |||
| e7a3665443 | |||
| 59e8010623 | |||
| ada5377f32 | |||
| df208e132b | |||
| 8c0fb15727 | |||
| 2e2fba4993 | |||
| 68668cee56 | |||
| f745b14b71 | |||
| dbba1485a7 | |||
| 7c435032b6 | |||
| e834e7d806 | |||
| a4901cbb0e | |||
| 5dcfc2e953 | |||
| a02561d40f | |||
| 6112ee8fb4 | |||
| 0aceff8c08 | |||
| cddb2b389c | |||
| e056cb1607 | |||
| 1e122f0473 | |||
| 2f0f5fa607 | |||
| 7d500ee62f | |||
| 7be677c80b | |||
| 1c075bd510 | |||
| b8dc8a49a9 | |||
| cd373ee46b | |||
| e881bbdcf4 | |||
| 106c808442 | |||
| 4ec35f5ca6 | |||
| 6a1a32ae49 | |||
| e85ea2db35 | |||
| 3ef2a9e18e | |||
| a85a6788b4 | |||
| 21fea46fb3 | |||
| 62bf82a56b | |||
| bd3dfa992c | |||
| 425f53a62e | |||
| fcc94eb9e9 | |||
| 95730a3c11 | |||
| c542b81666 | |||
| e8e112186e | |||
| aa08d46ef6 | |||
| 668d09997a | |||
| 69203dcb12 | |||
| 3b760310a6 | |||
| b39460a507 | |||
| 5177db0611 | |||
| 31f1e6a89d | |||
| f7085d76f6 | |||
| cf0bfd4d76 | |||
| 6b1323d1fe | |||
| fb87fabc2e | |||
| 5adc1c1e64 | |||
| 0d27252999 | |||
| d9f6aad305 | |||
| fda0338134 | |||
| 2b3d165ce2 | |||
| 43f55edb98 | |||
| 4950e1dee2 | |||
| d240364776 | |||
| 6f57f2f30a | |||
| 2c50eca033 | |||
| ac44a4f325 | |||
| 9e05c3ac8b | |||
| 2c83925c88 | |||
| e0cd63923d | |||
| 75204d6134 | |||
| 5ce6e01f27 | |||
| 5cec25a40c | |||
| 855aae6daa | |||
| 43abbcb414 | |||
| 9f33fed823 | |||
| 7a98feb399 | |||
| 626134e5e1 | |||
| e7ed315671 | |||
| d7d9a7dd65 | |||
| 18f6a474e0 | |||
| fb7ebe6a6c | |||
| a0441fb2d1 | |||
| 32547e1e29 | |||
| bcc9ebd2e0 | |||
| c7d4318122 | |||
| 175140ae5a | |||
| 66b98831d3 | |||
| 3e3cc677fb | |||
| 8896d8733b | |||
| 7bbdb55888 | |||
| 79dfd97afa | |||
| 995c309560 | |||
| ea4fec9b1b | |||
| 1de3e1fe38 | |||
| 061f01c081 | |||
| 937b9723d4 | |||
| 367af5feca | |||
| fd73cd0662 | |||
| 81a55bc62c | |||
| 2bc976de10 | |||
| fc568aa0b2 | |||
| b0bfe857fe | |||
| 2ccbee3758 | |||
| 0eee3ec962 | |||
| 130c4cc447 | |||
| f661176b06 | |||
| 3e67743fbf | |||
| 6b8eefd18b | |||
| 682cb84c82 | |||
| dbc345e566 | |||
| afcfae2aea | |||
| eaca50763b | |||
| cb83b8aca6 | |||
| 0b026861a5 | |||
| da8496b45b | |||
| e01b23259a | |||
| 500b0aa1c8 | |||
| fd6e9742a1 | |||
| a6f4d67e15 | |||
| c3ce15a4ab | |||
| 8e8f61f53f | |||
| b7565affe5 | |||
| aba890aa86 | |||
| 0b271267a0 | |||
| a4e97bf102 | |||
| 6ae005ed3a | |||
| 07a55e45f2 | |||
| 2b3875008b | |||
| a51d12d898 | |||
| 9178f3eb91 | |||
| a70fd7bb65 | |||
| 7091b51c87 | |||
| 8627c751a4 | |||
| 6fff3d822c | |||
| a1f12f3185 | |||
| 165cef7aec | |||
| 0221cd4610 | |||
| dfd8da843e | |||
| 29f104fbc0 | |||
| 8669294cf6 | |||
| c9b623b374 | |||
| 15f79ce7b1 | |||
| aa54b8d1c4 | |||
| 60c58260ec | |||
| 8dfe35a1f0 | |||
| 13b60a72ac | |||
| ee40a1bdd2 | |||
| 473fc1f6d7 | |||
| a10a9053b7 | |||
| ec3245c97e | |||
| 8cbb7c0daf | |||
| d40a010538 | |||
| 0cd833513c | |||
| 5e58b6944b | |||
| 0a85ac8cd1 | |||
| 89af2a1281 | |||
| 683689a4f8 | |||
| ddda2f8767 | |||
| 190ec69345 | |||
| b5866dd749 | |||
| c7490b783b | |||
| c7d1b0dca7 | |||
| 9e5ed08635 | |||
| 4bddbbe8f4 | |||
| 1516631a30 | |||
| cf997c2034 | |||
| a0ec67c256 | |||
| 75891b6561 | |||
| 5dc6370a97 | |||
| 4cda3674c4 | |||
| 5ea6f06a6d | |||
| 116bd0e39b | |||
| 208ff42915 | |||
| 972a103a97 | |||
| 2f735aedf9 | |||
| f4c721d934 | |||
| b7a951685b | |||
| abb2461244 | |||
| b268c55783 | |||
| f0182fb2c9 | |||
| 8d05b594e2 | |||
| edaaecc982 | |||
| 2e81c9e61b | |||
| 2b723760d1 | |||
| c5aed7a451 | |||
| d91afc61a4 | |||
| 6eeb0ee9c0 | |||
| b9c232ee63 | |||
| 7291d23587 | |||
| da1d196dc5 | |||
| b0de49a87a | |||
| 27ff580dae | |||
| ece764f19e | |||
| f91a02548e | |||
| 8fbd058b90 | |||
| c8f9a4def9 | |||
| d8b6c0b471 | |||
| 52cd94fcd1 | |||
| f1f72dfe8f | |||
| d4676bc870 | |||
| 64769883ae | |||
| 635674c066 | |||
| 50bc35516a | |||
| 18309f50db | |||
| 60784e7875 | |||
| f3148bfe52 | |||
| f315620bf7 | |||
| 97357d62b9 | |||
| 0c3651fceb | |||
| 5a5d47d36e | |||
| 143c6e9b3a | |||
| 48e4784fcd | |||
| 78af5069c1 | |||
| 18963266d7 | |||
| d039a019cb | |||
| d1c0281053 | |||
| 870e4ca4e6 | |||
| fd82524977 | |||
| 3f76c75919 | |||
| fe764f72d8 | |||
| 26ab566f38 | |||
| 18ca21fa45 | |||
| e32fdc22f4 | |||
| b02b802427 | |||
| 301192b974 | |||
| f2aeb5eb30 | |||
| a27ee8f7da | |||
| 4d1895c223 | |||
| d7fe70520a | |||
| 7325a34c07 | |||
| 03c44923c8 | |||
| dc0adf5751 | |||
| eae21797f7 | |||
| ea6c7d5f0f | |||
| 389ef26db3 | |||
| 8a5c0de162 | |||
| 80aecb646d | |||
| b7b4000aab | |||
| 11ecb923ba | |||
| f46d5a4591 | |||
| 743e78d514 | |||
| 3743221aa5 | |||
| a8863d49d0 | |||
| c5f733af45 | |||
| c09d2c6e20 | |||
| 6c5dec761c | |||
| 74b91fae99 | |||
| b7ff37118c | |||
| ef012ea1e5 | |||
| 04d5d695e0 | |||
| 78bff91395 | |||
| c63ca5352f | |||
| 3f157150e4 | |||
| f1e1a45a05 | |||
| 996295d5c1 | |||
| 58a5bbe790 | |||
| dd89be3b03 | |||
| ed7c27324d | |||
| 05fe13d80b | |||
| 0eb009ad5a | |||
| 3fde1f6cbb | |||
| 481e2d494c | |||
| eb830bb613 | |||
| dd6d4591ae | |||
| de6443f3ae | |||
| 5a63b649f9 | |||
| c7e22c0dbe | |||
| e69a847cf2 | |||
| 21815ad7c0 | |||
| a051edd82d | |||
| ee0ad99e17 | |||
| bc5a2da0f2 | |||
| 089e824a92 | |||
| 93e8874813 | |||
| cc0b6183bf | |||
| bf472dad8a | |||
| a6e0ffea8f | |||
| e459c7d54b | |||
| 71f6683fda | |||
| 7827ca043b | |||
| b595dca8c7 | |||
| d5f34642d7 | |||
| b1866946d1 | |||
| 05082c6fdf | |||
| 51b072965c | |||
| d5940856be | |||
| 0d163f8165 | |||
| 1889915a81 | |||
| 3ca01ae0b0 | |||
| 6e283d16c3 | |||
| 41b2021d8c | |||
| 3add7024db | |||
| a148a2b1f4 | |||
| 7310ef34a4 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -22,6 +22,9 @@
|
|||||||
go.work
|
go.work
|
||||||
go.work.sum
|
go.work.sum
|
||||||
|
|
||||||
|
# Local ReleaseKit delivery credentials
|
||||||
|
.releasekit/
|
||||||
|
|
||||||
# env file
|
# env file
|
||||||
.env
|
.env
|
||||||
|
|
||||||
@@ -67,4 +70,3 @@ Icon
|
|||||||
.AppleDesktop
|
.AppleDesktop
|
||||||
Network Trash Folder
|
Network Trash Folder
|
||||||
Temporary Items
|
Temporary Items
|
||||||
.apdisk
|
|
||||||
|
|||||||
48
README.md
48
README.md
@@ -4,7 +4,7 @@ Promptkit is a reusable Go library for preparing and executing prompt-defined
|
|||||||
LLM workflows. Its module path is:
|
LLM workflows. Its module path is:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
gitea.maximumdirect.net/eric/promptkit
|
go.maximumdirect.net/promptkit
|
||||||
```
|
```
|
||||||
|
|
||||||
The root `promptkit` package provides the supported public engine. Consumers
|
The root `promptkit` package provides the supported public engine. Consumers
|
||||||
@@ -21,15 +21,23 @@ Run the maintained offline preparation example from the repository root:
|
|||||||
go run ./examples/go-library/prepare
|
go run ./examples/go-library/prepare
|
||||||
```
|
```
|
||||||
|
|
||||||
It loads a repository-local prompt, supplies a programmatic profile and inline
|
It composes two explicitly selected synthetic collections, imports
|
||||||
input, inspects the captured configuration, and prints deterministic
|
source-relative prompt content, supplies an inline input, and prepares through
|
||||||
preparation metadata without contacting a provider or requiring credentials.
|
a scoped view. The example also demonstrates collection inventories and
|
||||||
|
namespace-filtered discovery, without contacting a provider or requiring
|
||||||
|
credentials.
|
||||||
|
The example disables the bundled OpenRouter configuration to keep its
|
||||||
|
inventories limited to its own offline fixtures.
|
||||||
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
||||||
ordered file/embedded sources, programmatic mappings, and how to replace an
|
ordered file/embedded sources, programmatic mappings, and how to replace an
|
||||||
engine when source files change. Read the
|
engine when source files change. Applications adapting the prior keyed
|
||||||
[example source](examples/go-library/prepare/main.go), the
|
configuration or versioned-prompt format can consult the
|
||||||
|
[configuration migration guide](docs/consumers/configuration-migration.md).
|
||||||
|
See the [example source](examples/go-library/prepare/main.go), the
|
||||||
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
[Go package consumer guide](docs/consumers/pkg-promptkit.md), and the
|
||||||
[framework format reference](docs/formats.md) to build a consumer workflow.
|
[framework format reference](docs/formats.md) to build a consumer workflow.
|
||||||
|
The consumer guide also covers explicit, live smoke diagnostics and interpreting
|
||||||
|
their factual reports; they are separate from the offline quickstart.
|
||||||
|
|
||||||
Contributors should start with the [development guide](docs/development.md).
|
Contributors should start with the [development guide](docs/development.md).
|
||||||
The [architecture policy](docs/policy/architecture.md) defines the library
|
The [architecture policy](docs/policy/architecture.md) defines the library
|
||||||
@@ -37,6 +45,34 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
The [v0.20.0 release notes](docs/releases/v0.20.0.md) cover resolved resource
|
||||||
|
inspection, generic JSON/YAML reports, typed prompt sources, and backend
|
||||||
|
inheritance. Upgrading consumers should review the changed inspection result
|
||||||
|
types and profile null-fallback behavior.
|
||||||
|
|
||||||
|
The [v0.19.0 release notes](docs/releases/v0.19.0.md) cover audit remediation
|
||||||
|
and compatibility changes to artifact ownership, programmatic sources, and schemas.
|
||||||
|
|
||||||
|
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
||||||
|
OpenRouter backend and profiles, recorded configuration normalization, and
|
||||||
|
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
||||||
|
the Go 1.27 minimum remains unchanged.
|
||||||
|
|
||||||
|
The [v0.17.0 release notes](docs/releases/v0.17.0.md) cover the vanity module
|
||||||
|
path, ConfigKit v1, identified resource lists, removal of prompt-version
|
||||||
|
metadata, and the Go 1.27 minimum. This is a breaking pre-v1 migration.
|
||||||
|
|
||||||
|
Consumers using the former Go module path should follow the
|
||||||
|
[module-path migration guidance](docs/consumers/configuration-migration.md#update-go-imports).
|
||||||
|
|
||||||
|
The [v0.16.0 release notes](docs/releases/v0.16.0.md) cover explicit smoke
|
||||||
|
diagnostics, configuration collections, scoped engine views, and discovery.
|
||||||
|
Upgrading consumers should also review the current identified-resource and
|
||||||
|
ConfigKit v1 changes in the [migration guide](docs/consumers/configuration-migration.md).
|
||||||
|
|
||||||
|
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
|
||||||
|
validation of concrete requests with `Engine.ValidateRequest`.
|
||||||
|
|
||||||
Consumers upgrading to `v0.12.0` should read the
|
Consumers upgrading to `v0.12.0` should read the
|
||||||
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
||||||
configuration and public construction API migration.
|
configuration and public construction API migration.
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
|
||||||
|
|||||||
43
artifact_ownership_contract_test.go
Normal file
43
artifact_ownership_contract_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type independentArtifactReader struct{}
|
||||||
|
|
||||||
|
func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
return &promptkit.Artifact{Body: []byte(ref.Body), Hash: ref.Body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("ownership", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithArtifactReader(independentArtifactReader{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var group sync.WaitGroup
|
||||||
|
for i := range 24 {
|
||||||
|
group.Go(func() {
|
||||||
|
value := fmt.Sprintf("body-%d", i)
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"document": promptkit.Inline(value)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("prepare: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != value || prepared.InputHashes["document"] != value {
|
||||||
|
t.Errorf("prepared = %+v", prepared)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
}
|
||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
artifactadapter "go.maximumdirect.net/promptkit/internal/artifact"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error")
|
var errNilArtifactReaderResponse = errors.New("artifact reader returned nil artifact without error")
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) {
|
func TestPublicArtifactReaderAdapterCopiesBody(t *testing.T) {
|
||||||
|
|||||||
237
assembled_workflow_benchmark_test.go
Normal file
237
assembled_workflow_benchmark_test.go
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
assembledPreparedSink *promptkit.PreparedRun
|
||||||
|
assembledResultSink *promptkit.RunResult
|
||||||
|
assembledDetailsSink promptkit.PreparedRun
|
||||||
|
assembledHandleSink *promptkit.PreparedExecution
|
||||||
|
)
|
||||||
|
|
||||||
|
type assembledWorkflowClient struct{ content string }
|
||||||
|
|
||||||
|
func (client assembledWorkflowClient) Generate(
|
||||||
|
context.Context,
|
||||||
|
promptkit.GenerateRequest,
|
||||||
|
) (*promptkit.GenerateResponse, error) {
|
||||||
|
return &promptkit.GenerateResponse{Content: client.content}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type assembledWorkflowFixture struct {
|
||||||
|
engine *promptkit.Engine
|
||||||
|
request promptkit.RunRequest
|
||||||
|
expectedMessage string
|
||||||
|
response string
|
||||||
|
}
|
||||||
|
|
||||||
|
func assembledWorkflowSchema(b *testing.B) []byte {
|
||||||
|
b.Helper()
|
||||||
|
properties := make(map[string]any, 256)
|
||||||
|
for index := range 256 {
|
||||||
|
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(map[string]any{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "object",
|
||||||
|
"properties": properties,
|
||||||
|
"additionalProperties": false,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func assembledWorkflowExtras(entries int) map[string]any {
|
||||||
|
providers := make(map[string]any, entries)
|
||||||
|
for index := range entries {
|
||||||
|
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||||
|
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||||
|
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{"provider_options": providers}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
|
||||||
|
b.Helper()
|
||||||
|
message := "Hello {{.name}}"
|
||||||
|
response := "ok"
|
||||||
|
output := map[string]any{"format": "text", "validation_mode": "none"}
|
||||||
|
profile := map[string]any{"id": "profile", "endpoint": "http://example.test/v1", "model": "benchmark-model"}
|
||||||
|
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
|
||||||
|
var expectedMessage string
|
||||||
|
|
||||||
|
switch name {
|
||||||
|
case "small":
|
||||||
|
expectedMessage = "Hello Ada"
|
||||||
|
case "large_message":
|
||||||
|
prefix := strings.Repeat("x", 256*1024)
|
||||||
|
message = prefix + "{{.name}}"
|
||||||
|
expectedMessage = prefix + "Ada"
|
||||||
|
case "mutable_payload":
|
||||||
|
message = `{"subject":"{{.name}}"}`
|
||||||
|
expectedMessage = `{"subject":"Ada"}`
|
||||||
|
response = "{}"
|
||||||
|
profile["extra_params"] = assembledWorkflowExtras(64)
|
||||||
|
output = map[string]any{
|
||||||
|
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
|
||||||
|
}
|
||||||
|
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
|
||||||
|
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
|
||||||
|
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
|
||||||
|
}, "."))
|
||||||
|
default:
|
||||||
|
b.Fatalf("unknown assembled workflow fixture %q", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
config := map[string]any{
|
||||||
|
"profiles": []any{profile},
|
||||||
|
"prompts": []any{map[string]any{"id": "workflow",
|
||||||
|
"default_profile": "profile",
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": message}},
|
||||||
|
"output": output,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("assembled workflow benchmark", config),
|
||||||
|
}}}, options...)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "workflow", Vars: map[string]string{"name": "Ada"},
|
||||||
|
}
|
||||||
|
return assembledWorkflowFixture{
|
||||||
|
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
|
||||||
|
b.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if prepared.PromptID != "workflow" || len(prepared.Messages) != 1 ||
|
||||||
|
prepared.Messages[0].Content != fixture.expectedMessage {
|
||||||
|
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
|
||||||
|
}
|
||||||
|
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
details := handle.Details()
|
||||||
|
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
|
||||||
|
handle.Discard()
|
||||||
|
b.Fatalf("prepared execution details are invalid: %+v", details)
|
||||||
|
}
|
||||||
|
handle.Discard()
|
||||||
|
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.RawOutput != fixture.response {
|
||||||
|
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
|
||||||
|
}
|
||||||
|
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
|
||||||
|
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEnginePreparation(b *testing.B) {
|
||||||
|
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||||
|
b.Run(name, func(b *testing.B) {
|
||||||
|
fixture := newAssembledWorkflowFixture(b, name)
|
||||||
|
verifyAssembledWorkflowFixture(b, fixture)
|
||||||
|
ctx := context.Background()
|
||||||
|
b.Run("prepare", func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
assembledPreparedSink = prepared
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledPreparedSink = nil
|
||||||
|
})
|
||||||
|
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
handle.Discard()
|
||||||
|
assembledHandleSink = handle
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledHandleSink = nil
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEngineRun(b *testing.B) {
|
||||||
|
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||||
|
b.Run(name, func(b *testing.B) {
|
||||||
|
fixture := newAssembledWorkflowFixture(b, name)
|
||||||
|
verifyAssembledWorkflowFixture(b, fixture)
|
||||||
|
ctx := context.Background()
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
result, err := fixture.engine.Run(ctx, fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
assembledResultSink = result
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledResultSink = nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkPreparedExecutionDetails(b *testing.B) {
|
||||||
|
for _, name := range []string{"small", "large_message", "mutable_payload"} {
|
||||||
|
b.Run(name, func(b *testing.B) {
|
||||||
|
fixture := newAssembledWorkflowFixture(b, name)
|
||||||
|
verifyAssembledWorkflowFixture(b, fixture)
|
||||||
|
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
defer handle.Discard()
|
||||||
|
initial := handle.Details()
|
||||||
|
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
|
||||||
|
b.Fatalf("prepared details fixture is invalid: %+v", initial)
|
||||||
|
}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for b.Loop() {
|
||||||
|
assembledDetailsSink = handle.Details()
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
assembledDetailsSink = promptkit.PreparedRun{}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
102
backends.go
102
backends.go
@@ -1,25 +1,31 @@
|
|||||||
package promptkit
|
package promptkit
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BackendOpenRouter identifies the maintained OpenRouter backend.
|
// BackendOpenRouter is the conventional ID for an OpenRouter backend.
|
||||||
const BackendOpenRouter = backend.OpenRouterID
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
// BackendRakestrawHome identifies the maintained Rakestrawhome backend.
|
// BackendRakestrawHome is the conventional ID for a Rakestrawhome backend.
|
||||||
const BackendRakestrawHome = backend.RakestrawHomeID
|
const BackendRakestrawHome = backend.RakestrawHomeID
|
||||||
|
|
||||||
// BackendLocal is a conventional ID for a caller-configured local backend.
|
// BackendLocal is a conventional ID for a caller-configured local backend.
|
||||||
const BackendLocal = "local"
|
const BackendLocal = "local"
|
||||||
|
|
||||||
// BackendConfig is a presence-aware convenience value for a keyed backend
|
// BackendConfig is a presence-aware convenience value for an identified backend
|
||||||
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
|
||||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||||
// empty map supplies an empty mapping. Use raw ConfigKit Mapping inputs for
|
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
||||||
// explicit null. BackendConfig has no stable JSON representation.
|
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
||||||
|
// inputs for explicit null. Extends names a sibling backend in the same list.
|
||||||
|
// BackendConfig has no stable JSON representation.
|
||||||
type BackendConfig struct {
|
type BackendConfig struct {
|
||||||
|
// ID is the local backend identity within its resource list.
|
||||||
|
ID string
|
||||||
|
// Extends names a sibling backend to inherit through ConfigKit.
|
||||||
|
Extends *string
|
||||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||||
Endpoint *string
|
Endpoint *string
|
||||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||||
@@ -31,17 +37,59 @@ type BackendConfig struct {
|
|||||||
QueueCapacity *int
|
QueueCapacity *int
|
||||||
// ExtraParams supplies JSON-compatible provider request defaults.
|
// ExtraParams supplies JSON-compatible provider request defaults.
|
||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
|
// SmokeTests supplies optional diagnostic defaults and HTTP probe settings.
|
||||||
|
SmokeTests *BackendSmokeTestConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
|
// BackendSmokeTestConfig contains optional diagnostic settings associated with
|
||||||
|
// a backend. Nil nested fields are omitted by BackendSource.
|
||||||
|
type BackendSmokeTestConfig struct {
|
||||||
|
// DefaultProfile is the exact profile ID selected as the backend's
|
||||||
|
// diagnostic inference default. A supplied value must not be blank; the ID
|
||||||
|
// is resolved only when that default is selected.
|
||||||
|
DefaultProfile *string `json:"default_profile,omitempty" yaml:"default_profile,omitempty"`
|
||||||
|
// Reachability optionally selects a full HTTP destination for reachability
|
||||||
|
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
||||||
|
Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"`
|
||||||
|
// Authentication optionally declares a named authentication diagnostic.
|
||||||
|
// A supplied object requires Kind.
|
||||||
|
Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReachabilityProbeConfig configures an optional reachability destination.
|
||||||
|
type ReachabilityProbeConfig struct {
|
||||||
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
|
// Nil selects the effective backend endpoint; an explicit blank value is
|
||||||
|
// invalid.
|
||||||
|
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AuthenticationProbeConfig configures an optional named authentication
|
||||||
|
// diagnostic.
|
||||||
|
type AuthenticationProbeConfig struct {
|
||||||
|
// Kind selects the authentication adapter. Supported values are "none" and
|
||||||
|
// "openrouter_key"; surrounding whitespace and case are normalized.
|
||||||
|
Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"`
|
||||||
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
|
// Nil selects the adapter's backend-relative destination; an explicit blank
|
||||||
|
// value is invalid.
|
||||||
|
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackendSource creates a labeled ConfigKit mapping containing backend records
|
||||||
|
// in caller order. IDs are explicit and duplicates are left for loading to reject.
|
||||||
// It encodes fields without validating them; NewEngine validates the final
|
// It encodes fields without validating them; NewEngine validates the final
|
||||||
// effective configuration after all selected source layers are combined.
|
// effective configuration after all selected source layers are combined.
|
||||||
// Mapping captures a copy of supported input values at creation; unsupported
|
// Mapping captures a copy of supported input values at creation; unsupported
|
||||||
// values fail when the source is loaded.
|
// values fail when the source is loaded. ExtraParams must use ConfigKit Mapping
|
||||||
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source {
|
// input forms; callers must explicitly convert numbers or containers as needed.
|
||||||
definitions := make(map[string]any, len(backends))
|
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||||
for id, config := range backends {
|
definitions := make([]any, 0, len(backends))
|
||||||
fields := make(map[string]any)
|
for _, config := range backends {
|
||||||
|
fields := map[string]any{"id": config.ID}
|
||||||
|
if config.Extends != nil {
|
||||||
|
fields["extends"] = map[string]any{"id": *config.Extends}
|
||||||
|
}
|
||||||
if config.Endpoint != nil {
|
if config.Endpoint != nil {
|
||||||
fields["endpoint"] = *config.Endpoint
|
fields["endpoint"] = *config.Endpoint
|
||||||
}
|
}
|
||||||
@@ -55,9 +103,33 @@ func BackendSource(label string, backends map[string]BackendConfig) configkit.So
|
|||||||
fields["queue_capacity"] = *config.QueueCapacity
|
fields["queue_capacity"] = *config.QueueCapacity
|
||||||
}
|
}
|
||||||
if config.ExtraParams != nil {
|
if config.ExtraParams != nil {
|
||||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
fields["extra_params"] = config.ExtraParams
|
||||||
}
|
}
|
||||||
definitions[id] = fields
|
if config.SmokeTests != nil {
|
||||||
|
smokeTests := make(map[string]any)
|
||||||
|
if config.SmokeTests.DefaultProfile != nil {
|
||||||
|
smokeTests["default_profile"] = *config.SmokeTests.DefaultProfile
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Reachability != nil {
|
||||||
|
reachability := make(map[string]any)
|
||||||
|
if config.SmokeTests.Reachability.Endpoint != nil {
|
||||||
|
reachability["endpoint"] = *config.SmokeTests.Reachability.Endpoint
|
||||||
|
}
|
||||||
|
smokeTests["reachability"] = reachability
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication != nil {
|
||||||
|
authentication := make(map[string]any)
|
||||||
|
if config.SmokeTests.Authentication.Kind != nil {
|
||||||
|
authentication["kind"] = *config.SmokeTests.Authentication.Kind
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication.Endpoint != nil {
|
||||||
|
authentication["endpoint"] = *config.SmokeTests.Authentication.Endpoint
|
||||||
|
}
|
||||||
|
smokeTests["authentication"] = authentication
|
||||||
|
}
|
||||||
|
fields["smoke_tests"] = smokeTests
|
||||||
|
}
|
||||||
|
definitions = append(definitions, fields)
|
||||||
}
|
}
|
||||||
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
return configkit.Mapping(label, map[string]any{"backends": definitions})
|
||||||
}
|
}
|
||||||
|
|||||||
114
bundled_composition_contract_test.go
Normal file
114
bundled_composition_contract_test.go
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
||||||
|
underlay := configkit.Input{Source: configkit.Mapping("caller underlay", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}},
|
||||||
|
})}
|
||||||
|
underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
||||||
|
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
|
||||||
|
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
||||||
|
}
|
||||||
|
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "gpt-sol-latest", "model": "main-model"},
|
||||||
|
map[string]any{"id": "custom", "extends": map[string]any{"id": "gpt-luna-latest"}, "model": "custom-model"},
|
||||||
|
map[string]any{"id": "auto", "backend": "other", "model": "other-model"},
|
||||||
|
},
|
||||||
|
"collections": map[string]any{"openrouter": map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://collection.invalid/v1"}},
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "backend": "openrouter/local", "model": "collection-model"}},
|
||||||
|
}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}, Main: &main}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspect := func(id string) *promptkit.ProfileInspection {
|
||||||
|
t.Helper()
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return profile
|
||||||
|
}
|
||||||
|
if target := inspect("gpt-sol-latest"); target.Model == nil || *target.Model != "main-model" || target.Backend != nil {
|
||||||
|
t.Fatalf("replacement = %+v", target)
|
||||||
|
}
|
||||||
|
if target := inspect("custom"); target.Model == nil || *target.Model != "custom-model" || target.Backend == nil || *target.Backend != "openrouter" || target.Endpoint != nil {
|
||||||
|
t.Fatalf("inherited = %+v", target)
|
||||||
|
}
|
||||||
|
if backend, err := engine.InspectBackend(context.Background(), "openrouter"); err != nil || backend.Endpoint != "https://local.invalid/v1" || backend.APIKeyEnv != nil {
|
||||||
|
t.Fatalf("backend replacement = %+v, %v", backend, err)
|
||||||
|
}
|
||||||
|
if target := inspect("openrouter/profile"); target.Backend == nil || *target.Backend != "openrouter/local" || target.Model == nil || *target.Model != "collection-model" {
|
||||||
|
t.Fatalf("collection = %+v", target)
|
||||||
|
}
|
||||||
|
target := inspect("auto-low")
|
||||||
|
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
||||||
|
if target.Backend == nil || *target.Backend != "other" || target.Model == nil || *target.Model != "other-model" || target.ExtraParams == nil || !reflect.DeepEqual(*target.ExtraParams, wantPlugins) {
|
||||||
|
t.Errorf("inherited tier = %+v", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledCompositionSequenceModes(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, mode, id string
|
||||||
|
wantErr bool
|
||||||
|
hasAuto bool
|
||||||
|
}{
|
||||||
|
{"duplicate append", "append", "auto", true, false},
|
||||||
|
{"distinct append", "append", "custom", false, true},
|
||||||
|
{"replace", "replace", "custom", false, false},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
|
||||||
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"profiles"}, "mode": test.mode}}},
|
||||||
|
"profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
|
if test.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate rejection")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil || !slices.Contains(profiles, "custom") || slices.Contains(profiles, "auto") != test.hasAuto {
|
||||||
|
t.Fatalf("profiles = %v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) {
|
||||||
|
profile := configkit.Input{Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})}
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("strict group accepted duplicate records")
|
||||||
|
}
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("replace backends", map[string]any{
|
||||||
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"backends"}, "mode": "replace"}}},
|
||||||
|
"backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
|
})}
|
||||||
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil {
|
||||||
|
t.Fatal("missing bundled backend was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
89
bundled_execution_contract_test.go
Normal file
89
bundled_execution_contract_test.go
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func bundledPromptConfig(client *http.Client) promptkit.Config {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||||
|
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
}}})}
|
||||||
|
return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledProfilePreparationAndExecution(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
engine, err := promptkit.NewEngine(bundledPromptConfig(nil), promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileRequired) {
|
||||||
|
t.Fatalf("missing profile = %v", err)
|
||||||
|
}
|
||||||
|
temperature := 0.3
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", ProfileID: "gpt-luna-latest", Execution: &promptkit.ExecutionTargetOverride{Temperature: &temperature}}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
details := prepared.Details()
|
||||||
|
if details.SelectedProfileID != "gpt-luna-latest" || details.EffectiveModelParams.Model != "~openai/gpt-luna-latest" || details.EffectiveModelParams.BackendID != "openrouter" || details.EffectiveModelParams.Temperature != temperature {
|
||||||
|
t.Fatalf("prepared target = %+v", details.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(context.Background(), request); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 || client.requests[0].Target.Model != "~openai/gpt-luna-latest" || client.requests[0].Target.BackendID != "openrouter" {
|
||||||
|
t.Fatalf("requests = %+v", client.requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
|
for _, test := range []struct{ id, model, tier string }{
|
||||||
|
{"auto-low", "openrouter/auto", "low"}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
||||||
|
} {
|
||||||
|
t.Run(test.id, func(t *testing.T) {
|
||||||
|
var body map[string]any
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||||
|
t.Errorf("authorization = %q", request.Header.Get("Authorization"))
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(bundledPromptConfig(&http.Client{Transport: transport}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", ProfileID: test.id}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if body["model"] != test.model || body["extra_params"] != nil {
|
||||||
|
t.Fatalf("request = %+v", body)
|
||||||
|
}
|
||||||
|
if test.tier == "" {
|
||||||
|
if _, exists := body["plugins"]; exists {
|
||||||
|
t.Fatalf("unexpected plugins = %+v", body)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
want := []any{map[string]any{"id": "auto-router", "cost_tier": test.tier}}
|
||||||
|
if !reflect.DeepEqual(body["plugins"], want) {
|
||||||
|
t.Fatalf("plugins = %#v, want %#v", body["plugins"], want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
117
bundled_openrouter_contract_test.go
Normal file
117
bundled_openrouter_contract_test.go
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type rejectBundleTransport struct{}
|
||||||
|
|
||||||
|
func (rejectBundleTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||||
|
return nil, errors.New("unexpected HTTP request")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
options []promptkit.Option
|
||||||
|
enabled bool
|
||||||
|
}{
|
||||||
|
{"default", nil, true}, {"enabled", []promptkit.Option{promptkit.WithBundledOpenRouter(true)}, true},
|
||||||
|
{"disabled", []promptkit.Option{promptkit.WithBundledOpenRouter(false)}, false},
|
||||||
|
{"last disables", []promptkit.Option{promptkit.WithBundledOpenRouter(true), promptkit.WithBundledOpenRouter(false)}, false},
|
||||||
|
{"last enables", []promptkit.Option{promptkit.WithBundledOpenRouter(false), promptkit.WithBundledOpenRouter(true)}, true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: rejectBundleTransport{}}}, test.options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
collections, err := engine.ListCollections(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if (len(profiles) > 0) != test.enabled || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
||||||
|
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
||||||
|
}
|
||||||
|
if test.enabled {
|
||||||
|
if !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
|
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
||||||
|
}
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), "auto")
|
||||||
|
if err != nil || profile.Backend == nil || *profile.Backend != "openrouter" {
|
||||||
|
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
|
t.Fatal("bundle source missing")
|
||||||
|
}
|
||||||
|
if explanation, found := view.Explain(configkit.Keys("backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 {
|
||||||
|
t.Fatalf("bundle explanation = %+v", explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolCount(value bool) int {
|
||||||
|
if value {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
func fmtSources(sources []configkit.SourceInfo) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, source := range sources {
|
||||||
|
b.WriteString(source.Label)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
|
||||||
|
source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}})
|
||||||
|
groups := make([]configkit.Underlay, 1, 2)
|
||||||
|
groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Source: source}}}
|
||||||
|
before := append([]configkit.Underlay(nil), groups...)
|
||||||
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(groups, before) || !reflect.DeepEqual(groups[:2][1], configkit.Underlay{}) {
|
||||||
|
t.Fatal("caller underlays were modified")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil || !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
|
t.Fatalf("backends = %v, %v", backends, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
181
bundled_smoke_contract_test.go
Normal file
181
bundled_smoke_contract_test.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cancelingAuthenticationBody struct {
|
||||||
|
remaining int
|
||||||
|
cancel context.CancelFunc
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *cancelingAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n := min(len(p), b.remaining)
|
||||||
|
for i := range p[:n] {
|
||||||
|
p[i] = 'x'
|
||||||
|
}
|
||||||
|
b.remaining -= n
|
||||||
|
if b.remaining == 0 {
|
||||||
|
b.cancel()
|
||||||
|
return n, context.Canceled
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
func (b *cancelingAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestAuthenticationReadCancellationWinsOverOverflow(t *testing.T) {
|
||||||
|
for _, size := range []int{1024, (1 << 20) + 1} {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
body := &cancelingAuthenticationBody{remaining: size, cancel: cancel}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter"})
|
||||||
|
cancel()
|
||||||
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != promptkit.ExecutionFailureCanceled || report.Facts.CredentialsAccepted != nil || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || !body.closed {
|
||||||
|
t.Fatalf("size %d: report=%+v error=%v closed=%v", size, report, err, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
|
var urls []string
|
||||||
|
var models []string
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
urls = append(urls, request.URL.String())
|
||||||
|
body := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
|
||||||
|
if request.Method == http.MethodPost {
|
||||||
|
var payload map[string]any
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
models = append(models, payload["model"].(string))
|
||||||
|
content := "PROMPTKIT_OK"
|
||||||
|
if _, structured := payload["response_format"]; structured {
|
||||||
|
content = `{"status":"ok"}`
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body = string(encoded)
|
||||||
|
} else if strings.HasSuffix(request.URL.Path, "/key") && request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||||
|
t.Errorf("authentication header missing")
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, _ := engine.ListProfiles(context.Background())
|
||||||
|
inspectionBefore, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, kind := range []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput} {
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: kind, BackendID: "openrouter"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Failure != "" {
|
||||||
|
t.Fatalf("%s report = %+v, %v", kind, report, err)
|
||||||
|
}
|
||||||
|
if kind == promptkit.SmokeTestAuthentication && (report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted) {
|
||||||
|
t.Fatalf("authentication facts = %+v", report.Facts)
|
||||||
|
}
|
||||||
|
if kind == promptkit.SmokeTestTextInference || kind == promptkit.SmokeTestStructuredOutput {
|
||||||
|
if report.Target.ProfileID != "gpt-luna-latest" || report.Target.EffectiveSettings.Model != "~openai/gpt-luna-latest" {
|
||||||
|
t.Fatalf("target = %+v", report.Target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantURLs := []string{"https://openrouter.ai/api/v1", "https://openrouter.ai/api/v1/key", "https://openrouter.ai/api/v1/chat/completions", "https://openrouter.ai/api/v1/chat/completions"}
|
||||||
|
if !reflect.DeepEqual(urls, wantURLs) || !reflect.DeepEqual(models, []string{"~openai/gpt-luna-latest", "~openai/gpt-luna-latest"}) {
|
||||||
|
t.Fatalf("urls %v, models %v", urls, models)
|
||||||
|
}
|
||||||
|
after, _ := engine.ListProfiles(context.Background())
|
||||||
|
if !reflect.DeepEqual(before, after) {
|
||||||
|
t.Fatalf("inventory changed: %v %v", before, after)
|
||||||
|
}
|
||||||
|
inspectionAfter, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
diff, err := inspectionBefore.Diff(inspectionAfter)
|
||||||
|
if err != nil || len(diff) != 0 {
|
||||||
|
t.Fatalf("inspection changed: %+v, %v", diff, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
||||||
|
replacement := configkit.Input{Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
|
||||||
|
replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||||
|
replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replacementReport, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||||
|
if err != nil || replacementReport.Target.ProfileID != "gpt-luna-latest" || replacementReport.Target.EffectiveSettings.Model != "replacement" {
|
||||||
|
t.Fatalf("replaced default = %+v, %v", replacementReport, err)
|
||||||
|
}
|
||||||
|
replacementClient.response.Content = "unexpected"
|
||||||
|
mismatch, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||||
|
if err != nil || mismatch == nil || !mismatch.Completed {
|
||||||
|
t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
|
||||||
|
}
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
request promptkit.SmokeTestRequest
|
||||||
|
profile, model string
|
||||||
|
}{
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter"}, "custom", "custom-model"},
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt"}, "free", "openrouter/free"},
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt", ProfileID: "gpt-luna-latest"}, "gpt-luna-latest", "replacement"},
|
||||||
|
} {
|
||||||
|
test.request.Kind = promptkit.SmokeTestTextInference
|
||||||
|
report, err := engine.SmokeTest(context.Background(), test.request)
|
||||||
|
if err != nil || report.Target.ProfileID != test.profile || report.Target.EffectiveSettings.Model != test.model {
|
||||||
|
t.Fatalf("report = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(client.requests[len(client.requests)-1].Prompt.Messages[0].Content, "private application marker") {
|
||||||
|
t.Fatal("application content used by diagnostic")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeAuthenticationResponseOverflow(t *testing.T) {
|
||||||
|
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(strings.Repeat("x", (1<<20)+1)))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter", APIKey: "private-key-marker"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Reason != promptkit.SmokeTestReasonAuthenticationResponseLimitExceeded || report.Failure != promptkit.ExecutionFailureOther || report.Facts.CredentialsAccepted != nil {
|
||||||
|
t.Fatalf("overflow = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,8 +7,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
|
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
|
||||||
@@ -247,11 +247,12 @@ func newBackendCapacityEngine(
|
|||||||
promptID, content = "input-prompt", `{{input "input"}}`
|
promptID, content = "input-prompt", `{{input "input"}}`
|
||||||
inputs = []any{map[string]any{"name": "input", "required": true}}
|
inputs = []any{map[string]any{"name": "input", "required": true}}
|
||||||
}
|
}
|
||||||
backend := map[string]any{"endpoint": "http://backend.example/v1", "concurrency_limit": limit}
|
backend := map[string]any{"id": "limited", "endpoint": "http://backend.example/v1", "concurrency_limit": limit}
|
||||||
if queueCapacity != nil {
|
if queueCapacity != nil {
|
||||||
backend["queue_capacity"] = *queueCapacity
|
backend["queue_capacity"] = *queueCapacity
|
||||||
}
|
}
|
||||||
definition := map[string]any{
|
definition := map[string]any{
|
||||||
|
"id": promptID,
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
@@ -265,9 +266,9 @@ func newBackendCapacityEngine(
|
|||||||
}
|
}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("capacity fixture", map[string]any{
|
Source: configkit.Mapping("capacity fixture", map[string]any{
|
||||||
"backends": map[string]any{"limited": backend},
|
"backends": []any{backend},
|
||||||
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
|
"profiles": []any{map[string]any{"id": "profile", "backend": "limited", "model": "model"}},
|
||||||
"prompts": map[string]any{promptID: map[string]any{"1": definition}},
|
"prompts": []any{definition},
|
||||||
}),
|
}),
|
||||||
}}}, options...)
|
}}}, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
261
configuration_diff_contract_test.go
Normal file
261
configuration_diff_contract_test.go
Normal file
@@ -0,0 +1,261 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||||
|
left := comparisonInspection(t, "earlier-model")
|
||||||
|
right := comparisonInspection(t, "later-model")
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare captured configurations: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 {
|
||||||
|
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||||
|
}
|
||||||
|
difference := differences[0]
|
||||||
|
if got, want := difference.Path.String(), "/profiles"; got != want {
|
||||||
|
t.Fatalf("difference path = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||||
|
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
|
||||||
|
}
|
||||||
|
|
||||||
|
reversed, err := right.Diff(left)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare reversed configurations: %v", err)
|
||||||
|
}
|
||||||
|
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
|
||||||
|
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
|
||||||
|
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
|
||||||
|
t.Fatalf("reversed difference = %+v", reversed)
|
||||||
|
}
|
||||||
|
|
||||||
|
equal := comparisonInspection(t, "earlier-model")
|
||||||
|
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
|
||||||
|
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
|
||||||
|
}
|
||||||
|
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
|
||||||
|
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
|
||||||
|
valid := comparisonInspection(t, "current-model")
|
||||||
|
var zero promptkit.ConfigurationInspection
|
||||||
|
|
||||||
|
for _, comparison := range []struct {
|
||||||
|
name string
|
||||||
|
left, right promptkit.ConfigurationInspection
|
||||||
|
}{
|
||||||
|
{name: "zero left", left: zero, right: valid},
|
||||||
|
{name: "zero right", left: valid, right: zero},
|
||||||
|
{name: "both zero", left: zero, right: zero},
|
||||||
|
} {
|
||||||
|
t.Run(comparison.name, func(t *testing.T) {
|
||||||
|
differences, err := comparison.left.Diff(comparison.right)
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
|
||||||
|
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
|
||||||
|
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
|
||||||
|
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
|
||||||
|
t.Fatal("comparison fixtures did not retain their distinct source labels")
|
||||||
|
}
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil || len(differences) != 0 {
|
||||||
|
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
|
||||||
|
inspect := func(endpoint string) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("backend", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": endpoint}},
|
||||||
|
})}}}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return view
|
||||||
|
}
|
||||||
|
left := inspect(" HTTPS://example.invalid/v1 ")
|
||||||
|
equal := inspect("https://example.invalid/v1")
|
||||||
|
differences, err := left.Diff(equal)
|
||||||
|
if err != nil || len(differences) != 0 {
|
||||||
|
t.Fatalf("normalized diff = %+v, %v", differences, err)
|
||||||
|
}
|
||||||
|
changed := inspect("https://other.invalid/v1")
|
||||||
|
differences, err = left.Diff(changed)
|
||||||
|
if err != nil || len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||||
|
t.Fatalf("changed diff = %+v, %v", differences, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"sequence": []any{"earlier", "unchanged"},
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"nullable": nil,
|
||||||
|
"sequence": []any{"later", "unchanged"},
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare presence and sequence values: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||||
|
t.Fatalf("atomic backend list difference = %+v", differences)
|
||||||
|
}
|
||||||
|
difference := differences[0]
|
||||||
|
if !difference.Left.Present || !difference.Right.Present ||
|
||||||
|
strings.Contains(difference.Left.YAML, "nullable:") || !strings.Contains(difference.Right.YAML, "nullable: null") ||
|
||||||
|
!strings.Contains(difference.Left.YAML, "earlier") || !strings.Contains(difference.Right.YAML, "later") {
|
||||||
|
t.Fatalf("atomic sequence snapshots = %+v", difference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"token": "left-secret-sentinel",
|
||||||
|
"visible": "before-value",
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"token": "right-secret-sentinel",
|
||||||
|
"visible": "after-value",
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare sensitive values: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||||
|
t.Fatalf("sensitive backend list difference = %+v", differences)
|
||||||
|
}
|
||||||
|
secret := differences[0]
|
||||||
|
if !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
||||||
|
t.Fatalf("sensitive difference = %+v", secret)
|
||||||
|
}
|
||||||
|
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
||||||
|
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
||||||
|
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
||||||
|
}
|
||||||
|
if !strings.Contains(secret.Left.YAML, "before-value") || !strings.Contains(secret.Right.YAML, "after-value") {
|
||||||
|
t.Fatalf("public sibling missing from snapshots: %+v", secret)
|
||||||
|
}
|
||||||
|
|
||||||
|
differences[0].Path = configkit.Keys("caller", "mutation")
|
||||||
|
differences[0].Left.YAML = "caller mutation"
|
||||||
|
differences[0].Right.YAML = "caller mutation"
|
||||||
|
again, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
||||||
|
}
|
||||||
|
if len(again) != 1 {
|
||||||
|
t.Fatalf("repeat difference count = %d, want 1: %+v", len(again), again)
|
||||||
|
}
|
||||||
|
againByPath := differencesByPath(again)
|
||||||
|
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||||
|
t.Fatal("mutating returned path reached retained inspection")
|
||||||
|
}
|
||||||
|
if _, exists := againByPath["/backends"]; !exists {
|
||||||
|
t.Fatalf("original backend path was lost after mutation: %+v", again)
|
||||||
|
}
|
||||||
|
for _, difference := range again {
|
||||||
|
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
||||||
|
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"details": map[string]any{
|
||||||
|
"token": "nested-secret-sentinel",
|
||||||
|
"label": "public detail",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"details": "replacement",
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||||
|
t.Fatalf("shape replacement differences = %+v", differences)
|
||||||
|
}
|
||||||
|
difference := differences[0]
|
||||||
|
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
|
||||||
|
t.Fatalf("shape replacement masking = %+v", difference)
|
||||||
|
}
|
||||||
|
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
|
||||||
|
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
backend := map[string]any{"id": "local", "endpoint": "https://example.invalid/v1"}
|
||||||
|
if extraParams != nil {
|
||||||
|
backend["extra_params"] = extraParams
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
||||||
|
"backends": []any{backend},
|
||||||
|
"profiles": []any{map[string]any{
|
||||||
|
"id": "offline", "backend": "local",
|
||||||
|
"model": model,
|
||||||
|
}},
|
||||||
|
})}}}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct comparison engine: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect comparison engine: %v", err)
|
||||||
|
}
|
||||||
|
return inspection
|
||||||
|
}
|
||||||
|
|
||||||
|
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
|
||||||
|
byPath := make(map[string]configkit.Difference, len(differences))
|
||||||
|
for _, difference := range differences {
|
||||||
|
byPath[difference.Path.String()] = difference
|
||||||
|
}
|
||||||
|
return byPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
|
||||||
|
for _, source := range view.Sources() {
|
||||||
|
if source.Label == label {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ConfigurationInspection is a read-only view of one Engine's captured
|
// ConfigurationInspection is a read-only view of one Engine's captured
|
||||||
@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
|||||||
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Diff compares this view's captured effective configuration with other. This
|
||||||
|
// view is the earlier (left) value and other is the later (right) value. The
|
||||||
|
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||||
|
// source or resolution history alone does not create differences. It does not
|
||||||
|
// reread external sources or compare referenced file contents, resolved
|
||||||
|
// credential contents, later execution preparation, or request-specific
|
||||||
|
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||||
|
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
|
||||||
|
// equivalent engine behavior or execution. Records can contain ordinary
|
||||||
|
// application data, so callers choose where and to whom to disclose them.
|
||||||
|
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
|
||||||
|
if v.source == nil || other.source == nil {
|
||||||
|
return nil, ErrInvalidConfig
|
||||||
|
}
|
||||||
|
return configuration.Diff(v.source, other.source)
|
||||||
|
}
|
||||||
|
|
||||||
// Metadata returns detached effective presence and history records.
|
// Metadata returns detached effective presence and history records.
|
||||||
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||||
if v.source == nil {
|
if v.source == nil {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -11,28 +12,37 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{
|
Underlays: []configkit.Underlay{
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
||||||
"backends": map[string]any{"local": map[string]any{
|
"backends": []any{map[string]any{
|
||||||
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY",
|
||||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||||
}},
|
}},
|
||||||
"profiles": map[string]any{
|
"collections": map[string]any{"literal": map[string]any{
|
||||||
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
"backends": []any{
|
||||||
"literal/id": map[string]any{"extends": "base.v1"},
|
map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "namespaced-private-sentinel"}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "local"}},
|
||||||
},
|
},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "base.v1", "backend": "local", "model": "default-model"},
|
||||||
|
map[string]any{"id": "id", "extends": map[string]any{"id": "base.v1"}},
|
||||||
|
},
|
||||||
|
}},
|
||||||
})}}},
|
})}}},
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
||||||
"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}},
|
"collections": map[string]any{"literal": map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
|
||||||
|
}},
|
||||||
})}}},
|
})}}},
|
||||||
},
|
},
|
||||||
}})
|
}}, promptkit.WithBundledOpenRouter(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct layered engine: %v", err)
|
t.Fatalf("construct layered engine: %v", err)
|
||||||
}
|
}
|
||||||
@@ -40,24 +50,43 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
modelPath := configkit.Keys("profiles", "base.v1", "model")
|
modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model")
|
||||||
model, found := view.Explain(modelPath)
|
model, found := view.Explain(modelPath)
|
||||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
|
||||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||||
}
|
}
|
||||||
inheritedPath := configkit.Keys("profiles", "literal/id", "model")
|
inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model")
|
||||||
inherited, found := view.Explain(inheritedPath)
|
inherited, found := view.Explain(inheritedPath)
|
||||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||||
}
|
}
|
||||||
if inheritedPath.String() != "/profiles/literal~1id/model" {
|
if inheritedPath.String() != "/collections/literal/profiles/1/model" {
|
||||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||||
}
|
}
|
||||||
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token")
|
||||||
|
endpointPath := configkit.Keys("backends").Index(0).Key("endpoint")
|
||||||
|
endpoint, found := view.Explain(endpointPath)
|
||||||
|
if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") {
|
||||||
|
t.Fatalf("normalized endpoint explanation = %+v", endpoint)
|
||||||
|
}
|
||||||
secret, found := view.Explain(secretPath)
|
secret, found := view.Explain(secretPath)
|
||||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||||
}
|
}
|
||||||
|
namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
|
||||||
|
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
||||||
|
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
||||||
|
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
||||||
|
}
|
||||||
|
inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
|
||||||
|
inheritedSecret, found := view.Explain(inheritedSecretPath)
|
||||||
|
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
|
||||||
|
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
|
||||||
|
}
|
||||||
|
metadata := fmt.Sprintf("%+v", view.Metadata().History(inheritedSecretPath))
|
||||||
|
if strings.Contains(metadata, "replacement-private-sentinel") || strings.Contains(metadata, "namespaced-private-sentinel") {
|
||||||
|
t.Fatal("metadata disclosed a sensitive provider value")
|
||||||
|
}
|
||||||
var effective bytes.Buffer
|
var effective bytes.Buffer
|
||||||
if err := view.WriteEffective(&effective); err != nil {
|
if err := view.WriteEffective(&effective); err != nil {
|
||||||
t.Fatalf("write effective configuration: %v", err)
|
t.Fatalf("write effective configuration: %v", err)
|
||||||
@@ -66,7 +95,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
t.Fatalf("unexpected effective report: %s", effective.String())
|
t.Fatalf("unexpected effective report: %s", effective.String())
|
||||||
}
|
}
|
||||||
var explanation bytes.Buffer
|
var explanation bytes.Buffer
|
||||||
if err := view.WriteExplanation(&explanation, secretPath); err != nil || strings.Contains(explanation.String(), "private-sentinel") {
|
if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") {
|
||||||
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,6 +122,45 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("invalid backend", map[string]any{
|
||||||
|
"collections": map[string]any{"private": map[string]any{"backends": []any{
|
||||||
|
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
|
||||||
|
}}},
|
||||||
|
}),
|
||||||
|
}}})
|
||||||
|
var loadErr *configkit.LoadError
|
||||||
|
if !errors.As(err, &loadErr) {
|
||||||
|
t.Fatalf("invalid backend error = %v, want load error", err)
|
||||||
|
}
|
||||||
|
if text := fmt.Sprintf("%v %+v", err, loadErr.Diagnostics()); strings.Contains(text, "failure-private-sentinel") {
|
||||||
|
t.Fatalf("failure diagnostics disclosed a sensitive value: %s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("main", map[string]any{"version": "2"}),
|
||||||
|
}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with accepted declaration: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
|
}
|
||||||
|
path := configkit.Keys("version")
|
||||||
|
field, found := inspection.Explain(path)
|
||||||
|
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
|
||||||
|
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
||||||
|
}
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") {
|
||||||
|
t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type failingInspectionWriter struct{ err error }
|
type failingInspectionWriter struct{ err error }
|
||||||
|
|
||||||
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }
|
||||||
@@ -117,7 +185,12 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
t.Fatalf("zero explanation writer = %v", err)
|
t.Fatalf("zero explanation writer = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
valid, err := promptkit.NewEngine(promptkit.Config{})
|
profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
|
||||||
|
})}
|
||||||
|
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}},
|
||||||
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
@@ -130,61 +203,109 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
|||||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("effective writer error = %v", err)
|
t.Fatalf("effective writer error = %v", err)
|
||||||
}
|
}
|
||||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) {
|
if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
|
||||||
t.Fatalf("explanation writer error = %v", err)
|
t.Fatalf("explanation writer error = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
||||||
|
const promptDocument = "prompts:\n - id: captured\n default_profile: offline\n messages:\n - role: user\n content_file: body.tmpl\n output:\n format: text\n validation_mode: none\n"
|
||||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
|
||||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||||
}}
|
}}
|
||||||
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("select inputs: %v", err)
|
t.Fatalf("select inputs: %v", err)
|
||||||
}
|
}
|
||||||
|
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||||
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
|
||||||
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||||
|
}}
|
||||||
|
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("select comparison inputs: %v", err)
|
||||||
|
}
|
||||||
var modelCalls atomic.Int64
|
var modelCalls atomic.Int64
|
||||||
client := countingLLMClient{calls: &modelCalls}
|
client := countingLLMClient{calls: &modelCalls}
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{
|
||||||
"offline": {Endpoint: &endpoint, Model: &model},
|
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
|
||||||
})}}}},
|
})}}}},
|
||||||
}}, promptkit.WithLLMClient(client))
|
}}, promptkit.WithLLMClient(client))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
|
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
|
||||||
|
})}}}},
|
||||||
|
}}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct comparison engine: %v", err)
|
||||||
|
}
|
||||||
source.closed.Store(true)
|
source.closed.Store(true)
|
||||||
|
otherSource.closed.Store(true)
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
otherView, err := otherEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect comparison configuration: %v", err)
|
||||||
|
}
|
||||||
|
var expected configkit.Difference
|
||||||
|
for index := 0; index < 3; index++ {
|
||||||
|
differences, err := view.Diff(otherView)
|
||||||
|
if err != nil || len(differences) != 1 {
|
||||||
|
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||||
|
}
|
||||||
|
if differences[0].Path.String() != "/profiles" ||
|
||||||
|
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||||
|
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||||
|
t.Fatalf("captured difference = %+v", differences[0])
|
||||||
|
}
|
||||||
|
if index == 0 {
|
||||||
|
expected = differences[0]
|
||||||
|
} else if differences[0].Path.String() != expected.Path.String() ||
|
||||||
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
|
||||||
|
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
|
||||||
t.Fatal("captured prompt content path was not inspectable")
|
t.Fatal("captured prompt content path was not inspectable")
|
||||||
}
|
}
|
||||||
var report bytes.Buffer
|
var report bytes.Buffer
|
||||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
||||||
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
||||||
}
|
}
|
||||||
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||||
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
|
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||||
}
|
}
|
||||||
|
|
||||||
const calls = 16
|
const calls = 24
|
||||||
var wait sync.WaitGroup
|
var wait sync.WaitGroup
|
||||||
errs := make(chan error, calls)
|
errs := make(chan error, calls)
|
||||||
for index := 0; index < calls; index++ {
|
for index := 0; index < calls; index++ {
|
||||||
wait.Add(1)
|
wait.Add(1)
|
||||||
go func(index int) {
|
go func(index int) {
|
||||||
defer wait.Done()
|
defer wait.Done()
|
||||||
if index%2 == 0 {
|
switch index % 3 {
|
||||||
|
case 0:
|
||||||
_, err := engine.InspectConfiguration()
|
_, err := engine.InspectConfiguration()
|
||||||
errs <- err
|
errs <- err
|
||||||
return
|
case 1:
|
||||||
}
|
|
||||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||||
errs <- err
|
errs <- err
|
||||||
|
case 2:
|
||||||
|
differences, err := view.Diff(otherView)
|
||||||
|
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
|
||||||
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
|
||||||
|
err = errors.New("concurrent comparison returned an unexpected difference")
|
||||||
|
}
|
||||||
|
errs <- err
|
||||||
|
}
|
||||||
}(index)
|
}(index)
|
||||||
}
|
}
|
||||||
wait.Wait()
|
wait.Wait()
|
||||||
@@ -194,7 +315,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
|
||||||
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
|
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,10 +7,27 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/filecatalog"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CollectionInput mounts source at the selected collection's configuration
|
||||||
|
// destination, collections/<namespace>, and enables ConfigKit imports for that
|
||||||
|
// input. An invalid namespace returns an error matching ErrInvalidConfig. The
|
||||||
|
// source is not read or validated until ConfigKit loads it. Callers may adjust
|
||||||
|
// the returned input, including its filesystem context or imports setting.
|
||||||
|
func CollectionInput(namespace string, source configkit.Source) (configkit.Input, error) {
|
||||||
|
if !domain.IsValidResourceNamespace(namespace) {
|
||||||
|
return configkit.Input{}, fmt.Errorf("%w: invalid collection namespace %q", ErrInvalidConfig, namespace)
|
||||||
|
}
|
||||||
|
return configkit.Input{
|
||||||
|
Source: source,
|
||||||
|
Destination: configkit.Keys("collections", namespace),
|
||||||
|
Imports: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
|
||||||
// directory as one strict-composition group. The returned order is deterministic;
|
// directory as one strict-composition group. The returned order is deterministic;
|
||||||
// it does not assign precedence among those files. Discovery is recursive over
|
// it does not assign precedence among those files. Discovery is recursive over
|
||||||
|
|||||||
@@ -1,14 +1,16 @@
|
|||||||
package promptkit_test
|
package promptkit_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type sourceDocument struct {
|
type sourceDocument struct {
|
||||||
@@ -16,6 +18,78 @@ type sourceDocument struct {
|
|||||||
Second string `yaml:"second"`
|
Second string `yaml:"second"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputValidatesWithoutReadingSource(t *testing.T) {
|
||||||
|
input, err := promptkit.CollectionInput("docs", configkit.File(filepath.Join(t.TempDir(), "not-read.yaml")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if input.Destination.String() != "/collections/docs" || !input.Imports {
|
||||||
|
t.Fatalf("collection input = %+v, want destination /collections/docs with imports enabled", input)
|
||||||
|
}
|
||||||
|
for _, namespace := range []string{"", " ", "docs/api"} {
|
||||||
|
if _, err := promptkit.CollectionInput(namespace, configkit.NoSource()); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Errorf("namespace %q error = %v, want ErrInvalidConfig", namespace, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
mainPath := filepath.Join(directory, "collection.yaml")
|
||||||
|
defaultsPath := filepath.Join(directory, "defaults.yaml")
|
||||||
|
if err := os.WriteFile(mainPath, []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(defaultsPath, []byte("prompts:\n - id: greeting\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input, err := promptkit.CollectionInput("docs", configkit.File(mainPath))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
|
||||||
|
Destination: configkit.Keys("collections", "docs"),
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with native collection import: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
||||||
|
files := fstest.MapFS{
|
||||||
|
"collection.yaml": {Data: []byte("config:\n imports:\n - path: defaults.yaml\n required: true\n")},
|
||||||
|
"defaults.yaml": {Data: []byte("prompts:\n - id: greeting\n messages:\n - role: user\n content: Hello\n output:\n format: text\n validation_mode: none\n")},
|
||||||
|
}
|
||||||
|
filesystem := configkit.NewFilesystem("embedded collection", files)
|
||||||
|
input, err := promptkit.CollectionInput("docs", filesystem.File("collection.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input.Filesystem = &configkit.FilesystemContext{FS: &filesystem, Root: "."}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
Main: &configkit.Input{
|
||||||
|
Source: configkit.Mapping("later collection override", map[string]any{"prompts": []any{map[string]any{"id": "greeting", "inputs": []any{map[string]any{"name": "overridden"}}, "messages": []any{map[string]any{"role": "user", "content": "Updated"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}}),
|
||||||
|
Destination: configkit.Keys("collections", "docs"),
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with embedded collection import: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
func TestDirectoryInputsSelectsSortedFilesWithRootContext(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package promptkit_test
|
package promptkit_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
@@ -11,8 +12,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
||||||
@@ -37,18 +38,19 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
cfg := simpleConfig("hello")
|
cfg := simpleConfig("hello")
|
||||||
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
|
cfg.Sources.Underlays = []configkit.Underlay{{Inputs: []configkit.Input{*cfg.Sources.Main}}}
|
||||||
|
endpoint, model := "http://example.test/v1", "model"
|
||||||
lowerName := name
|
lowerName := name
|
||||||
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
cfg.Sources.Underlays = append(cfg.Sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||||
Source: promptkit.ProfileSource("default credentials", map[string]promptkit.ProfileConfig{
|
Source: promptkit.ProfileSource("default credentials", []promptkit.ProfileConfig{
|
||||||
"profile": {APIKeyEnv: &lowerName},
|
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &lowerName},
|
||||||
}),
|
}),
|
||||||
}}})
|
}}})
|
||||||
env := " " + name + " "
|
env := " " + name + " "
|
||||||
if test.clear {
|
if test.clear {
|
||||||
env = ""
|
env = ""
|
||||||
}
|
}
|
||||||
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", map[string]promptkit.ProfileConfig{
|
cfg.Sources.Main = &configkit.Input{Source: promptkit.ProfileSource("credentials", []promptkit.ProfileConfig{
|
||||||
"profile": {APIKeyEnv: &env},
|
{ID: "profile", Endpoint: &endpoint, Model: &model, APIKeyEnv: &env},
|
||||||
})}
|
})}
|
||||||
var calls int
|
var calls int
|
||||||
var auth string
|
var auth string
|
||||||
@@ -72,20 +74,23 @@ func TestProfileEnvironmentLookupAcceptsNonShellNames(t *testing.T) {
|
|||||||
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
|
||||||
backendID, model := "workstation", "offline-model"
|
backendID, model := "workstation", "offline-model"
|
||||||
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
|
||||||
"prompts": map[string]any{"hello": map[string]any{"1": map[string]any{
|
"prompts": []any{map[string]any{
|
||||||
|
"id": "hello",
|
||||||
"default_profile": "offline",
|
"default_profile": "offline",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}}},
|
}},
|
||||||
})}
|
})}
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
|
||||||
|
localBackend := promptkit.LocalBackend("http://localhost:8000/v1", 2)
|
||||||
|
localBackend.ID = backendID
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
{Source: promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
|
{Source: promptkit.BackendSource("local backend", []promptkit.BackendConfig{
|
||||||
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
localBackend,
|
||||||
})},
|
})},
|
||||||
{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||||
"offline": {Backend: &backendID, Model: &model},
|
{ID: "offline", Backend: &backendID, Model: &model},
|
||||||
})},
|
})},
|
||||||
}}},
|
}}},
|
||||||
Main: &prompt,
|
Main: &prompt,
|
||||||
@@ -105,16 +110,45 @@ func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
||||||
|
openRouterID, rakestrawhomeID := promptkit.BackendOpenRouter, promptkit.BackendRakestrawHome
|
||||||
|
openRouterEndpoint, rakestrawhomeEndpoint := "https://openrouter.example/v1", "https://inference.example/v1"
|
||||||
|
model := "configured-model"
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
|
{Source: promptkit.BackendSource("application backends", []promptkit.BackendConfig{
|
||||||
|
{ID: openRouterID, Endpoint: &openRouterEndpoint},
|
||||||
|
{ID: rakestrawhomeID, Endpoint: &rakestrawhomeEndpoint},
|
||||||
|
})},
|
||||||
|
{Source: promptkit.ProfileSource("application profiles", []promptkit.ProfileConfig{
|
||||||
|
{ID: "openrouter-profile", Backend: &openRouterID, Model: &model},
|
||||||
|
{ID: "rakestrawhome-profile", Backend: &rakestrawhomeID, Model: &model},
|
||||||
|
})},
|
||||||
|
}}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with explicitly configured provider IDs: %v", err)
|
||||||
|
}
|
||||||
|
for profileID, backendID := range map[string]string{
|
||||||
|
"openrouter-profile": openRouterID,
|
||||||
|
"rakestrawhome-profile": rakestrawhomeID,
|
||||||
|
} {
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||||
|
if err != nil || inspection.Backend == nil || *inspection.Backend != backendID {
|
||||||
|
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
|
||||||
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
|
||||||
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
"config/default.yaml": &fstest.MapFile{Data: []byte(`
|
||||||
profiles:
|
profiles:
|
||||||
application:
|
- id: application
|
||||||
endpoint: https://example.invalid/v1
|
endpoint: https://example.invalid/v1
|
||||||
model: default-model
|
model: default-model
|
||||||
prompts:
|
prompts:
|
||||||
hello:
|
- id: hello
|
||||||
"1":
|
|
||||||
default_profile: application
|
default_profile: application
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -128,16 +162,16 @@ prompts:
|
|||||||
t.Fatalf("select embedded defaults: %v", err)
|
t.Fatalf("select embedded defaults: %v", err)
|
||||||
}
|
}
|
||||||
operatorDirectory := t.TempDir()
|
operatorDirectory := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n application:\n model: operator-model\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n - id: application\n endpoint: https://example.invalid/v1\n model: operator-model\n"), 0o644); err != nil {
|
||||||
t.Fatalf("write operator input: %v", err)
|
t.Fatalf("write operator input: %v", err)
|
||||||
}
|
}
|
||||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("select operator inputs: %v", err)
|
t.Fatalf("select operator inputs: %v", err)
|
||||||
}
|
}
|
||||||
programmaticModel := "programmatic-model"
|
programmaticModel, programmaticEndpoint := "programmatic-model", "https://example.invalid/v1"
|
||||||
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", map[string]promptkit.ProfileConfig{
|
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", []promptkit.ProfileConfig{
|
||||||
"application": {Model: &programmaticModel},
|
{ID: "application", Endpoint: &programmaticEndpoint, Model: &programmaticModel},
|
||||||
})}
|
})}
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -173,7 +207,7 @@ prompts:
|
|||||||
|
|
||||||
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||||
Source: configkit.Mapping("invalid", map[string]any{"profiles": map[string]any{"broken": map[string]any{"unknown": true}}}),
|
Source: configkit.Mapping("invalid", map[string]any{"profiles": []any{map[string]any{"id": "broken", "unknown": true}}}),
|
||||||
}}})
|
}}})
|
||||||
var loadErr *configkit.LoadError
|
var loadErr *configkit.LoadError
|
||||||
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
|
||||||
@@ -189,43 +223,96 @@ func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) {
|
func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T) {
|
||||||
endpoint := "http://local-openrouter.example/v1"
|
endpoint, backendID, model := "http://application.example/v1", "application-backend", "synthetic-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
|
||||||
promptkit.BackendOpenRouter: {Endpoint: &endpoint},
|
{Source: promptkit.BackendSource("operator backend", []promptkit.BackendConfig{
|
||||||
})}}}},
|
{ID: backendID, Endpoint: &endpoint},
|
||||||
|
})},
|
||||||
|
{Source: promptkit.ProfileSource("operator profile", []promptkit.ProfileConfig{
|
||||||
|
{ID: "application-profile", Backend: &backendID, Model: &model},
|
||||||
|
})},
|
||||||
|
}}},
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine with backend overlay: %v", err)
|
t.Fatalf("construct engine with backend overlay: %v", err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3")
|
inspection, err := engine.InspectProfile(context.Background(), "application-profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect catalog profile: %v", err)
|
t.Fatalf("inspect configured profile: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
backendInspection, err := engine.InspectBackend(context.Background(), backendID)
|
||||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
if err != nil || inspection.Endpoint != nil || backendInspection.Endpoint != endpoint {
|
||||||
|
t.Fatalf("profile/backend = %+v / %+v, %v", inspection, backendInspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparationRejectsAmbiguousPromptVersion(t *testing.T) {
|
func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
||||||
definition := func(content string) map[string]any {
|
backendEndpoint := "https://backend.example/v1"
|
||||||
return map[string]any{
|
defaultProfile := "not-yet-listed"
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
reachabilityEndpoint := " https://probe.example/ready?check=1 "
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
authenticationKind := " OPENROUTER_KEY "
|
||||||
}
|
authenticationEndpoint := "https://probe.example/key?scope=current"
|
||||||
}
|
source := promptkit.BackendSource("diagnostic backend", []promptkit.BackendConfig{
|
||||||
|
{
|
||||||
|
ID: "local",
|
||||||
|
Endpoint: &backendEndpoint,
|
||||||
|
SmokeTests: &promptkit.BackendSmokeTestConfig{
|
||||||
|
DefaultProfile: &defaultProfile,
|
||||||
|
Reachability: &promptkit.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||||
|
Authentication: &promptkit.AuthenticationProbeConfig{
|
||||||
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
backendEndpoint = "https://mutated.example/v1"
|
||||||
|
defaultProfile = "mutated-profile"
|
||||||
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
|
authenticationKind = "none"
|
||||||
|
authenticationEndpoint = "https://mutated.example/key"
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("versions", map[string]any{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: source}}}},
|
||||||
"prompts": map[string]any{"hello": map[string]any{
|
|
||||||
"1": definition("first"), "2": definition("second"),
|
|
||||||
}},
|
|
||||||
})},
|
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine with dangling diagnostic default: %v", err)
|
||||||
}
|
}
|
||||||
if result, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}); result != nil || !errors.Is(err, promptkit.ErrPromptLoad) {
|
inspection, err := engine.InspectConfiguration()
|
||||||
t.Fatalf("ambiguous preparation = (%#v, %v), want ErrPromptLoad", result, err)
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&effective); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
contents := effective.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"https://backend.example/v1",
|
||||||
|
"not-yet-listed",
|
||||||
|
"https://probe.example/ready?check=1",
|
||||||
|
"openrouter_key",
|
||||||
|
"https://probe.example/key?scope=current",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(contents, want) {
|
||||||
|
t.Errorf("effective configuration does not contain %q:\n%s", want, contents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, unwanted := range []string{
|
||||||
|
"mutated.example",
|
||||||
|
"mutated-profile",
|
||||||
|
} {
|
||||||
|
if strings.Contains(contents, unwanted) {
|
||||||
|
t.Errorf("effective configuration unexpectedly contains %q:\n%s", unwanted, contents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparationSelectsSinglePromptDefinitionByID(t *testing.T) {
|
||||||
|
engine := newSimpleEngine(t, "selected")
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil || prepared == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Content != "selected" {
|
||||||
|
t.Fatalf("prepare prompt by ID = (%+v, %v)", prepared, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type fakeLLMClient struct {
|
type fakeLLMClient struct {
|
||||||
@@ -60,14 +60,14 @@ func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option)
|
|||||||
func simpleConfig(content string) promptkit.Config {
|
func simpleConfig(content string) promptkit.Config {
|
||||||
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
return promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
|
||||||
"profiles": map[string]any{"profile": map[string]any{
|
"profiles": []any{map[string]any{"id": "profile",
|
||||||
"endpoint": "http://example.test/v1", "model": "model",
|
"endpoint": "http://example.test/v1", "model": "model",
|
||||||
}},
|
}},
|
||||||
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
|
"prompts": []any{map[string]any{"id": "prompt",
|
||||||
"default_profile": "profile",
|
"default_profile": "profile",
|
||||||
"messages": []any{map[string]any{"role": "user", "content": content}},
|
"messages": []any{map[string]any{"role": "user", "content": content}},
|
||||||
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
}}},
|
}},
|
||||||
})},
|
})},
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|||||||
173
convert.go
173
convert.go
@@ -2,11 +2,11 @@ package promptkit
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||||
@@ -25,7 +25,6 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
}
|
}
|
||||||
return domain.RunRequest{
|
return domain.RunRequest{
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
APIKey: req.APIKey,
|
APIKey: req.APIKey,
|
||||||
@@ -73,7 +72,6 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
}
|
}
|
||||||
return &PreparedRun{
|
return &PreparedRun{
|
||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
SelectedBackendID: prepared.SelectedBackendID,
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
@@ -100,7 +98,6 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
RawOutput: result.RawOutput,
|
RawOutput: result.RawOutput,
|
||||||
Validation: fromDomainValidationResult(result.Validation),
|
Validation: fromDomainValidationResult(result.Validation),
|
||||||
PromptID: result.PromptID,
|
PromptID: result.PromptID,
|
||||||
PromptVersion: result.PromptVersion,
|
|
||||||
PromptHash: result.PromptHash,
|
PromptHash: result.PromptHash,
|
||||||
SessionID: result.SessionID,
|
SessionID: result.SessionID,
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
@@ -211,6 +208,12 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
|
return fromDomainExecutionTargetWithOwnedParams(target, copyAnyMap(target.ExtraParams))
|
||||||
|
}
|
||||||
|
|
||||||
|
// fromDomainExecutionTargetWithOwnedParams projects scalar settings and takes
|
||||||
|
// ownership of extraParams, which the caller has already copied.
|
||||||
|
func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, extraParams map[string]any) ExecutionTarget {
|
||||||
return ExecutionTarget{
|
return ExecutionTarget{
|
||||||
BackendID: target.BackendID,
|
BackendID: target.BackendID,
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
@@ -222,42 +225,62 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
|||||||
ServiceTier: target.ServiceTier,
|
ServiceTier: target.ServiceTier,
|
||||||
ReasoningEffort: target.ReasoningEffort,
|
ReasoningEffort: target.ReasoningEffort,
|
||||||
APIKeyEnv: target.APIKeyEnv,
|
APIKeyEnv: target.APIKeyEnv,
|
||||||
ExtraParams: copyAnyMap(target.ExtraParams),
|
ExtraParams: extraParams,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
|
||||||
if inspection == nil {
|
out := &ProfileInspection{ProfileID: v.ID, Backend: copyStringPtr(v.Backend), Endpoint: copyStringPtr(v.Endpoint), Model: copyStringPtr(v.Model), Temperature: copyFloat64Ptr(v.Temperature), TopP: copyFloat64Ptr(v.TopP), MaxTokens: copyIntPtr(v.MaxTokens), TimeoutSeconds: copyIntPtr(v.TimeoutSeconds), ServiceTier: copyStringPtr(v.ServiceTier), ReasoningEffort: copyStringPtr(v.ReasoningEffort), APIKeyEnv: copyStringPtr(v.APIKeyEnv), APIKeyRequired: copyBoolPtr(v.APIKeyRequired)}
|
||||||
return nil
|
if v.ExtraParams != nil {
|
||||||
}
|
data := copyAnyMap(v.ExtraParams)
|
||||||
return &ProfileInspection{
|
out.ExtraParams = &data
|
||||||
ProfileID: inspection.ProfileID,
|
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
|
||||||
APIKeyRequired: inspection.APIKeyRequired,
|
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
||||||
if inspection == nil {
|
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||||
|
if v.ExtraParams != nil {
|
||||||
|
data := copyAnyMap(v.ExtraParams)
|
||||||
|
out.ExtraParams = &data
|
||||||
|
}
|
||||||
|
if v.SmokeTests != nil {
|
||||||
|
out.SmokeTests = &BackendSmokeTestConfig{DefaultProfile: copyStringPtr(v.SmokeTests.DefaultProfile)}
|
||||||
|
if v.SmokeTests.Reachability != nil {
|
||||||
|
out.SmokeTests.Reachability = &ReachabilityProbeConfig{Endpoint: copyStringPtr(v.SmokeTests.Reachability.Endpoint)}
|
||||||
|
}
|
||||||
|
if v.SmokeTests.Authentication != nil {
|
||||||
|
out.SmokeTests.Authentication = &AuthenticationProbeConfig{Kind: copyStringPtr(v.SmokeTests.Authentication.Kind), Endpoint: copyStringPtr(v.SmokeTests.Authentication.Endpoint)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
|
||||||
|
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: copyIntPtr(v.Output.RepairAttempts)}}
|
||||||
|
if v.Inputs != nil {
|
||||||
|
inputs := make([]PromptInputInspection, len(*v.Inputs))
|
||||||
|
for i, input := range *v.Inputs {
|
||||||
|
inputs[i] = PromptInputInspection{Name: input.Name, Required: copyBoolPtr(input.Required), ContentType: copyStringPtr(input.ContentType), Description: copyStringPtr(input.Description)}
|
||||||
|
}
|
||||||
|
out.Inputs = &inputs
|
||||||
|
}
|
||||||
|
out.Messages = make([]PromptMessageInspection, len(v.Messages))
|
||||||
|
for i, message := range v.Messages {
|
||||||
|
out.Messages[i] = PromptMessageInspection{Role: message.Role, Content: message.Content}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
out.Messages[i].CacheControl = &PromptCacheControlInspection{Type: CacheControlType(message.CacheControl.Type), TTL: copyStringPtr(message.CacheControl.TTL)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBoolPtr(src *bool) *bool {
|
||||||
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
value := *src
|
||||||
for i, input := range inspection.Inputs {
|
return &value
|
||||||
inputs[i] = PromptInputDefinition{
|
|
||||||
Name: input.Name,
|
|
||||||
Required: input.Required,
|
|
||||||
ContentType: input.ContentType,
|
|
||||||
Description: input.Description,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &PromptInspection{
|
|
||||||
PromptID: inspection.PromptID,
|
|
||||||
PromptVersion: inspection.PromptVersion,
|
|
||||||
PromptHash: inspection.PromptHash,
|
|
||||||
DefaultProfileID: inspection.DefaultProfileID,
|
|
||||||
Inputs: inputs,
|
|
||||||
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
@@ -393,88 +416,10 @@ func copyAnyMap(src map[string]any) map[string]any {
|
|||||||
if src == nil {
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make(map[string]any, len(src))
|
return jsonvalue.CloneTrusted(src).(map[string]any)
|
||||||
for k, v := range src {
|
|
||||||
out[k] = copyAny(v)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyAny(value any) any {
|
func copyAny(value any) any { return jsonvalue.CloneTrusted(value) }
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
switch v := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
return copyAnyMap(v)
|
|
||||||
case []any:
|
|
||||||
out := make([]any, len(v))
|
|
||||||
for i, item := range v {
|
|
||||||
out[i] = copyAny(item)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case []string:
|
|
||||||
return copyStringSlice(v)
|
|
||||||
case []byte:
|
|
||||||
return copyBytes(v)
|
|
||||||
default:
|
|
||||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
|
||||||
if !value.IsValid() {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
switch value.Kind() {
|
|
||||||
case reflect.Interface:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
copied := copyReflectValue(value.Elem())
|
|
||||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
|
||||||
return copied
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
out.Set(copied)
|
|
||||||
return out
|
|
||||||
case reflect.Pointer:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type().Elem())
|
|
||||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
|
||||||
return out
|
|
||||||
case reflect.Map:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
|
||||||
iter := value.MapRange()
|
|
||||||
for iter.Next() {
|
|
||||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Slice:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Array:
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
default:
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyStringSlice(src []string) []string {
|
func copyStringSlice(src []string) []string {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestToDomainAppendedMessages(t *testing.T) {
|
func TestToDomainAppendedMessages(t *testing.T) {
|
||||||
|
|||||||
39
doc.go
39
doc.go
@@ -4,27 +4,30 @@
|
|||||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||||
// sources for prompts, profiles, and backends, and
|
// sources for prompts, profiles, and backends, and
|
||||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||||
// [Engine.InspectProfile], [Engine.Prepare],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.InspectBackend],
|
||||||
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
|
// [Engine.ListProfiles], [Engine.ListPrompts], [Engine.Prepare],
|
||||||
// registries, repositories, validators, and the built-in OpenAI-compatible
|
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
||||||
// client remain internal implementation details.
|
// validators, and the built-in OpenAI-compatible client remain internal
|
||||||
|
// implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||||
// InspectProfile, Prepare,
|
// InspectRequirements, InspectProfile, InspectBackend, ListProfiles, ListPrompts,
|
||||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
// Prepare, PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||||
// bound admitted Run and RunPrepared calls and model generations where
|
// bound admitted Run and RunPrepared calls and model generations where
|
||||||
// configured, while different backend pools and unlimited backends continue
|
// configured, while different backend pools and unlimited backends continue
|
||||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||||
// still receive concurrent calls and must be safe for that use.
|
// still receive concurrent calls and must be safe for that use.
|
||||||
//
|
//
|
||||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||||
// Engine. Prepare,
|
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||||
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||||
// JSON-compatible extra parameters before using them. InspectPrompt and
|
// InspectRequirements, InspectProfile, and InspectBackend return copied
|
||||||
// InspectProfile return copied inspection values. Returned values and values
|
// inspection values.
|
||||||
// passed to extension interfaces are likewise isolated from engine state.
|
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||||
|
// Returned values and values passed to extension interfaces are likewise
|
||||||
|
// isolated from engine state.
|
||||||
// Callers own those copies and may mutate them after the call that supplied or
|
// Callers own those copies and may mutate them after the call that supplied or
|
||||||
// returned them. [CapacityError] values are caller-owned and may be mutated
|
// returned them. [CapacityError] values are caller-owned and may be mutated
|
||||||
// without affecting engine state or another error. Immutable [GenerationError]
|
// without affecting engine state or another error. Immutable [GenerationError]
|
||||||
@@ -54,16 +57,22 @@
|
|||||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
// used by those values.
|
// used by those values.
|
||||||
//
|
//
|
||||||
// Construction, inspection, handle, and error values, including [Config],
|
// Resource inspection results support direct JSON and YAML encoding.
|
||||||
|
// [PromptInspection], [ProfileInspection], [BackendInspection], and their nested
|
||||||
|
// values define field names, nesting, and omission of absent optional fields.
|
||||||
|
// Formatting belongs to the selected encoder.
|
||||||
|
//
|
||||||
|
// Other construction, inspection, handle, and error values, including [Config],
|
||||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||||
// [ProfileConfig], [RunRequest],
|
// [ProfileConfig], [RunRequest],
|
||||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
// [ArtifactRef], [ExecutionTargetOverride],
|
||||||
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
|
// [PromptIdentity], [PromptInputDefinition],
|
||||||
|
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||||
// [ExecutionEvent], do not have stable JSON
|
// [ExecutionEvent], do not have stable JSON
|
||||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||||
// every public value.
|
// every public value.
|
||||||
|
//
|
||||||
// Provider-derived [GenerationError] accessor values are untrusted and can
|
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||||
// contain sensitive request or schema fragments. Applications must apply their
|
// contain sensitive request or schema fragments. Applications must apply their
|
||||||
// own disclosure policy before logging, displaying, or returning them.
|
// own disclosure policy before logging, displaying, or returning them.
|
||||||
|
|||||||
235
docs/consumers/configuration-migration.md
Normal file
235
docs/consumers/configuration-migration.md
Normal file
@@ -0,0 +1,235 @@
|
|||||||
|
# Configuration And API Migration
|
||||||
|
|
||||||
|
This breaking pre-v1 minor migration adopts Promptkit's vanity module path,
|
||||||
|
changes the v0.16-era keyed resources to identified lists, adopts ConfigKit v1,
|
||||||
|
and removes prompt-version labels.
|
||||||
|
Promptkit installs a default-on bundled OpenRouter catalog; applications choose configuration
|
||||||
|
inputs explicitly, and execution selects one prompt by its exact ID. This guide
|
||||||
|
explains the shape changes and decisions an application must make; it is not an
|
||||||
|
automatic migration tool. The [framework format reference](../formats.md) owns
|
||||||
|
the current configuration contract, and the root package GoDoc owns exact API
|
||||||
|
signatures and errors.
|
||||||
|
|
||||||
|
## Update Go Imports
|
||||||
|
|
||||||
|
Promptkit's module and public package import path is now
|
||||||
|
`go.maximumdirect.net/promptkit`, replacing
|
||||||
|
`gitea.maximumdirect.net/eric/promptkit`. Update application, library, test, and
|
||||||
|
generated imports, then select a release using the new declaration:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@VERSION
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace `VERSION` with a release that declares the vanity module path.
|
||||||
|
Earlier tags, including v0.16.0, retain their original module declaration and
|
||||||
|
use the former import path. No permanent `replace` directive is needed for the
|
||||||
|
cutover. Source browser, licensing, and hosted-release links continue to use
|
||||||
|
the Forge repository URLs, independently of Go's module identifier.
|
||||||
|
|
||||||
|
The package name remains `promptkit`. The path change itself does not rename
|
||||||
|
exported APIs or change their behavior, but Go treats packages imported through
|
||||||
|
the two paths as distinct. Update dependent packages that exchange Promptkit
|
||||||
|
types together and review any old-path dependencies retained after tidying.
|
||||||
|
The configuration and API adaptations below are also part of this migration.
|
||||||
|
|
||||||
|
## Select And Compose The Configuration You Need
|
||||||
|
|
||||||
|
Supply each backend, profile, prompt, schema, and collection source that the
|
||||||
|
application intends to use. Promptkit does not discover directories or install
|
||||||
|
provider definitions implicitly. An application may select an explicit
|
||||||
|
collection input with `CollectionInput`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
input, err := promptkit.CollectionInput(
|
||||||
|
"openrouter",
|
||||||
|
configkit.File("config/openrouter.yaml"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{input}}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Collection resource IDs are local IDs, while their runtime identities are
|
||||||
|
qualified. For example, a profile with `id: gemini-flash-latest` in collection
|
||||||
|
`openrouter` is selected as `openrouter/gemini-flash-latest`. Stored backend,
|
||||||
|
prompt-default, and other resource references must use their exact intended IDs;
|
||||||
|
Promptkit does not rewrite references when a source is mounted. `extends`
|
||||||
|
selects a sibling ID in the same list, not a cross-collection reference.
|
||||||
|
Review collisions and all references when moving resources into collections.
|
||||||
|
|
||||||
|
The optional source-root `version` header now accepts `2` or `"2"`; headerless
|
||||||
|
sources remain valid. A collection's `release_version` remains an optional
|
||||||
|
label. Prompt-version labels have been removed: choose one definition per ID
|
||||||
|
and store any application-specific release label outside Promptkit. In a
|
||||||
|
standalone imported prompt item, root `version` is a source header, not a label.
|
||||||
|
|
||||||
|
For example, change a version-keyed definition:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prompts:
|
||||||
|
report:
|
||||||
|
"2.0":
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review this report.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
```
|
||||||
|
|
||||||
|
to one identified definition:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prompts:
|
||||||
|
- id: report
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review this report.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
```
|
||||||
|
|
||||||
|
The same conversion applies to backends and profiles. Sibling inheritance uses
|
||||||
|
identity references, and a later same-ID record replaces the whole lower record:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
profiles:
|
||||||
|
- id: base
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: example-model
|
||||||
|
- id: fast
|
||||||
|
extends: {id: base}
|
||||||
|
timeout_seconds: 30
|
||||||
|
```
|
||||||
|
|
||||||
|
To customize a lower resource's fields, add a distinct inherited variant as
|
||||||
|
above; when replacing the same ID, supply the complete intended record. Generic
|
||||||
|
append, replace, imports, and inheritance are described by the
|
||||||
|
[ConfigKit v1 configuration contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md).
|
||||||
|
Source-root controls are now `config:`, not `configkit:`. Imports are optional
|
||||||
|
unless the selected import declares `required: true`.
|
||||||
|
|
||||||
|
Update ConfigKit imports to `go.maximumdirect.net/configkit` at v1.0.0: its Go
|
||||||
|
types are incompatible with the old module path. Promptkit now requires Go
|
||||||
|
1.27.0 or newer. Source builders take ordered slices with explicit IDs:
|
||||||
|
|
||||||
|
```go
|
||||||
|
endpoint, model := "https://example.invalid/v1", "example-model"
|
||||||
|
base := "base"
|
||||||
|
source := promptkit.ProfileSource("application", []promptkit.ProfileConfig{
|
||||||
|
{ID: "base", Endpoint: &endpoint, Model: &model},
|
||||||
|
{ID: "fast", Extends: &base},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace former maps passed to `BackendSource` in the same way, setting each
|
||||||
|
`BackendConfig.ID`. Duplicates are retained for loading to reject. Resource
|
||||||
|
inspection paths now address list indexes, for example
|
||||||
|
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
||||||
|
and diffs expose resolved values, recorded normalization and provenance.
|
||||||
|
`InspectProfile` reports the selected resolved profile without backend completion.
|
||||||
|
Use `Prepare` or `PreparedExecution.Details` for completed request settings,
|
||||||
|
including backend defaults and request overrides; see the
|
||||||
|
[resource inspection guidance](pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
|
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
||||||
|
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
||||||
|
values and may also change because redundant captured fields were removed.
|
||||||
|
Invalidate persisted caches or comparisons that span this change; hashes from
|
||||||
|
inspection and preparation of the same captured definition still agree.
|
||||||
|
|
||||||
|
## Identity Mapping
|
||||||
|
|
||||||
|
The reviewed catalog identities have slash-free profile keys. Their documented
|
||||||
|
mapping is:
|
||||||
|
|
||||||
|
| Previous identity | Namespaced identity or action |
|
||||||
|
| --- | --- |
|
||||||
|
| Backend `openrouter` | `openrouter/default` |
|
||||||
|
| Backend `rakestrawhome` | `rakestrawhome/default` |
|
||||||
|
| Each OpenRouter catalog profile `p` | `openrouter/p`, retaining the exact local key `p` |
|
||||||
|
| Profile `rakestrawhome-gemma-4-31b` | `rakestrawhome/rakestrawhome-gemma-4-31b` |
|
||||||
|
| Built-in `promptkit-openrouter-standard` | No automatic replacement; optionally define caller profile `openrouter/standard` |
|
||||||
|
| Built-in `promptkit-openrouter-auto` | No automatic replacement; optionally define caller profile `openrouter/auto` |
|
||||||
|
|
||||||
|
These are conversion examples, not aliases or bundled resources. Mounting old
|
||||||
|
`/v3` assets does not update backend references, profile inheritance, prompt
|
||||||
|
defaults, or diagnostic defaults inside them. Edit those values to the exact
|
||||||
|
qualified IDs you chose.
|
||||||
|
|
||||||
|
For example, an optional caller-owned standard profile may extend a selected
|
||||||
|
Gemini profile in the same `openrouter` collection:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
collections:
|
||||||
|
openrouter:
|
||||||
|
profiles:
|
||||||
|
- id: standard
|
||||||
|
extends: {id: gemini-flash-latest}
|
||||||
|
```
|
||||||
|
|
||||||
|
An optional auto profile may instead select the exact backend and model values:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
collections:
|
||||||
|
openrouter:
|
||||||
|
profiles:
|
||||||
|
- id: auto
|
||||||
|
backend: openrouter/default
|
||||||
|
model: openrouter/auto
|
||||||
|
```
|
||||||
|
|
||||||
|
Neither optional profile name is a Promptkit contract. Keep provider model
|
||||||
|
strings and settings as selected by the application, including any meaningful
|
||||||
|
prefixes. If a backend declares diagnostic defaults, update them to exact
|
||||||
|
qualified IDs; an unselected diagnostic default is not automatically made
|
||||||
|
resolvable by migration.
|
||||||
|
|
||||||
|
## Prompt Selection API
|
||||||
|
|
||||||
|
Prompt selection is ID-only. Remove version selectors from calls to `Run`,
|
||||||
|
`Prepare`, `PrepareExecution`, `ValidateRequest`, `InspectRequirements`, and
|
||||||
|
`SmokeTest`. Remove the `PromptVersion` field from `RunRequest`,
|
||||||
|
`RequirementInspectionRequest`, and `SmokeTestRequest`; `InspectPrompt` now
|
||||||
|
has the form `InspectPrompt(ctx, promptID)`. For example, select the prompt
|
||||||
|
explicitly in the request rather than setting a version selector:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "support/report",
|
||||||
|
ProfileID: "support/standard",
|
||||||
|
}
|
||||||
|
_, err = engine.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
If the former format contained multiple versions under one ID, choose the one
|
||||||
|
definition the application should install. Promptkit no longer retains or
|
||||||
|
returns its former descriptive label.
|
||||||
|
|
||||||
|
The public inventory APIs can help an operator inspect a captured effective
|
||||||
|
configuration: `ListProfiles`, `ListPrompts`, and `ListBackends` return exact
|
||||||
|
identities, while `ListNamespaces` and `ListCollections` expose resource-bearing
|
||||||
|
namespaces and effective collection declarations. Inventory does not select
|
||||||
|
which configuration version the application should deploy.
|
||||||
|
|
||||||
|
## Deployment Selection Remains With The Application
|
||||||
|
|
||||||
|
Applications decide which checked-in, embedded, operator-supplied, or
|
||||||
|
programmatic inputs to compose for each deployment. They also decide when to
|
||||||
|
construct a replacement engine after changing those sources. Promptkit does
|
||||||
|
not fetch catalog updates, select provider availability, validate upstream
|
||||||
|
catalog release status, or automatically migrate arbitrary consumer files.
|
||||||
|
The maintained [offline collection example](../../examples/go-library/prepare/main.go)
|
||||||
|
shows explicit source selection, local imports, layering, a scoped view, and
|
||||||
|
operator inventory without a provider call.
|
||||||
@@ -12,19 +12,72 @@ file contracts.
|
|||||||
Import the package as:
|
Import the package as:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
import "gitea.maximumdirect.net/eric/promptkit"
|
import "go.maximumdirect.net/promptkit"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
For migration from the former module path, see
|
||||||
|
[Update Go Imports](configuration-migration.md#update-go-imports).
|
||||||
|
|
||||||
|
Configuration APIs use `go.maximumdirect.net/configkit` v1.1.0. Its types are
|
||||||
|
not interchangeable with the old ConfigKit module path.
|
||||||
|
|
||||||
|
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
||||||
|
demonstrates explicit collection composition and inventory. Applications
|
||||||
|
adapting older provider or versioned-prompt configuration can use the
|
||||||
|
[configuration migration guide](configuration-migration.md) for the required
|
||||||
|
caller-owned choices.
|
||||||
|
|
||||||
The following Go fragments are illustrative and omit surrounding package,
|
The following Go fragments are illustrative and omit surrounding package,
|
||||||
import, and error-handling code. Use the maintained examples for complete
|
import, and error-handling code. Use the maintained examples for complete
|
||||||
programs.
|
programs.
|
||||||
|
|
||||||
|
## Go Compatibility
|
||||||
|
|
||||||
|
The `go` directive in the selected release's [module metadata](../../go.mod)
|
||||||
|
declares its minimum supported consumer Go version, currently Go 1.27.0.
|
||||||
|
Maintainers build and
|
||||||
|
validate releases with the locally selected Go patch release; the minimum
|
||||||
|
does not simply track that patch version.
|
||||||
|
|
||||||
|
Successful release validation on the selected patch is sufficient to presume
|
||||||
|
support across its Go series, subject to the declared minimum and concrete
|
||||||
|
evidence of incompatibility. Specific compatibility concerns are investigated
|
||||||
|
as they arise. Minor releases may raise the minimum; patch releases preserve
|
||||||
|
it. Release notes explain increases and any required consumer action. See the
|
||||||
|
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
|
||||||
|
|
||||||
## Construct An Engine
|
## Construct An Engine
|
||||||
|
|
||||||
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
||||||
explicitly and place its inputs in an ordered ConfigKit underlay. A group
|
explicitly and place its inputs in an ordered ConfigKit underlay. A group
|
||||||
strictly composes its files; later groups can overlay earlier ones:
|
strictly composes its files; later groups can overlay earlier ones:
|
||||||
|
|
||||||
|
The bundled OpenRouter backend and profiles are installed in a lower underlay
|
||||||
|
by default. Select a profile explicitly, for example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
||||||
|
"prompts": []any{map[string]any{
|
||||||
|
"id": "my-prompt",
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
|
if err != nil { return err }
|
||||||
|
result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "my-prompt", ProfileID: "gpt-luna-latest"})
|
||||||
|
```
|
||||||
|
|
||||||
|
The backend looks up
|
||||||
|
`OPENROUTER_API_KEY` when available; credentials are not required at engine
|
||||||
|
construction. For the complete inventory and diagnostic defaults see the
|
||||||
|
[format reference](../formats.md#bundled-openrouter-configuration). Use
|
||||||
|
`promptkit.WithBundledOpenRouter(false)` for application-only inventories.
|
||||||
|
Caller records replace matching bundled records in full; use `extends` with a
|
||||||
|
distinct ID to inherit. Replacing `auto` also changes its cost-tier children's
|
||||||
|
inherited target, while their plugins remain. External collections remain
|
||||||
|
supported.
|
||||||
|
|
||||||
```go
|
```go
|
||||||
promptInputs, err := promptkit.DirectoryInputs("prompts")
|
promptInputs, err := promptkit.DirectoryInputs("prompts")
|
||||||
if err != nil { return err }
|
if err != nil { return err }
|
||||||
@@ -41,7 +94,7 @@ engine, err := promptkit.NewEngine(promptkit.Config{
|
|||||||
|
|
||||||
For a single highest-priority document, set `Sources.Main` to a
|
For a single highest-priority document, set `Sources.Main` to a
|
||||||
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
||||||
can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
|
can use `configkit.File` directly. `PromptSource`, `ProfileSource`, `BackendSource`, and raw
|
||||||
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
||||||
collaborator options remain available. Construction validates the final
|
collaborator options remain available. Construction validates the final
|
||||||
effective configuration and captures referenced prompt content; construct a
|
effective configuration and captures referenced prompt content; construct a
|
||||||
@@ -49,6 +102,34 @@ new engine to observe configuration edits. See the [constructor GoDoc](../../eng
|
|||||||
for the exact public contract and the [framework format reference](../formats.md)
|
for the exact public contract and the [framework format reference](../formats.md)
|
||||||
for definition fields.
|
for definition fields.
|
||||||
|
|
||||||
|
### Use Scoped Engine Views
|
||||||
|
|
||||||
|
Create a view when an application wants short request IDs to resolve within a
|
||||||
|
selected collection:
|
||||||
|
|
||||||
|
```go
|
||||||
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||||
|
Prompt: "support",
|
||||||
|
Profile: "support",
|
||||||
|
})
|
||||||
|
if err != nil { return err }
|
||||||
|
|
||||||
|
prepared, err := scoped.Prepare(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "answer",
|
||||||
|
ProfileID: "concise",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Execution, preparation, prompt/profile inspection, requirement inspection, and
|
||||||
|
smoke diagnostics qualify slashless caller selections with their respective
|
||||||
|
namespaces. Already-qualified IDs pass through unchanged; configured defaults
|
||||||
|
and references are not rewritten. Each call replaces the view's scope defaults,
|
||||||
|
including clearing a scope with an empty field, without changing the receiver
|
||||||
|
or rebuilding its captured configuration and execution collaborators.
|
||||||
|
Namespace syntax is checked when creating the view; resource existence is
|
||||||
|
checked when selected. Configuration inspection remains global. See
|
||||||
|
[`WithNamespaces`](../../namespaces.go) for the exact public contract.
|
||||||
|
|
||||||
## Supply Embedded Application Defaults
|
## Supply Embedded Application Defaults
|
||||||
|
|
||||||
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
|
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
|
||||||
@@ -60,8 +141,8 @@ defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS,
|
|||||||
if err != nil { return err }
|
if err != nil { return err }
|
||||||
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
operator, err := promptkit.DirectoryInputs(operatorDirectory)
|
||||||
if err != nil { return err }
|
if err != nil { return err }
|
||||||
profile := promptkit.ProfileSource("application choices", map[string]promptkit.ProfileConfig{
|
profile := promptkit.ProfileSource("application choices", []promptkit.ProfileConfig{
|
||||||
"summary": {Model: &selectedModel},
|
{ID: "summary", Endpoint: &selectedEndpoint, Model: &selectedModel},
|
||||||
})
|
})
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{
|
Underlays: []configkit.Underlay{
|
||||||
@@ -72,7 +153,9 @@ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.Configura
|
|||||||
}})
|
}})
|
||||||
```
|
```
|
||||||
|
|
||||||
Only add the operator group when the application selects that directory; an
|
Each later same-ID resource replaces the complete earlier record; use a distinct
|
||||||
|
inherited sibling for field-level customization. Only add the operator group
|
||||||
|
when the application selects that directory; an
|
||||||
explicitly selected missing or unreadable root is an error. Each helper finds
|
explicitly selected missing or unreadable root is an error. Each helper finds
|
||||||
`.yaml` and `.yml` files recursively and disables imports by default. The
|
`.yaml` and `.yml` files recursively and disables imports by default. The
|
||||||
caller may enable imports on individual inputs. A nil `Sources.Main` is an
|
caller may enable imports on individual inputs. A nil `Sources.Main` is an
|
||||||
@@ -83,8 +166,8 @@ labeled mapping as an input. A mapping owns a snapshot of supported values:
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
document := configkit.Mapping("application configuration", map[string]any{
|
document := configkit.Mapping("application configuration", map[string]any{
|
||||||
"profiles": map[string]any{
|
"profiles": []any{
|
||||||
"local-summary": map[string]any{
|
map[string]any{"id": "local-summary",
|
||||||
"backend": "local", "model": "example-model",
|
"backend": "local", "model": "example-model",
|
||||||
"temperature": nil, // Suppress an inherited provider control.
|
"temperature": nil, // Suppress an inherited provider control.
|
||||||
},
|
},
|
||||||
@@ -94,9 +177,18 @@ input := configkit.Input{Source: document}
|
|||||||
// Add input to a later underlay, or set Sources.Main = &input.
|
// Add input to a later underlay, or set Sources.Main = &input.
|
||||||
```
|
```
|
||||||
|
|
||||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
`PromptSource`, `ProfileSource`, and `BackendSource` are typed conveniences when
|
||||||
is not needed. A caller underlay can overlay catalog backend fields just as it
|
null is not needed. Prompt and backend helpers, like profiles, support sibling
|
||||||
can overlay profiles; there is no registration-only backend namespace.
|
inheritance via `Extends`; partial records can be completed by other layers.
|
||||||
|
For relative prompt `content_file` references, supply the ordinary ConfigKit
|
||||||
|
`Input.Filesystem` context for that source. The helpers do not read files or
|
||||||
|
validate final definitions. Profile and backend provider maps use ConfigKit Mapping input forms: built-in
|
||||||
|
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
|
||||||
|
typed containers explicitly to an appropriate supported value, choosing numeric
|
||||||
|
precision for your application. Mapping captures supported values immediately;
|
||||||
|
unsupported or cyclic values are reported when loaded. Runtime override values
|
||||||
|
follow their separate rules. Backend and profile definitions are supplied by the caller;
|
||||||
|
there is no registration-only backend namespace.
|
||||||
|
|
||||||
When configuration or message files change, build and validate a new engine,
|
When configuration or message files change, build and validate a new engine,
|
||||||
then switch the application to that engine for subsequent work. Existing
|
then switch the application to that engine for subsequent work. Existing
|
||||||
@@ -104,17 +196,65 @@ engines and prepared handles retain their captured definitions; schema and
|
|||||||
request artifact resources still follow their runtime lifecycle. Inspection
|
request artifact resources still follow their runtime lifecycle. Inspection
|
||||||
does not watch or reload sources automatically.
|
does not watch or reload sources automatically.
|
||||||
|
|
||||||
|
## Discover Available Resources And Collections
|
||||||
|
|
||||||
|
Use [`Engine.ListProfiles`](../../listing.go),
|
||||||
|
[`Engine.ListPrompts`](../../listing.go), and
|
||||||
|
[`Engine.ListBackends`](../../listing.go) to discover exact identities from the
|
||||||
|
engine's final captured configuration. A scoped view filters by its respective
|
||||||
|
namespace by default; pass `NamespaceNamed`, `NamespaceUnqualified`, or
|
||||||
|
`NamespaceAll` to override that listing filter without changing execution
|
||||||
|
scope. Returned IDs remain full identities and can be used for exact selection:
|
||||||
|
|
||||||
|
```go
|
||||||
|
profiles, err := engine.ListProfiles(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(profiles) > 0 {
|
||||||
|
_, err := engine.InspectProfile(ctx, profiles[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(prompts) > 0 {
|
||||||
|
selected := prompts[0]
|
||||||
|
inspection, err := engine.InspectPrompt(ctx, selected.ID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = inspection
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Use [`Engine.ListNamespaces`](../../listing.go) to discover resource-bearing
|
||||||
|
namespaces across the entire captured configuration, independent of view
|
||||||
|
scopes. [`Engine.ListCollections`](../../listing.go) reports effective
|
||||||
|
collection declarations, including metadata-only entries; it does not infer
|
||||||
|
collection metadata from resource membership or source history.
|
||||||
|
|
||||||
|
These lists describe effective membership, not ConfigKit contribution history,
|
||||||
|
and do not promise that a profile is ready to execute. Use inspection for
|
||||||
|
details and preparation to resolve request resources. The linked method GoDoc
|
||||||
|
owns ordering, ownership, concurrency, and error behavior; identifiers may be
|
||||||
|
sensitive application data.
|
||||||
|
|
||||||
## Inspect The Captured Configuration
|
## Inspect The Captured Configuration
|
||||||
|
|
||||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||||
operator-selected report of effective configuration, source descriptions,
|
operator-selected report of resolved configuration, source descriptions,
|
||||||
diagnostics, and path history. For a literal ID containing a slash, construct
|
diagnostics, and path history. Configuration inspection uses list indexes in
|
||||||
the semantic path with `configkit.Keys`:
|
the collection's nested path rather than qualified runtime IDs. For a profile
|
||||||
|
at index `i` in collection `team`, construct the path with `configkit.Keys`:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil { return err }
|
if err != nil { return err }
|
||||||
path := configkit.Keys("profiles", "team/summary", "model")
|
path := configkit.Keys("collections", "team", "profiles").Index(i).Key("model")
|
||||||
explanation, found := view.Explain(path)
|
explanation, found := view.Explain(path)
|
||||||
if found && explanation.Effective.Redacted {
|
if found && explanation.Effective.Redacted {
|
||||||
// Apply the application's disclosure policy before sharing the report.
|
// Apply the application's disclosure policy before sharing the report.
|
||||||
@@ -125,10 +265,113 @@ if found && explanation.Effective.Redacted {
|
|||||||
the application decides whether those reports may be disclosed. Credential-
|
the application decides whether those reports may be disclosed. Credential-
|
||||||
shaped provider keys are redacted, but arbitrary prompt and provider content
|
shaped provider keys are redacted, but arbitrary prompt and provider content
|
||||||
is not necessarily safe to share. Inspection uses the engine's captured result
|
is not necessarily safe to share. Inspection uses the engine's captured result
|
||||||
and does not reopen sources or resolve credentials. `InspectProfile` instead
|
and does not reopen sources or resolve credentials. It retains effective values,
|
||||||
shows backend-completed execution settings for one profile, and `Prepare` or
|
including recorded field normalization and queue defaulting, alongside source
|
||||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
provenance.
|
||||||
and validation resources. None of those views is a substitute for the others.
|
`InspectPrompt`, `InspectProfile`, and `InspectBackend` instead show individual
|
||||||
|
resolved resources without backend-to-profile execution completion or provenance.
|
||||||
|
`Prepare` or `PreparedExecution.Details` shows completed request-specific settings,
|
||||||
|
rendered messages, inputs, and validation resources. None substitutes for the others.
|
||||||
|
|
||||||
|
To review changes between two independently captured configurations, compare
|
||||||
|
their inspection views:
|
||||||
|
|
||||||
|
```go
|
||||||
|
earlier, err := earlierEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
later, err := candidateEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
differences, err := earlier.Diff(later)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, difference := range differences {
|
||||||
|
if difference.HiddenChanged {
|
||||||
|
// Report a protected change without inferring its value.
|
||||||
|
}
|
||||||
|
if difference.Left.Redacted || difference.Right.Redacted {
|
||||||
|
// Apply the application's disclosure policy before presenting snapshots.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||||
|
and ConfigKit's [v1.1.0 effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md#effective-configuration-diffs)
|
||||||
|
for the exact comparison contract. The operation compares only frozen effective
|
||||||
|
configuration values: it does not reread external files or compare their
|
||||||
|
contents, resolved credential contents, later execution preparation, or
|
||||||
|
request-specific overrides. Inputs normalizing to the same effective value do
|
||||||
|
not create a value diff; lists are compared atomically. An empty diff is not
|
||||||
|
proof of equivalent execution.
|
||||||
|
Records can contain ordinary application data, so use the application's
|
||||||
|
disclosure policy when presenting them.
|
||||||
|
|
||||||
|
## Run Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
[`Engine.SmokeTest`](../../smoke_diagnostics.go) runs one explicit diagnostic
|
||||||
|
using a target resolved from the engine's captured configuration. It does not
|
||||||
|
execute the selected application prompt or its inputs, session, artifacts, or
|
||||||
|
output schema. Unlike the offline examples and default test suite, calling it
|
||||||
|
can contact the selected provider and use configured credentials. It is a
|
||||||
|
consumer operation, not an automatic readiness verdict or a release check.
|
||||||
|
|
||||||
|
For example, a caller can request the built-in text inference check and treat
|
||||||
|
its optional fact as an observation rather than assuming that a nil Go error
|
||||||
|
means success:
|
||||||
|
|
||||||
|
```go
|
||||||
|
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{
|
||||||
|
Kind: promptkit.SmokeTestTextInference,
|
||||||
|
ProfileID: profileID,
|
||||||
|
APIKey: apiKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if report != nil {
|
||||||
|
retainDisclosureSafeSummary(report)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if report != nil {
|
||||||
|
matched := report.Facts.ExpectedTextMatched
|
||||||
|
switch {
|
||||||
|
case matched == nil:
|
||||||
|
// The comparison was not evaluated or is unknown/inapplicable.
|
||||||
|
case *matched:
|
||||||
|
// The private expected-text comparison matched.
|
||||||
|
default:
|
||||||
|
// It did not match; the operation may still have concluded normally.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
A nil Go error means the diagnostic concluded, not that its finding was
|
||||||
|
positive. Fact pointers are tri-state: nil means unknown, not evaluated, or
|
||||||
|
inapplicable. Reports contain factual observations and effective settings, not
|
||||||
|
generated text or provider bodies; identifiers and settings can still be
|
||||||
|
sensitive. The
|
||||||
|
[`SmokeTest` GoDoc](../../smoke_diagnostics.go) owns selection precedence,
|
||||||
|
exact fields, timeout and retry behavior, supported transports, and disclosure
|
||||||
|
details.
|
||||||
|
|
||||||
|
Caller cancellation can return both an error and a partial report. Preserve the
|
||||||
|
context error identity and inspect any available observations before returning:
|
||||||
|
|
||||||
|
```go
|
||||||
|
report, err := engine.SmokeTest(ctx, request)
|
||||||
|
if report != nil {
|
||||||
|
retainDisclosureSafeSummary(report)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
// The report may describe work completed before interruption.
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## Inspect A Prompt Before Preparation
|
## Inspect A Prompt Before Preparation
|
||||||
|
|
||||||
@@ -137,14 +380,16 @@ declared inputs and output workflow without creating placeholder inputs or
|
|||||||
resolving a profile:
|
resolving a profile:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
|
inspection, err := engine.InspectPrompt(ctx, "meeting.summary")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, input := range inspection.Inputs {
|
if inspection.Inputs != nil {
|
||||||
|
for _, input := range *inspection.Inputs {
|
||||||
// Compare the declared input with application configuration.
|
// Compare the declared input with application configuration.
|
||||||
}
|
}
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Use this configuration-time boundary when the application needs only the
|
Use this configuration-time boundary when the application needs only the
|
||||||
@@ -154,6 +399,85 @@ rendered messages, and use prepared execution when that work must remain tied
|
|||||||
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
|
||||||
hash, ownership, and error semantics.
|
hash, ownership, and error semantics.
|
||||||
|
|
||||||
|
## Inspect Requirements Before Preparation
|
||||||
|
|
||||||
|
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
|
||||||
|
needs the prompt's recognizable input and caller-variable dependencies for a
|
||||||
|
particular context before it has input artifacts. Unlike `InspectPrompt`, it
|
||||||
|
also walks captured template syntax and selects a profile identity; unlike
|
||||||
|
`Prepare`, it does not load artifacts or render messages.
|
||||||
|
|
||||||
|
```go
|
||||||
|
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Vars: map[string]string{"audience": "project leads"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, input := range requirements.Inputs {
|
||||||
|
if input.Kind == promptkit.RequirementRequired {
|
||||||
|
// Plan to bind this input before preparation.
|
||||||
|
}
|
||||||
|
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
|
||||||
|
// A recognizable reference may occur only on a known skipped path.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, variable := range requirements.Variables {
|
||||||
|
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
|
||||||
|
// The key is required even though this inspection did not receive it.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !requirements.Complete {
|
||||||
|
// Preserve the partial inventory and account for its discovery issues.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Optional entries can include recognizable inputs and variables found only in
|
||||||
|
inactive branches. A missing required variable is reported separately from
|
||||||
|
discovery completeness: missing a value used only for direct text substitution
|
||||||
|
does not by itself make the inventory incomplete. Conversely, an unresolved
|
||||||
|
dependency-bearing branch can make discovery incomplete while retaining known
|
||||||
|
conditional entries. Do not treat an empty partial inventory as proof that no
|
||||||
|
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
|
||||||
|
the exact classifications, supported syntax, ownership, and error contract.
|
||||||
|
|
||||||
|
## Validate A Concrete Request
|
||||||
|
|
||||||
|
Use [`Engine.ValidateRequest`](../../engine.go) when the application has a
|
||||||
|
complete `RunRequest` and needs to know whether that concrete invocation passes
|
||||||
|
preparation:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Vars: map[string]string{"audience": "project leads"},
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := engine.ValidateRequest(ctx, request); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Unlike `InspectRequirements`, this checks supplied values, the selected template
|
||||||
|
branches, and request bindings through the existing preparation path. It checks
|
||||||
|
only this invocation, does not aggregate errors, and loads every supplied input
|
||||||
|
according to ordinary preparation behavior, even if a branch does not use it.
|
||||||
|
It makes no model call and returns no prepared snapshot, but artifact and schema
|
||||||
|
resource I/O and its time and memory costs remain. Success does not certify
|
||||||
|
provider acceptance or later execution success. See the method's GoDoc for its
|
||||||
|
exact error and context behavior.
|
||||||
|
|
||||||
|
Scriptorium can map the returned error to its own diagnostics and exit status.
|
||||||
|
Its `validate` and `run` commands should construct equivalent requests for
|
||||||
|
equivalent caller inputs so their preparation behavior agrees. Separate
|
||||||
|
subprocess calls prepare independently; an opaque `PrepareExecution` handle
|
||||||
|
cannot be transferred from `validate` to `run`. Use `PrepareExecution` when one
|
||||||
|
process and engine must execute the exact prepared snapshot.
|
||||||
|
|
||||||
## Prepare Without Model Execution
|
## Prepare Without Model Execution
|
||||||
|
|
||||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||||
@@ -174,8 +498,11 @@ prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
|||||||
The maintained
|
The maintained
|
||||||
[offline preparation example](../../examples/go-library/prepare/main.go)
|
[offline preparation example](../../examples/go-library/prepare/main.go)
|
||||||
shows a complete runnable setup with a prompt file, in-memory profile, and
|
shows a complete runnable setup with a prompt file, in-memory profile, and
|
||||||
inline input. Exact request requirements and prepared-result fields belong to
|
inline input. It demonstrates validation and preparation with the same request
|
||||||
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
to make the two operations concrete; this performs preparation twice and is
|
||||||
|
illustrative, not a requirement to validate before `Prepare` or `Run`. Exact
|
||||||
|
request requirements and prepared-result fields belong to the
|
||||||
|
[`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||||
|
|
||||||
## Prepare Now And Execute The Same Snapshot Later
|
## Prepare Now And Execute The Same Snapshot Later
|
||||||
|
|
||||||
@@ -345,7 +672,7 @@ copy, and credential behavior. The
|
|||||||
interact with prompt definitions, layered profiles, built-ins, schemas, and
|
interact with prompt definitions, layered profiles, built-ins, schemas, and
|
||||||
framework defaults.
|
framework defaults.
|
||||||
|
|
||||||
For programmatic profiles, use [`ProfileSource`](../../profiles.go) with keyed
|
For programmatic profiles, use [`ProfileSource`](../../profiles.go) with identified
|
||||||
`ProfileConfig` values. Nil fields are omitted; pointers preserve explicit
|
`ProfileConfig` values. Nil fields are omitted; pointers preserve explicit
|
||||||
zero, false, and blank values. Raw `configkit.Mapping` expresses explicit null.
|
zero, false, and blank values. Raw `configkit.Mapping` expresses explicit null.
|
||||||
|
|
||||||
@@ -358,107 +685,38 @@ and `WithBackend` options have no compatibility wrappers. Use
|
|||||||
`DirectoryInputs` or `FilesystemInputs` for a strict-composition file group,
|
`DirectoryInputs` or `FilesystemInputs` for a strict-composition file group,
|
||||||
ConfigKit file inputs for individual files, and `ProfileSource`,
|
ConfigKit file inputs for individual files, and `ProfileSource`,
|
||||||
`BackendSource`, or `configkit.Mapping` for programmatic values. Backend IDs
|
`BackendSource`, or `configkit.Mapping` for programmatic values. Backend IDs
|
||||||
are keys in configuration, not registrations. Choose caller underlays in
|
are fields in configuration, not registrations. Choose caller underlays in
|
||||||
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
|
fallback-to-operator-to-programmatic order; use `Sources.Main` for one final
|
||||||
highest-priority input.
|
highest-priority input.
|
||||||
|
|
||||||
Move embedded `id` and `version` fields into keyed `profiles` and `prompts`
|
Move resource IDs into identified `backends`, `profiles`, and `prompts` lists;
|
||||||
maps; convert `base_profile` to ConfigKit `extends`. IDs and references are
|
each prompt ID identifies one installed definition without a version label.
|
||||||
exact literal keys, without trimming. A supplied zero, false, or empty string
|
Convert `base_profile` to ConfigKit `extends: {id: parent}`. For the current
|
||||||
is now distinguishable from omission; null can clear or suppress an inherited
|
collection identity mapping, ID-only prompt selection, and required
|
||||||
field according to the [field table](../formats.md#defaults-and-overrides).
|
caller-owned choices, see the
|
||||||
Maps merge recursively across inheritance and layers. `{}` does not clear an
|
[configuration migration guide](configuration-migration.md). Exact current
|
||||||
inherited map: use null to clear the whole map. At request time, a literal
|
format, reference, inheritance, and presence semantics belong to the
|
||||||
`extra_params["extends"]` is ordinary provider data, not an inheritance
|
[framework format reference](../formats.md). Prompt text and static templates
|
||||||
instruction; a nonempty request ExtraParams map replaces configured params.
|
are captured at construction, not automatically reloaded between runs. Replace
|
||||||
Changing a model or backend does not clear unrelated explicit fields, though
|
the engine to pick up changed definitions or message files.
|
||||||
backend-derived defaults are applied after inheritance to the final backend.
|
|
||||||
|
|
||||||
Invalid final definitions, including unused ones, now fail `NewEngine`; only
|
|
||||||
request-dependent failures remain at preparation or execution. Prompt text
|
|
||||||
and static templates are captured at construction, not automatically reloaded
|
|
||||||
between runs. Replace the engine to pick up changed definitions or message
|
|
||||||
files. The [format reference](../formats.md) owns exact field and presence
|
|
||||||
rules; the maintained examples show complete runnable workflows.
|
|
||||||
|
|
||||||
### Choose A Profile Ownership Model
|
### Choose A Profile Ownership Model
|
||||||
|
|
||||||
Choose the profile source that matches the application's continuity needs:
|
Supply backend and profile definitions through the configuration sources your
|
||||||
|
application selects, alongside the default-on bundled OpenRouter inventory.
|
||||||
- Select `promptkit-openrouter-standard` or `promptkit-openrouter-auto` when
|
Conventional backend ID constants are names only. Define or replace the model,
|
||||||
the maintained literal name is the desired consumer contract. Their current
|
endpoint, credentials lookup, and settings that your application
|
||||||
selections are in the [format reference](../formats.md#built-in-profile-catalog).
|
intends to support. A profile may inherit a sibling profile by its local key in
|
||||||
- Select an external catalog profile directly when the application accepts that
|
the same root or collection list; its backend reference uses the exact
|
||||||
profile's catalog volatility. The selected external inventories are linked
|
configured identity. Caller-selected source layers determine the final values. The
|
||||||
from the [format reference](../formats.md#external-profile-inventories).
|
[format reference](../formats.md) describes the definition fields and
|
||||||
- Define a complete application-owned profile with an explicit backend or
|
composition rules.
|
||||||
endpoint, model, and settings when the application owns those choices.
|
|
||||||
|
|
||||||
An application profile that inherits an external base follows later changes to
|
|
||||||
that base and fails if the base disappears; it is not a frozen copy. Promptkit
|
|
||||||
tests the catalog versions selected in its module graph. Independently advancing
|
|
||||||
a catalog dependency can remove an inherited base and produce an untested
|
|
||||||
combination. Pinning the resolved module graph preserves the embedded data for
|
|
||||||
a build, but cannot preserve provider availability. Running binaries neither
|
|
||||||
download catalog updates nor automatically replace withdrawn models. See the
|
|
||||||
[release policy](../policy/release.md#catalog-profile-compatibility) for the exact
|
|
||||||
cross-version guarantees.
|
|
||||||
|
|
||||||
### Alias A Built-In Profile
|
|
||||||
|
|
||||||
Give an application-owned profile ID a built-in base when prompts should select
|
|
||||||
the application ID while inheriting the built-in target. The child can override
|
|
||||||
only the setting it owns:
|
|
||||||
|
|
||||||
```go
|
|
||||||
profile := promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
|
|
||||||
"weather-light": {
|
|
||||||
Extends: &standardID,
|
|
||||||
ReasoningEffort: &high,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
Place `profile` in a ConfigKit underlay, with `standardID` set to
|
|
||||||
`"promptkit-openrouter-standard"` and `high` to `"high"`. Select
|
|
||||||
`weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
|
||||||
reported selected profile. Definition IDs and inheritance references are exact
|
|
||||||
keys. If the base is an external profile, apply the volatility guidance
|
|
||||||
above when deciding whether to use an application-owned complete definition
|
|
||||||
instead.
|
|
||||||
|
|
||||||
### Use The Rakestrawhome Built-In Profile
|
|
||||||
|
|
||||||
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
|
||||||
select `rakestrawhome-gemma-4-31b` as an ordinary profile ID. For example, a
|
|
||||||
prepared result identifies the selected built-in through
|
|
||||||
`BackendRakestrawHome`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
|
||||||
PromptID: "meeting.summary",
|
|
||||||
ProfileID: "rakestrawhome-gemma-4-31b",
|
|
||||||
Inputs: inputs,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
|
|
||||||
return fmt.Errorf("unexpected backend %q", prepared.SelectedBackendID)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The maintained catalog supplies this backend, and callers may overlay its
|
|
||||||
effective configuration through a later ConfigKit source. Direct request credentials
|
|
||||||
and runtime endpoint overrides remain supported under their ordinary GoDoc and
|
|
||||||
format contracts.
|
|
||||||
|
|
||||||
### Inspect A Profile Before Prompt Work
|
### Inspect A Profile Before Prompt Work
|
||||||
|
|
||||||
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
Use [`Engine.InspectProfile`](../../engine.go) to inspect a resolved profile
|
||||||
profile without constructing a synthetic prompt or placeholder inputs. It
|
without constructing a synthetic prompt or placeholder inputs. It reports the
|
||||||
resolves the profile's effective target but does not prepare or execute a
|
profile's own fields, not a backend-completed target:
|
||||||
prompt:
|
|
||||||
|
|
||||||
```go
|
```go
|
||||||
inspection, err := engine.InspectProfile(ctx, profileID)
|
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||||
@@ -466,23 +724,48 @@ if err != nil {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
if inspection.APIKeyRequired != nil && *inspection.APIKeyRequired {
|
||||||
if target.APIKeyEnv != "" {
|
// Arrange a usable direct or configured environment credential.
|
||||||
// This is a configured optional environment lookup source.
|
}
|
||||||
} else if inspection.APIKeyRequired {
|
if inspection.APIKeyEnv != nil {
|
||||||
// Arrange a direct credential before later execution.
|
// This is the configured environment lookup source, not its value.
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Use this configuration-time boundary when only the profile and its target need
|
Use [`InspectBackend`](../../engine.go) to inspect the referenced backend
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
separately. Its diagnostics are configured metadata, not live checks or derived
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
destinations; `ErrBackendNotFound` identifies an absent backend. References in
|
||||||
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
all three resource reports remain as authored. Use `Prepare` for completed
|
||||||
`APIKeyRequired` is the explicit local requirement. The
|
request settings and rendering, or prepared execution when that work must remain
|
||||||
|
tied to later execution. A reported `APIKeyEnv` identifies a configured source,
|
||||||
|
not its value; `APIKeyRequired` can also be present. The
|
||||||
[credential format reference](../formats.md#credentials) and the method's
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
contracts.
|
contracts.
|
||||||
|
|
||||||
|
Resource inspection pointers represent optional fields: nil is absent, while a
|
||||||
|
non-nil pointer preserves zero, false, or an empty string. Pointer-to-collection
|
||||||
|
fields distinguish omission from an empty map or list. Results and nested mutable
|
||||||
|
values belong to the caller. Ordinary generic encoders need no per-field adapter:
|
||||||
|
|
||||||
|
```go
|
||||||
|
jsonBytes, err := json.Marshal(inspection)
|
||||||
|
if err != nil { return err }
|
||||||
|
yamlBytes, err := yaml.Marshal(inspection) // go.yaml.in/yaml/v4
|
||||||
|
if err != nil { return err }
|
||||||
|
_, _ = jsonBytes, yamlBytes
|
||||||
|
```
|
||||||
|
|
||||||
|
For migration, replace `ProfileInspection.EffectiveModelParams` accesses with
|
||||||
|
the optional resource fields, and use `Prepare` or `PreparedExecution.Details`
|
||||||
|
where backend completion and request overrides are needed. Prompt inspection's
|
||||||
|
`DefaultProfileID`, `Inputs`, nested input metadata, and output fields now use
|
||||||
|
optional pointers where presence matters; check for nil before dereferencing.
|
||||||
|
Consumers such as CommandKit can render the results directly instead of copying
|
||||||
|
each field into a parallel report. See the result types' [GoDoc](../../resource_inspection.go)
|
||||||
|
for keys and ownership; resource inspection does not replace `InspectConfiguration`
|
||||||
|
provenance, `InspectRequirements`, or concrete-request `ValidateRequest`.
|
||||||
|
|
||||||
### Set A Per-Run Session And Reasoning
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
Supply a direct session ID when one prompt should be correlated with a
|
Supply a direct session ID when one prompt should be correlated with a
|
||||||
@@ -521,8 +804,8 @@ it and shared backend identity or capacity policy is unnecessary:
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
endpoint, model := "http://localhost:8000/v1", "example-model"
|
endpoint, model := "http://localhost:8000/v1", "example-model"
|
||||||
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
|
profile := promptkit.ProfileSource("local profile", []promptkit.ProfileConfig{
|
||||||
"local-summary": {Endpoint: &endpoint, Model: &model},
|
{ID: "local-summary", Endpoint: &endpoint, Model: &model},
|
||||||
})
|
})
|
||||||
// Add configkit.Input{Source: profile} to a caller underlay.
|
// Add configkit.Input{Source: profile} to a caller underlay.
|
||||||
```
|
```
|
||||||
@@ -537,11 +820,13 @@ identity, endpoint, and concurrency limit:
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
backendID, model := promptkit.BackendLocal, "example-model"
|
backendID, model := promptkit.BackendLocal, "example-model"
|
||||||
backend := promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
|
localBackend := promptkit.LocalBackend("http://localhost:8000/v1", 2)
|
||||||
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
localBackend.ID = backendID
|
||||||
|
backend := promptkit.BackendSource("local backend", []promptkit.BackendConfig{
|
||||||
|
localBackend,
|
||||||
})
|
})
|
||||||
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
|
profile := promptkit.ProfileSource("local profile", []promptkit.ProfileConfig{
|
||||||
"local-summary": {Backend: &backendID, Model: &model},
|
{ID: "local-summary", Backend: &backendID, Model: &model},
|
||||||
})
|
})
|
||||||
// Add both sources as inputs in a caller underlay.
|
// Add both sources as inputs in a caller underlay.
|
||||||
```
|
```
|
||||||
@@ -552,13 +837,14 @@ capacity of 1024.
|
|||||||
|
|
||||||
#### Configure A Complete Backend
|
#### Configure A Complete Backend
|
||||||
|
|
||||||
Use a keyed `BackendConfig` value for authentication, extra request parameters, an
|
Use an identified `BackendConfig` value for authentication, extra request parameters, an
|
||||||
explicit queue capacity, a custom ID, or multiple local endpoints:
|
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
endpoint, keyName, limit, noWaiting := "http://gpu-host:8000/v1", "LOCAL_GPU_API_KEY", 2, 0
|
endpoint, keyName, limit, noWaiting := "http://gpu-host:8000/v1", "LOCAL_GPU_API_KEY", 2, 0
|
||||||
backend := promptkit.BackendSource("GPU backend", map[string]promptkit.BackendConfig{
|
backend := promptkit.BackendSource("GPU backend", []promptkit.BackendConfig{
|
||||||
"local-gpu": {
|
{
|
||||||
|
ID: "local-gpu",
|
||||||
Endpoint: &endpoint, APIKeyEnv: &keyName,
|
Endpoint: &endpoint, APIKeyEnv: &keyName,
|
||||||
ConcurrencyLimit: &limit, QueueCapacity: &noWaiting,
|
ConcurrencyLimit: &limit, QueueCapacity: &noWaiting,
|
||||||
ExtraParams: map[string]any{"provider_option": "enabled"},
|
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||||
@@ -567,7 +853,7 @@ backend := promptkit.BackendSource("GPU backend", map[string]promptkit.BackendCo
|
|||||||
```
|
```
|
||||||
|
|
||||||
Place `backend` in a caller underlay and select `"local-gpu"` from a
|
Place `backend` in a caller underlay and select `"local-gpu"` from a
|
||||||
`ProfileConfig.Backend` pointer or a keyed file profile. All backend
|
`ProfileConfig.Backend` pointer or an identified file profile. All backend
|
||||||
contributions use the same ConfigKit layering rules, including overlays of
|
contributions use the same ConfigKit layering rules, including overlays of
|
||||||
maintained definitions. The [`BackendConfig` and `LocalBackend` GoDoc](../../backends.go)
|
maintained definitions. The [`BackendConfig` and `LocalBackend` GoDoc](../../backends.go)
|
||||||
defines exact omission and construction behavior.
|
defines exact omission and construction behavior.
|
||||||
@@ -597,7 +883,7 @@ backend := promptkit.BackendConfig{
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
The pointer distinguishes an explicit zero from omission. Use keyed
|
The pointer distinguishes an explicit zero from omission. Use identified
|
||||||
`BackendConfig` literals inside `BackendSource` for programmatic definitions.
|
`BackendConfig` literals inside `BackendSource` for programmatic definitions.
|
||||||
Capacity belongs to one engine and the selected backend ID; endpoint-only
|
Capacity belongs to one engine and the selected backend ID; endpoint-only
|
||||||
profiles and custom backends without a configured limit remain unrestricted.
|
profiles and custom backends without a configured limit remain unrestricted.
|
||||||
@@ -629,6 +915,12 @@ secret-handling policies.
|
|||||||
|
|
||||||
## Extension Interfaces
|
## Extension Interfaces
|
||||||
|
|
||||||
|
JSON Schema validation and native structured output use a self-contained root
|
||||||
|
document. Internal references are supported; references to other files or
|
||||||
|
network resources fail during preparation. `WithSchemaFile` selects one root
|
||||||
|
file, while `WithSchemaFS` and `SchemaDir` select a root source. Prepared
|
||||||
|
executions retain their captured schema even if the source later changes.
|
||||||
|
|
||||||
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||||
behavior does not fit the application. Their GoDoc defines concurrent use,
|
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||||
context handling, ownership of copied values, nil responses, and preservation
|
context handling, ownership of copied values, nil responses, and preservation
|
||||||
@@ -636,6 +928,11 @@ of collaborator errors. Implementations must honor cancellation, safely manage
|
|||||||
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||||
the application policy that motivated the injection.
|
the application policy that motivated the injection.
|
||||||
|
|
||||||
|
A successful `ArtifactReader.Read` transfers the returned artifact and mutable
|
||||||
|
body storage to Promptkit. Readers that previously reused a returned buffer
|
||||||
|
must instead return independent storage and stop mutating it after return;
|
||||||
|
private scratch buffers may still be reused when the returned bytes are copied.
|
||||||
|
|
||||||
## Handle Errors
|
## Handle Errors
|
||||||
|
|
||||||
Use `errors.Is` with the
|
Use `errors.Is` with the
|
||||||
|
|||||||
@@ -43,13 +43,13 @@ Before starting work:
|
|||||||
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
| Root public API | The [architecture policy](policy/architecture.md), [consumer guide](consumers/pkg-promptkit.md), [testing policy](policy/testing.md), and existing GoDoc. |
|
||||||
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
| Prompt, profile, or schema formats | The [framework format reference](formats.md), owning parser or validator package, and [documentation policy](policy/documentation.md). |
|
||||||
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
| Source loading or validation | The [framework format reference](formats.md), [internal source document](internal/sources.md), and owning package tests. |
|
||||||
| Maintained external catalogs or built-ins | The [internal source document](internal/sources.md), [framework format reference](formats.md), [testing policy](policy/testing.md), both catalog module repositories, each catalog's release procedure, and the selected module versions in `go.mod`. |
|
| Caller-supplied configuration sources | The [internal source document](internal/sources.md), [framework format reference](formats.md), and [testing policy](policy/testing.md). |
|
||||||
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
| Model-client behavior | The [OpenAI-compatible integration contract](integrations/openai-compatible-chat.md), [internal model-client document](internal/llm.md), and owning package tests. |
|
||||||
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
| Internal package implementation | The [architecture policy](policy/architecture.md), [internal component overview](internal/overview.md), and focused internal document listed for that package. |
|
||||||
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
| Tests or test fixtures | The [testing policy](policy/testing.md), owning package, and focused internal document listed by the component overview. |
|
||||||
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
| Maintained example | The [preparation](../examples/go-library/prepare/main.go), [execution](../examples/go-library/run/main.go), and [observation](../examples/go-library/observe/main.go) examples, [consumer guide](consumers/pkg-promptkit.md), [framework format reference](formats.md), and [documentation policy](policy/documentation.md). |
|
||||||
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
| Documentation | The [documentation policy](policy/documentation.md) and canonical owner of every affected contract. |
|
||||||
| Licensing, dependencies/catalog updates, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
| Licensing, dependencies, copied material, or attribution | The [OSS policy](policy/oss.md), [project grant](../COPYRIGHT), [license texts](../LICENSE), and [attribution entries](oss/attribution.md); inspect the selected upstream source's notices. |
|
||||||
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
| Release preparation or publication, including release notes | The [release policy](policy/release.md), [release procedure](release.md), and [OSS discoverability rules](policy/oss.md#discoverability). |
|
||||||
|
|
||||||
For cross-cutting changes, follow every applicable row. Do not create
|
For cross-cutting changes, follow every applicable row. Do not create
|
||||||
@@ -58,38 +58,51 @@ exist.
|
|||||||
|
|
||||||
## Maintainer Validation
|
## Maintainer Validation
|
||||||
|
|
||||||
|
Use the newest Go installation already available locally for ordinary
|
||||||
|
development checks. Put its `bin` directory first on `PATH` and set
|
||||||
|
`GOTOOLCHAIN=local` to keep that toolchain selected. For repository-isolated
|
||||||
|
checks, set `GOWORK=off`; consumers may use their own workspaces. The selected
|
||||||
|
validation patch does not by itself change the module's consumer minimum. See
|
||||||
|
the [release toolchain policy](policy/release.md#consumer-minimum-and-release-toolchain)
|
||||||
|
and [release procedure](release.md#prepare-the-candidate) for release selection.
|
||||||
|
|
||||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||||
successful checks for unchanged content merely to create a commit. The full
|
successful checks for unchanged content merely to create a commit. During
|
||||||
release-candidate checks have one executable entry point from the repository root:
|
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||||
|
the maintained examples when they are affected:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./scripts/release-check.sh
|
go test ./...
|
||||||
|
go vet ./...
|
||||||
|
gofmt -w path/to/changed.go
|
||||||
|
git diff --check
|
||||||
```
|
```
|
||||||
|
|
||||||
The [script](../scripts/release-check.sh) owns the executable checks:
|
Release candidates use `./scripts/release.sh VERSION`, which runs the selected
|
||||||
race-enabled tests, vet, the three offline consumer examples with semantic output
|
profile checks and publishes the tag. The [release procedure](release.md)
|
||||||
assertions, tracked Go formatting, local Markdown links, module/workspace
|
describes its requirements and recovery. For an optional diagnostic of a
|
||||||
hygiene, whitespace, and the credential-pattern scan. It does not publish,
|
committed candidate without publication, run `./scripts/releasekit.sh check
|
||||||
change files, validate release-note structure, or retire roadmaps. It requires Go,
|
VERSION`; this is not a required pre-publication step.
|
||||||
Git, a POSIX shell, and Python 3. Tests and examples require no provider
|
|
||||||
|
The example workflows are covered by the Go suite; use
|
||||||
|
`go test ./examples/go-library/...` to run them separately during development.
|
||||||
|
The focused `python3 scripts/test-release.py` command exercises the release
|
||||||
|
tooling against disposable local repositories; use it when changing that
|
||||||
|
tooling, not as a recurring release gate. Tests and examples require no provider
|
||||||
credentials or live inference calls.
|
credentials or live inference calls.
|
||||||
|
|
||||||
The script can run on intended uncommitted changes during development.
|
Ordinary validation can run on intended uncommitted work. Release validation
|
||||||
Formatting and documentation checks use tracked files; stage intended new
|
uses committed candidate contents and the selected local Go installation.
|
||||||
files before final validation so those checks include them. The release
|
|
||||||
procedure separately enforces a clean, committed candidate.
|
|
||||||
|
|
||||||
### Change Review
|
### Change Review
|
||||||
|
|
||||||
Inspect `git status --short --untracked-files=all` and the complete diff before
|
Inspect `git status --short --untracked-files=all` and the complete diff before
|
||||||
accepting a change. Reject unintended files, credentials, private data,
|
accepting a change. Reject unintended files, credentials, private data,
|
||||||
generated binaries, test output, downloaded assets, and template residue.
|
generated binaries, test output, downloaded assets, and template residue.
|
||||||
The automated credential scan does not replace review of the actual change.
|
|
||||||
|
|
||||||
For releases, follow the [release procedure](release.md). Its publication
|
For releases, follow the [release procedure](release.md). Do not repeat its
|
||||||
command runs the full local checks once on the final committed candidate and
|
candidate checks as a separate sign-off. Roadmap retirement remains preparation
|
||||||
automates release-note structure and Git checks. Do not repeat ordinary change
|
housekeeping.
|
||||||
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
|
|
||||||
|
|
||||||
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
|
||||||
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
commits. Ordinary commits may leave unrelated work in the workspace; preserve
|
||||||
|
|||||||
323
docs/formats.md
323
docs/formats.md
@@ -6,29 +6,28 @@ This is the canonical reference for Promptkit's backend, profile, prompt, and
|
|||||||
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
||||||
how to select sources and use an engine; the
|
how to select sources and use an engine; the
|
||||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
||||||
owns outbound wire behavior. [ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit)
|
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md)
|
||||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||||
This reference describes what Promptkit does with the final effective values.
|
This reference describes what Promptkit does with the final effective values.
|
||||||
|
|
||||||
A configuration document has keyed top-level `backends`, `profiles`, and
|
A configuration document may contain top-level `backends`, `profiles`, and
|
||||||
`prompts` mappings. Their keys are literal IDs, not filename-derived identities.
|
`prompts` lists, plus a `collections` mapping of namespaced resource groups.
|
||||||
Prompt versions are a second keyed mapping. Keys and references must be
|
Each resource has an explicit literal `id`, not a filename-derived identity.
|
||||||
nonblank and remain exact: Promptkit does not trim or re-key them. Old embedded
|
IDs and references must be nonblank and remain exact: Promptkit does not trim or
|
||||||
`id`, `version`, and `base_profile` fields are invalid. Unknown fields are
|
re-key them. Unknown fields are errors. Promptkit installs a default-on bundled
|
||||||
errors.
|
OpenRouter underlay; callers still explicitly select a profile.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backends:
|
backends:
|
||||||
local:
|
- id: local
|
||||||
endpoint: http://localhost:8000/v1
|
endpoint: http://localhost:8000/v1
|
||||||
concurrency_limit: 2
|
concurrency_limit: 2
|
||||||
profiles:
|
profiles:
|
||||||
local-summary:
|
- id: local-summary
|
||||||
backend: local
|
backend: local
|
||||||
model: example-model
|
model: example-model
|
||||||
prompts:
|
prompts:
|
||||||
meeting.summary:
|
- id: meeting.summary
|
||||||
"1.0.0":
|
|
||||||
default_profile: local-summary
|
default_profile: local-summary
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -36,8 +35,69 @@ prompts:
|
|||||||
output:
|
output:
|
||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
|
collections:
|
||||||
|
support:
|
||||||
|
release_version: "1.4.0"
|
||||||
|
backends:
|
||||||
|
- id: offline
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
- id: base
|
||||||
|
backend: support/offline
|
||||||
|
model: offline-model
|
||||||
|
- id: fast
|
||||||
|
extends: {id: base}
|
||||||
|
timeout_seconds: 30
|
||||||
|
prompts:
|
||||||
|
- id: answer
|
||||||
|
default_profile: support/fast
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Answer the selected caller input.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Source Headers, Collections, And Resource IDs
|
||||||
|
|
||||||
|
An input may omit a source header. When present, the source-root `version` is
|
||||||
|
reserved for ConfigKit's specification header and accepts only the exact string
|
||||||
|
`"2"` or strict decimal integer `2`; null and other values are invalid. This
|
||||||
|
header is distinct from a collection's `release_version` and ConfigKit's own
|
||||||
|
software version. ConfigKit checks and consumes accepted headers on non-main
|
||||||
|
inputs; a declaration on the main input remains visible in explicit
|
||||||
|
configuration inspection. Headerless inputs still undergo ordinary final-value
|
||||||
|
validation. ConfigKit owns generic import and source-history behavior.
|
||||||
|
For a standalone prompt item, a root `version` is only a source header, not
|
||||||
|
prompt metadata.
|
||||||
|
|
||||||
|
The root resource lists contain unqualified IDs. Each `collections` entry
|
||||||
|
has a namespace key and may contain `release_version` plus the same three
|
||||||
|
resource lists. Collection resource IDs are local keys; their runtime IDs
|
||||||
|
are `<namespace>/<local-key>`. Root and collection keys may be the same. A
|
||||||
|
namespace and local key must be nonblank and slash-free, and references must
|
||||||
|
name their intended exact ID; mounting a source does not rewrite references.
|
||||||
|
For example, the `support/fast` profile above stores `support/offline` as its
|
||||||
|
backend ID. `extends: {id: base}` refers to a sibling in the same resource
|
||||||
|
list. It is not a qualified or cross-collection reference.
|
||||||
|
|
||||||
|
`release_version` is an optional opaque collection label. Absent or null means
|
||||||
|
no label; an explicitly empty string is retained. It does not select a resource
|
||||||
|
or change execution. Runtime lookups
|
||||||
|
use the captured effective resource indexes. `InspectConfiguration` is an
|
||||||
|
explicit operator view of resolved settings and provenance. Selected field
|
||||||
|
normalizations and the derived queue default are recorded in that tree, with
|
||||||
|
source history retained. Request-time completion remains separate. Source
|
||||||
|
history is not a second runtime resolver, and the
|
||||||
|
retained effective shape is not flattened or rewritten into those indexes.
|
||||||
|
|
||||||
|
For examples of explicitly selecting native collection inputs and inspecting
|
||||||
|
inventories, see the [consumer guide](consumers/pkg-promptkit.md) and its
|
||||||
|
[maintained preparation example](../examples/go-library/prepare/main.go). For
|
||||||
|
breaking identity and prompt-selection changes, see the
|
||||||
|
[configuration migration guide](consumers/configuration-migration.md).
|
||||||
|
|
||||||
Every effective backend, profile, and prompt definition is validated when
|
Every effective backend, profile, and prompt definition is validated when
|
||||||
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
`NewEngine` constructs the engine, even if it is never selected. A profile may
|
||||||
be partial: an endpoint and model need only be complete after request overrides
|
be partial: an endpoint and model need only be complete after request overrides
|
||||||
@@ -50,17 +110,17 @@ and static template syntax at construction. It does not reread configuration or
|
|||||||
message files on ordinary runs; construct a new engine to observe edits.
|
message files on ordinary runs; construct a new engine to observe edits.
|
||||||
Schema and request artifact reads still happen during preparation. Use
|
Schema and request artifact reads still happen during preparation. Use
|
||||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
report, resource inspection for resolved prompt, profile, and backend fields,
|
||||||
for request-specific rendered details.
|
and preparation for completed request-specific details. For identity discovery before inspection,
|
||||||
|
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
The `prompts` mapping selects a literal prompt ID, then a string version key:
|
The `prompts` list contains definitions selected by their literal IDs:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
prompts:
|
prompts:
|
||||||
meeting.summary:
|
- id: meeting.summary
|
||||||
"1.0.0":
|
|
||||||
default_profile: local-summary
|
default_profile: local-summary
|
||||||
description: Summarize a synthetic meeting note.
|
description: Summarize a synthetic meeting note.
|
||||||
session_id: '{{.session}}'
|
session_id: '{{.session}}'
|
||||||
@@ -88,9 +148,8 @@ prompts:
|
|||||||
fields may be absent or cleared with null where appropriate. A null whole
|
fields may be absent or cleared with null where appropriate. A null whole
|
||||||
definition, required role, output format, or validation mode is invalid.
|
definition, required role, output format, or validation mode is invalid.
|
||||||
`required` and `repair_attempts` default to false and zero, respectively, and
|
`required` and `repair_attempts` default to false and zero, respectively, and
|
||||||
are not nullable. A request that omits a prompt version succeeds only if that
|
are not nullable. `InspectPrompt` selects the same captured definition without
|
||||||
prompt ID has exactly one effective version. `InspectPrompt` selects from the
|
rendering or resolving a profile.
|
||||||
same captured definitions without rendering or resolving a profile.
|
|
||||||
|
|
||||||
### Inputs
|
### Inputs
|
||||||
|
|
||||||
@@ -124,6 +183,12 @@ Optional `session_id` uses the same template variables and input helper. Its
|
|||||||
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
|
||||||
points. A nonblank direct request session ID bypasses the template.
|
points. A nonblank direct request session ID bypasses the template.
|
||||||
|
|
||||||
|
`InspectRequirements` can enumerate recognizable dependencies from captured
|
||||||
|
templates before artifact content is available. Its conservative syntax
|
||||||
|
coverage does not change template validity or rendering; see the [consumer
|
||||||
|
guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
|
||||||
|
the method's [GoDoc](../requirements.go) for that inspection boundary.
|
||||||
|
|
||||||
### Cache Control
|
### Cache Control
|
||||||
|
|
||||||
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,
|
||||||
@@ -149,36 +214,74 @@ last candidate and its failed validation result. Operational errors remain
|
|||||||
errors. A request-level `OutputContract` replaces the whole configured one,
|
errors. A request-level `OutputContract` replaces the whole configured one,
|
||||||
not individual fields; an empty request format defaults to `text`.
|
not individual fields; an empty request format defaults to `text`.
|
||||||
|
|
||||||
## Built-In Backends
|
## Backend Definitions
|
||||||
|
|
||||||
The selected catalog modules contribute these backend definitions as the
|
Backend definitions are supplied through caller-selected configuration
|
||||||
earliest underlay. Their settings can be overlaid by caller sources; these IDs
|
sources. The conventional backend IDs are names only; PromptKit does not
|
||||||
are not reserved against configuration changes.
|
install definitions for them.
|
||||||
|
|
||||||
| ID | Catalog endpoint | API-key environment name | Active generation limit |
|
|
||||||
| --- | --- | --- | ---: |
|
|
||||||
| `openrouter` | `https://openrouter.ai/api/v1` | `OPENROUTER_API_KEY` | 16 |
|
|
||||||
| `rakestrawhome` | `https://inference.ai.rakestrawhome.com/v1` | `RAKESTRAWHOME_INFERENCE_API_KEY` | 4 |
|
|
||||||
|
|
||||||
A backend definition may have `endpoint`, `api_key_env`,
|
A backend definition may have `endpoint`, `api_key_env`,
|
||||||
`concurrency_limit`, `queue_capacity`, and `extra_params`. The final endpoint
|
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
|
||||||
is required and must be an absolute HTTP(S) URL with a host, no user
|
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
|
||||||
information, query, or fragment. An omitted or null API-key environment name
|
user information, query, or fragment. An omitted or null API-key environment
|
||||||
means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
name means no lookup. Omitted/null concurrency is zero (unlimited); a supplied
|
||||||
negative value is invalid. For positive concurrency, omitted/null queue
|
negative value is invalid. For positive concurrency, omitted/null queue
|
||||||
capacity defaults to 1024. An explicit queue capacity, including zero,
|
capacity defaults to 1024. An explicit queue capacity, including zero,
|
||||||
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
requires positive concurrency and must be nonnegative. Unlimited concurrency
|
||||||
has no queue. Provider maps contain JSON-compatible values; null entries are
|
has no queue. Provider maps contain JSON-compatible values; null entries are
|
||||||
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
ordinary JSON null. Unknown fields, including a raw `api_key`, fail.
|
||||||
|
|
||||||
|
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
||||||
|
It may contain `default_profile`, `reachability`, and `authentication`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
backends:
|
||||||
|
- id: openrouter
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
smoke_tests:
|
||||||
|
default_profile: application-profile
|
||||||
|
reachability:
|
||||||
|
endpoint: https://status.example/ready?check=1
|
||||||
|
authentication:
|
||||||
|
kind: openrouter_key
|
||||||
|
endpoint: https://openrouter.ai/api/v1/key
|
||||||
|
```
|
||||||
|
|
||||||
|
An omitted or null `smoke_tests` block disables these metadata defaults.
|
||||||
|
`default_profile` is preserved as an exact, nonblank profile ID; its existence
|
||||||
|
is checked only if an inference diagnostic selects that default. Omitted/null
|
||||||
|
`default_profile` means there is no inference default. Omitted/null
|
||||||
|
`reachability` or its endpoint uses the effective backend endpoint. An explicit
|
||||||
|
reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
|
||||||
|
host; unlike the backend base endpoint, its path and query are allowed. Null
|
||||||
|
means fallback, while an explicit blank value is invalid.
|
||||||
|
|
||||||
|
Omitted/null `authentication` means no authentication adapter is declared. A
|
||||||
|
non-null object requires `kind`, which is trimmed and case-normalized; the
|
||||||
|
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
|
||||||
|
uses the adapter's backend-relative destination (`openrouter_key` appends
|
||||||
|
`/key` to the effective backend base); an explicit endpoint follows the same
|
||||||
|
full-URL rules as reachability. `none` declares authentication inapplicable.
|
||||||
|
These values are configuration only: engine construction does not contact a
|
||||||
|
backend.
|
||||||
|
|
||||||
|
These fields are resolved through the ordinary ConfigKit source layers before
|
||||||
|
Promptkit validates the final effective values. Sibling inheritance can overlay
|
||||||
|
individual nested fields; a later same-ID resource replaces the whole record.
|
||||||
|
An explicit null can clear the whole `smoke_tests` block or one of its nested
|
||||||
|
objects. An endpoint set to null uses
|
||||||
|
its documented fallback, while an explicit blank endpoint is invalid. The
|
||||||
|
metadata does not change ordinary generation credential requirements or
|
||||||
|
provider settings.
|
||||||
|
|
||||||
## Profile Definitions
|
## Profile Definitions
|
||||||
|
|
||||||
Profiles are keyed by literal ID and may inherit a named parent with
|
Profiles have literal IDs and may inherit a sibling parent with
|
||||||
`extends`:
|
`extends`:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
profiles:
|
profiles:
|
||||||
local-summary:
|
- id: local-summary
|
||||||
backend: local
|
backend: local
|
||||||
model: example-model
|
model: example-model
|
||||||
temperature: 0.2
|
temperature: 0.2
|
||||||
@@ -186,8 +289,8 @@ profiles:
|
|||||||
timeout_seconds: 90
|
timeout_seconds: 90
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
local-summary-fast:
|
- id: local-summary-fast
|
||||||
extends: local-summary
|
extends: {id: local-summary}
|
||||||
timeout_seconds: 30
|
timeout_seconds: 30
|
||||||
reasoning_effort: low
|
reasoning_effort: low
|
||||||
```
|
```
|
||||||
@@ -212,84 +315,136 @@ only a model or backend does not clear unrelated explicitly inherited fields.
|
|||||||
| Profile field | Absent | Explicit value | Explicit null |
|
| Profile field | Absent | Explicit value | Explicit null |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
||||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
|
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Selected backend endpoint |
|
||||||
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
||||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
||||||
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
||||||
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
||||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
|
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Backend lookup name, if any |
|
||||||
| `api_key_required` | False | Boolean, including false | False |
|
| `api_key_required` | False | Boolean, including false | False |
|
||||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
|
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Backend defaults |
|
||||||
|
|
||||||
|
An empty profile mapping does not clear backend map entries. After ConfigKit
|
||||||
|
composition and inheritance, a whole-map null and omission both permit backend
|
||||||
|
defaults. Nested maps merge recursively, while scalar and array values replace;
|
||||||
|
nulls within provider maps or arrays remain literal data.
|
||||||
|
When `api_key_required` is true and the profile source is absent or null,
|
||||||
|
execution suppresses the backend credential source; an explicitly configured
|
||||||
|
profile source remains in use.
|
||||||
|
|
||||||
|
For example, if the backend provides an endpoint, API-key environment name,
|
||||||
|
and provider map, a profile containing `endpoint: null`, `api_key_env: null`,
|
||||||
|
and `extra_params: null` now inherits those backend values after handoff (unless
|
||||||
|
the required-key rule suppresses the credential source). Earlier versions used
|
||||||
|
those nulls to suppress the corresponding fallback. Use `endpoint: ""` or
|
||||||
|
`api_key_env: ""` to clear those individual strings. There is no profile-level
|
||||||
|
whole-map suppression replacement; change the backend map through ordinary
|
||||||
|
ConfigKit composition when different provider defaults are needed. ConfigKit's
|
||||||
|
retained configuration and provenance continue to show the nulls.
|
||||||
|
|
||||||
An empty mapping does not clear inherited map entries; null clears the whole
|
|
||||||
map. Nested maps merge recursively, while scalar and array values replace.
|
|
||||||
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
||||||
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
||||||
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
||||||
data, not a configuration inheritance instruction. An explicit zero timeout
|
data, not a configuration inheritance instruction. An explicit zero timeout
|
||||||
does not disable the caller context or HTTP transport timeout.
|
does not disable the caller context or HTTP transport timeout.
|
||||||
|
|
||||||
|
### Bundled OpenRouter Configuration
|
||||||
|
|
||||||
|
The default bundle installs root backend `openrouter`, endpoint
|
||||||
|
`https://openrouter.ai/api/v1`, optional credential environment name
|
||||||
|
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
|
||||||
|
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
|
||||||
|
applies. No prompt, implicit profile selection, or tuning defaults are installed.
|
||||||
|
Disable the entire bundle with `WithBundledOpenRouter(false)`.
|
||||||
|
|
||||||
|
The root profile IDs and their models are:
|
||||||
|
|
||||||
|
| Profile | Model |
|
||||||
|
| --- | --- |
|
||||||
|
| `gpt-astra-latest` | `~openai/gpt-astra-latest` |
|
||||||
|
| `gpt-sol-latest` | `~openai/gpt-sol-latest` |
|
||||||
|
| `gpt-terra-latest` | `~openai/gpt-terra-latest` |
|
||||||
|
| `gpt-luna-latest` | `~openai/gpt-luna-latest` |
|
||||||
|
| `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
||||||
|
| `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
||||||
|
| `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
||||||
|
| `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
||||||
|
| `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
||||||
|
| `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
||||||
|
| `auto` | `openrouter/auto` |
|
||||||
|
| `free` | `openrouter/free` |
|
||||||
|
|
||||||
|
The profiles `auto-low`, `auto-medium`, `auto-high`, `auto-xhigh`, and
|
||||||
|
`auto-max` extend `auto` and add an `auto-router` plugin with the corresponding
|
||||||
|
`cost_tier`. Replacing `auto` changes their inherited backend and model while
|
||||||
|
retaining their plugins. These are root IDs even when a model contains `/`.
|
||||||
|
For example, a caller can replace `auto` as a complete record and extend another
|
||||||
|
bundled profile under a new ID:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
profiles:
|
||||||
|
- id: auto
|
||||||
|
backend: openrouter
|
||||||
|
model: openrouter/free
|
||||||
|
- id: my-luna
|
||||||
|
extends: {id: gpt-luna-latest}
|
||||||
|
timeout_seconds: 60
|
||||||
|
```
|
||||||
|
|
||||||
|
See OpenRouter's [alias examples](https://openrouter.ai/~openai/gpt-luna-latest),
|
||||||
|
[auto router and cost tiers](https://openrouter.ai/docs/guides/routing/routers/auto-router),
|
||||||
|
and [free router](https://openrouter.ai/docs/guides/routing/routers/free-router)
|
||||||
|
for provider behavior and availability.
|
||||||
|
|
||||||
### Source And Profile Precedence
|
### Source And Profile Precedence
|
||||||
|
|
||||||
`NewEngine` composes catalog definitions first, Promptkit-owned definitions
|
`NewEngine` composes the bundled OpenRouter underlay first by default, then
|
||||||
next, then caller underlays in their given order, with an optional main input
|
caller underlays in their given order, followed by an
|
||||||
last. Later tiers overlay earlier ones. Inputs within one underlay group are
|
optional main input. Later layers overlay earlier ones. Inputs within one
|
||||||
strictly composed: conflicting contributions are errors, not priority
|
underlay group are strictly composed: conflicting contributions are errors,
|
||||||
signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
||||||
in sorted order, but that order does not give one file higher precedence.
|
in sorted order, but that order does not give one file higher precedence.
|
||||||
An explicitly selected missing/unreadable root fails; an empty root contributes
|
An explicitly selected missing/unreadable root fails; an empty root contributes
|
||||||
nothing. Imports are disabled by these helpers unless the caller enables them
|
nothing. Imports are disabled by these helpers unless the caller enables them
|
||||||
on an input. Single files can use ConfigKit inputs directly.
|
on an input. Single files can use ConfigKit inputs directly.
|
||||||
|
|
||||||
Backends, profiles, and prompts all follow these same source rules. A later
|
Root backends, profiles, and prompts, as well as collection declarations and
|
||||||
profile or backend can change individual fields of a lower definition. A
|
their resource lists, follow these same source rules. `CollectionInput`
|
||||||
request `ProfileID` takes precedence over a prompt `default_profile`; if
|
mounts one explicitly selected source at a collection destination and enables
|
||||||
|
ConfigKit imports for that input; it does not discover or select sources.
|
||||||
|
A later same-ID record replaces the lower definition in full. Use a distinct
|
||||||
|
inherited sibling when a variant should retain its parent's fields.
|
||||||
|
A request `ProfileID` takes precedence over a prompt `default_profile`; if
|
||||||
neither is selected, preparation fails. Exact profile inspection selects one
|
neither is selected, preparation fails. Exact profile inspection selects one
|
||||||
explicit ID and does not consult a prompt default.
|
explicit ID and does not consult a prompt default.
|
||||||
|
|
||||||
### Profile Inheritance
|
### Resource Inheritance
|
||||||
|
|
||||||
ConfigKit resolves `extends` on effective profile definitions. One profile
|
ConfigKit resolves `extends: {id: parent}` for backends, profiles, and prompts.
|
||||||
may inherit another keyed profile; presence, explicit null, and recursively
|
Each child inherits a surviving sibling in the same root or collection list;
|
||||||
merged maps determine the final fields. References are literal IDs, including
|
collection inheritance cannot cross namespaces. Parents are ordinary validated
|
||||||
IDs containing punctuation. There is no Promptkit-specific depth limit or
|
resources, and profiles may remain partial until request completion. Presence, explicit
|
||||||
second base-chain resolver. Missing references and cycles fail at construction.
|
null, and recursively merged maps determine the final fields. Other resource
|
||||||
For generic inheritance and layer mechanics, consult
|
references are exact IDs and use their qualified form when they select a
|
||||||
[ConfigKit](https://gitea.maximumdirect.net/eric/ConfigKit).
|
collection resource. There is no Promptkit-specific depth limit or second
|
||||||
|
base-chain resolver. Missing references and cycles fail at construction. For
|
||||||
## Built-In Profile Catalog
|
generic inheritance and layer mechanics, consult
|
||||||
|
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md).
|
||||||
Promptkit contributes two opt-in definitions; neither is selected by default:
|
|
||||||
|
|
||||||
| ID | Current definition |
|
|
||||||
| --- | --- |
|
|
||||||
| `promptkit-openrouter-standard` | Extends the selected OpenRouter catalog's `gemini-flash-latest` profile. |
|
|
||||||
| `promptkit-openrouter-auto` | Selects backend `openrouter` and model `openrouter/auto`. |
|
|
||||||
|
|
||||||
Select one through request `ProfileID` or prompt `default_profile`.
|
|
||||||
Caller layers may overlay these definitions or their catalog bases. The
|
|
||||||
[release policy](policy/release.md#catalog-profile-compatibility) owns their
|
|
||||||
cross-version compatibility boundary.
|
|
||||||
|
|
||||||
## External Profile Inventories
|
|
||||||
|
|
||||||
The selected catalog modules, not this document, own their profile inventories:
|
|
||||||
|
|
||||||
- [OpenRouter catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/tag/v3.20260930.0/catalog/profiles)
|
|
||||||
- [Rakestrawhome catalog v3.20260930.0 profiles](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/tag/v3.20260930.0/catalog/profiles)
|
|
||||||
|
|
||||||
These are tested local data, not a promise of live provider availability.
|
|
||||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns
|
|
||||||
the compatibility boundary.
|
|
||||||
|
|
||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
Directory and `fs.FS` schema paths resolve within the configured root. Each
|
||||||
referenced nested schemas resolve relative to the owning schema document.
|
schema must be self-contained: in-document references, IDs, and anchors are
|
||||||
|
supported, but references requiring another file or network resource fail.
|
||||||
|
References must also work without the source filename as a base; use fragment
|
||||||
|
references or explicit schema IDs rather than relying on that filename.
|
||||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
||||||
invalid, or unresolvable schema causes an operational validation error.
|
invalid, or unresolvable schema causes an operational validation error.
|
||||||
Valid JSON that fails the schema instead yields a failed validation result.
|
Valid JSON that fails the schema instead yields a failed validation result.
|
||||||
|
Schema documents are detached without applying the provider extra-parameter
|
||||||
|
copy limits; this does not change the schema decoder, compiler, or
|
||||||
|
self-contained-resource rules.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
|
|||||||
@@ -16,14 +16,13 @@ Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
|||||||
Before the client is called, the engine resolves framework, backend, profile,
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
and request values into one execution target. Endpoint configuration is trimmed
|
and request values into one execution target. Endpoint configuration is trimmed
|
||||||
and must be an absolute HTTP or HTTPS URL with a host and without user
|
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||||
information, a query, or a fragment. A non-empty endpoint from the target
|
information, a query, or a fragment. The target's endpoint is validated again
|
||||||
overrides the client's configured base URL. The final selected endpoint is
|
before transport.
|
||||||
validated again before transport.
|
|
||||||
|
|
||||||
The completion URL is composed through parsed URL path operations. Nested base
|
The completion URL is composed through parsed URL path operations. Nested base
|
||||||
paths are retained, repeated trailing slashes are normalized, and the result
|
paths are retained, repeated trailing slashes are normalized, and the result
|
||||||
has exactly one appended `/chat/completions` suffix. Generation fails before
|
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||||
sending when neither source supplies a valid endpoint.
|
sending when the target does not supply a valid endpoint.
|
||||||
|
|
||||||
The target's backend ID is routing metadata for prepared values, results, and
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
injected clients. The built-in client does not derive the URL from that ID and
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
@@ -45,11 +44,51 @@ precedence over the backend default. Only the name reaches prepared metadata;
|
|||||||
the environment value is read just before the provider call and is never added
|
the environment value is read just before the provider call and is never added
|
||||||
to the JSON body.
|
to the JSON body.
|
||||||
|
|
||||||
|
## Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
Promptkit's explicit smoke diagnostics are separate consumer operations; they
|
||||||
|
are not part of ordinary `Run`, the offline examples, or release validation.
|
||||||
|
With the built-in client, inference diagnostics use the same configured client
|
||||||
|
and `/chat/completions` request as generation. Their content cases are private
|
||||||
|
to Promptkit and are not application prompt, schema, or provider protocol
|
||||||
|
extensions. The public
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns request selection,
|
||||||
|
report facts, and injected-client behavior.
|
||||||
|
|
||||||
|
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
|
||||||
|
one unauthenticated `GET` to the selected diagnostic URL; an observed response
|
||||||
|
counts as reachable even when its status is non-2xx. Authentication behavior
|
||||||
|
is selected from backend metadata, not inferred from the host. The
|
||||||
|
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
|
||||||
|
backend base, unless configuration supplies a full diagnostic endpoint (see
|
||||||
|
the [backend format reference](../formats.md#backend-definitions)). It may send
|
||||||
|
the available bearer credential, but can dispatch without one. A 401 is
|
||||||
|
recognized as rejection. A 200 is recognized as acceptance only when its JSON
|
||||||
|
object has a `data` object containing string `label`, Boolean `is_free_tier`,
|
||||||
|
and numeric `usage`; unrelated fields and account limits are ignored. This is
|
||||||
|
only a minimal identifying subset of OpenRouter's documented
|
||||||
|
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||||
|
not validation of the full account document. Other statuses and unrecognized
|
||||||
|
200 responses do not establish acceptance. A 200 response exceeding the bounded
|
||||||
|
recognition allowance reports `authentication_response_limit_exceeded`, leaving
|
||||||
|
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||||
|
RoundTripper boundary and any response associated with it. It includes the
|
||||||
|
selected destination without URL user information, response status, and
|
||||||
|
credential presence when observed. Redirects can change the last observed
|
||||||
|
destination or strip credentials. This evidence does not prove that bytes
|
||||||
|
reached a peer or reveal dispatch hidden inside a custom transport. Diagnostic
|
||||||
|
reports do not retain headers, credentials, provider
|
||||||
|
bodies, or generated text. A consumer-injected `LLMClient` can support inference
|
||||||
|
diagnostics, but does not establish HTTP status, destination, reachability, or
|
||||||
|
upstream credential-supply facts; Promptkit does not fall back to a built-in
|
||||||
|
transport for HTTP-only diagnostics.
|
||||||
|
|
||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
target's model takes precedence over the client's configured model, and one
|
target's model must be available.
|
||||||
must be available.
|
|
||||||
|
|
||||||
Each ordinary message contains its `role` and string `content`. A
|
Each ordinary message contains its `role` and string `content`. A
|
||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
@@ -76,11 +115,11 @@ The client conditionally includes:
|
|||||||
strict flag, and schema document. Plain JSON validation does not add an
|
strict flag, and schema document. Plain JSON validation does not add an
|
||||||
object-only response constraint.
|
object-only response constraint.
|
||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
Configured profile maps recursively overlay backend defaults, while a nonempty
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
runtime override replaces the complete configured map. The [format reference](../formats.md#defaults-and-overrides)
|
||||||
then merged directly into the top-level body after JSON serialization is
|
owns these resolution rules. The resulting effective map is merged into the
|
||||||
verified. Empty keys and collisions with these reserved fields are rejected
|
top-level body. Empty keys, values that cannot be encoded, and
|
||||||
before any provider call:
|
collisions with these reserved fields are rejected before any provider call:
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
- `session_id`
|
- `session_id`
|
||||||
|
|||||||
@@ -68,9 +68,8 @@ Preparation and validation never hold an active permit.
|
|||||||
|
|
||||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||||
returns an idempotent release function. The coordinator uses it around
|
returns an idempotent release function. The coordinator uses it around
|
||||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
individual attempts. There is one queue and one active limit per limited backend
|
||||||
operation for focused internal callers and tests, but engine assembly no
|
pool.
|
||||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
|
||||||
|
|
||||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||||
@@ -88,6 +87,17 @@ release, including canceled acquisition without dispatch. A retry wait holds
|
|||||||
the logical admission lease but no active permit. Observer callbacks must not
|
the logical admission lease but no active permit. Observer callbacks must not
|
||||||
block on work that requires the same run's admission slot.
|
block on work that requires the same run's admission slot.
|
||||||
|
|
||||||
|
## Smoke-Test Capacity
|
||||||
|
|
||||||
|
The engine's smoke tester shares this manager and generation coordinator with
|
||||||
|
the ordinary runner. Inference diagnostics reserve the same logical admission
|
||||||
|
slot and acquire the same backend-scoped active permits for actual model
|
||||||
|
attempts, including retries. They therefore participate in the same configured
|
||||||
|
backend limits as `Run`. Reachability and authentication HTTP probes do not
|
||||||
|
enter generation admission or consume active permits. The public
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go) owns the caller-visible
|
||||||
|
operation and injected-client limits.
|
||||||
|
|
||||||
## Cancellation And Release
|
## Cancellation And Release
|
||||||
|
|
||||||
Admission checks the caller context before reserving a slot. A call canceled
|
Admission checks the caller context before reserving a slot. A call canceled
|
||||||
@@ -110,7 +120,7 @@ validation, bounded admission, idempotent release, context handling, and
|
|||||||
unlimited admission. The
|
unlimited admission. The
|
||||||
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
unlimited calls, passthrough behavior, and panic release.
|
unlimited calls, and panic release using test-only one-shot calls.
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
admission, lease lifetime, failure release, and shared initial/repair
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
@@ -127,4 +137,7 @@ cover shared and independent backend pools during retry and repair. The
|
|||||||
own the public prepared-capacity boundary. The
|
own the public prepared-capacity boundary. The
|
||||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||||
the public generation category and context identity when generation is
|
the public generation category and context identity when generation is
|
||||||
canceled.
|
canceled. The assembled
|
||||||
|
[smoke diagnostic contract tests](../../smoke_integration_contract_test.go)
|
||||||
|
verify that inference diagnostics share the engine's Run admission and recover
|
||||||
|
capacity after cancellation or failure.
|
||||||
|
|||||||
@@ -41,9 +41,22 @@ original error chain. This keeps injected attempt classifications consistent
|
|||||||
with terminal observations without parsing diagnostic text or retrying injected
|
with terminal observations without parsing diagnostic text or retrying injected
|
||||||
clients.
|
clients.
|
||||||
|
|
||||||
|
The diagnostic path uses the same private attempt and retry implementation as
|
||||||
|
ordinary generation. It retains invocation-local transport, response,
|
||||||
|
generation, usage, capacity-wait, and retry-delay evidence before normal
|
||||||
|
result-discard and cancellation checks. Built-in preparation also supplies its
|
||||||
|
frozen request URL to the evidence path before attempts begin; injected
|
||||||
|
generation has no equivalent HTTP destination. This evidence is returned to
|
||||||
|
diagnostic orchestration; it is not added to public `ExecutionEvent` values or ordinary
|
||||||
|
observer accounting. The [model-client document](llm.md) describes the
|
||||||
|
transport-observation boundary, and the root
|
||||||
|
[`SmokeTestReport` GoDoc](../../smoke_diagnostics.go) owns report semantics.
|
||||||
|
|
||||||
Focused coordinator tests use controlled transport and wait functions. They
|
Focused coordinator tests use controlled transport and wait functions. They
|
||||||
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
cover eligibility, exhaustion, cancellation, backoff, request freezing, permit
|
||||||
release, and the injected-client path. The capacity manager tests own permit
|
release, and the injected-client path. The
|
||||||
|
[evidence tests](../../internal/generation/evidence_test.go) own diagnostic
|
||||||
|
attempt retention and retry evidence. The capacity manager tests own permit
|
||||||
acquisition and idempotent release. Public
|
acquisition and idempotent release. Public
|
||||||
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
[integration tests](../../reliable_execution_contract_test.go) cover retry,
|
||||||
repair, observation, frozen prepared execution, and shared-backend behavior
|
repair, observation, frozen prepared execution, and shared-backend behavior
|
||||||
|
|||||||
@@ -26,17 +26,24 @@ client uses its endpoint, credential metadata, generation fields, and extra
|
|||||||
parameters. `BackendID` remains routing metadata
|
parameters. `BackendID` remains routing metadata
|
||||||
for the generation boundary and is not mapped into the provider payload.
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
Construction trims and validates a nonempty configured base URL and clones any
|
Construction clones any supplied `http.Client` so Promptkit can apply its
|
||||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
timeout default without mutating the caller's client. Generation preparation
|
||||||
mutating the caller's client. An empty configured base remains valid because a
|
then:
|
||||||
resolved request target may supply the endpoint. Generation preparation then:
|
|
||||||
|
|
||||||
1. validates shared execution-setting invariants and the final selected base
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
endpoint;
|
endpoint;
|
||||||
2. maps the internal request into the OpenAI-compatible chat payload;
|
2. maps the internal request into the OpenAI-compatible chat payload;
|
||||||
3. validates and merges extra parameters;
|
3. checks extra-parameter keys against empty and reserved names, then merges
|
||||||
|
their values into the payload;
|
||||||
4. composes `/chat/completions` through parsed URL path operations;
|
4. composes `/chat/completions` through parsed URL path operations;
|
||||||
5. resolves authentication and freezes the URL, body, and optional credential.
|
5. encodes the complete body once on success. If encoding fails, extra values
|
||||||
|
are checked individually to retain invalid-request classification; otherwise
|
||||||
|
the complete-request failure remains a request failure;
|
||||||
|
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||||
|
|
||||||
|
Provider values are detached with trusted copying before encoding. Custom JSON
|
||||||
|
encoding methods and per-value encoding-failure diagnostics operate on those
|
||||||
|
private copies rather than retained prepared or diagnostic metadata.
|
||||||
|
|
||||||
Each execution of that prepared request creates a fresh HTTP request, body
|
Each execution of that prepared request creates a fresh HTTP request, body
|
||||||
reader, and attempt timeout context. It closes the response body and returns
|
reader, and attempt timeout context. It closes the response body and returns
|
||||||
@@ -55,6 +62,34 @@ This client does not schedule retries; the [generation coordinator](generation.m
|
|||||||
does so around its prepared attempts. The client has no tool-call support,
|
does so around its prepared attempts. The client has no tool-call support,
|
||||||
provider catalog, inbound HTTP behavior, or durable session store.
|
provider catalog, inbound HTTP behavior, or durable session store.
|
||||||
|
|
||||||
|
## Diagnostic HTTP Evidence
|
||||||
|
|
||||||
|
Built-in diagnostics wrap the selected HTTP client's `RoundTripper` for one
|
||||||
|
invocation using a shallow client copy, preserving its timeout, redirect policy,
|
||||||
|
cookie jar, and underlying transport without mutating the configured client.
|
||||||
|
Evidence distinguishes a diagnostic dispatch invocation from an observed
|
||||||
|
request and response. A newly observed request replaces the previous
|
||||||
|
destination-specific response evidence, so redirects retain only the last
|
||||||
|
observed hop. A response is recorded before envelope decoding or later context
|
||||||
|
checks can discard a generation result.
|
||||||
|
|
||||||
|
The collector keeps only the last observed destination (without URL user
|
||||||
|
information), status, credential-presence, response-availability, decoded
|
||||||
|
generation, and presence-aware usage facts needed by the diagnostic. It does
|
||||||
|
not retain headers, credentials, raw request or response bodies, or a redirect
|
||||||
|
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||||
|
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||||
|
Reachability and named authentication probes use the same configured built-in
|
||||||
|
client. The OpenRouter adapter owns endpoint derivation, bounded account-response
|
||||||
|
recognition, and body closure; the smoke coordinator interprets its evidence. The
|
||||||
|
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||||
|
owns their observable HTTP semantics.
|
||||||
|
|
||||||
|
An injected `LLMClient` does not pass through this HTTP boundary. Its returned
|
||||||
|
response can establish response and inference facts, but not an HTTP
|
||||||
|
destination, status, reachability, or upstream credential-supply fact. The
|
||||||
|
facade does not silently install a built-in transport for those diagnostics.
|
||||||
|
|
||||||
## Prepared Generation
|
## Prepared Generation
|
||||||
|
|
||||||
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||||
@@ -74,14 +109,13 @@ public ownership and redaction semantics belong to the
|
|||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid generation
|
||||||
configuration, invalid generation requests, request execution failures,
|
requests, request execution failures, non-success provider statuses, and
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
malformed successful responses. Provider response bodies are never exposed in
|
||||||
response bodies are never exposed in raw form through non-success errors.
|
raw form through non-success errors.
|
||||||
|
|
||||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
A missing or invalid final selected endpoint is an invalid generation request
|
||||||
invalid final selected endpoint is an invalid generation request and is
|
and is rejected before transport.
|
||||||
rejected before transport.
|
|
||||||
|
|
||||||
Authentication resolves a trimmed direct key before a trimmed configured
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
@@ -133,7 +167,10 @@ own configuration, client cloning, deterministic deadline precedence,
|
|||||||
authentication, request and response mapping, malformed data, error identity,
|
authentication, request and response mapping, malformed data, error identity,
|
||||||
cancellation, endpoint selection and composition, pre-transport rejection, and
|
cancellation, endpoint selection and composition, pre-transport rejection, and
|
||||||
bounded single-document successful-response framing, closure, and
|
bounded single-document successful-response framing, closure, and
|
||||||
response-body suppression. The focused
|
response-body suppression. The [HTTP evidence tests](../../internal/llm/http_evidence_test.go)
|
||||||
|
own invocation-local last-hop request/response facts, while the
|
||||||
|
[prepared-generation evidence tests](../../internal/llm/prepared_generation_evidence_test.go)
|
||||||
|
own retention across decoding and cancellation. The focused
|
||||||
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
[provider HTTP error tests](../../internal/llm/provider_http_error_test.go)
|
||||||
own envelope parsing, normalization, and bounded-reader cases; their
|
own envelope parsing, normalization, and bounded-reader cases; their
|
||||||
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
[transport tests](../../internal/llm/provider_http_error_transport_test.go)
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ lifecycle instrumentation spans the facade and runner boundaries.
|
|||||||
Prepared execution starts only after its private payload is atomically claimed;
|
Prepared execution starts only after its private payload is atomically claimed;
|
||||||
the facade receives that invocation's recorder privately and emits the terminal
|
the facade receives that invocation's recorder privately and emits the terminal
|
||||||
event after public error mapping. Failed claims and preparation emit nothing.
|
event after public error mapping. Failed claims and preparation emit nothing.
|
||||||
|
The public projection retains private operation-owner metadata separately from
|
||||||
|
inspectable child causes. Terminal classification uses that owner, preserving
|
||||||
|
context precedence and generation-specific transport/provider details rather
|
||||||
|
than mistaking a collaborator's nested lookup error for the parent's operation.
|
||||||
|
|
||||||
The facade masks inherited recorder and prepared-claim state when entering
|
The facade masks inherited recorder and prepared-claim state when entering
|
||||||
preparation or execution. Delegated calls from an injected collaborator retain
|
preparation or execution. Delegated calls from an injected collaborator retain
|
||||||
@@ -23,6 +27,9 @@ callback. The caller supplies normalized failure metadata; the recorder never
|
|||||||
parses an error string or receives provider bodies, credentials, headers, or
|
parses an error string or receives provider bodies, credentials, headers, or
|
||||||
other content. Events are values with no mutable containers. Callback panics
|
other content. Events are values with no mutable containers. Callback panics
|
||||||
propagate to the caller.
|
propagate to the caller.
|
||||||
|
Resolved prompt and profile IDs are the exact runtime identities selected for
|
||||||
|
the invocation, including namespace qualification when supplied by an engine
|
||||||
|
view.
|
||||||
Per-operation callbacks are ordered and synchronous; different operations may
|
Per-operation callbacks are ordered and synchronous; different operations may
|
||||||
invoke the same callback concurrently. The recorder marks each measured phase's
|
invoke the same callback concurrently. The recorder marks each measured phase's
|
||||||
end before delivery, so callback work does not inflate provider duration.
|
end before delivery, so callback work does not inflate provider duration.
|
||||||
@@ -41,6 +48,11 @@ attempts above the first in a generation, and repair generations with at least
|
|||||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||||
attempt events.
|
attempt events.
|
||||||
|
|
||||||
|
Smoke diagnostics mask inherited execution-recorder state while invoking
|
||||||
|
generation or HTTP collaborators. Their private attempt evidence is assembled
|
||||||
|
into a `SmokeTestReport`; it does not create ordinary run events or enter an
|
||||||
|
enclosing run's observation totals.
|
||||||
|
|
||||||
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
The [recorder tests](../../internal/observation/recorder_test.go) own usage
|
||||||
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
presence, phase sums, event ordering, empty-run accounting, identity snapshots,
|
||||||
and callback panic behavior. The [generation coordinator](generation.md) owns
|
and callback panic behavior. The [generation coordinator](generation.md) owns
|
||||||
|
|||||||
@@ -11,28 +11,27 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, resource and requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||||
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
|
| `internal/configuration` | Loads root and namespaced resources through ConfigKit; validates final values, captures indexed resource views and source-relative prompt content, compiled templates, requirement metadata, identity inventories, backend and presence-aware execution profiles, and the read-only provenance inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||||
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
|
|
||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
|
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and identity listing directly to the captured configuration rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
|
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||||
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The internal engine assembles these internal components without exposing their
|
The internal engine assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -32,30 +32,73 @@ The runner has no durable run or session store.
|
|||||||
|
|
||||||
## Shared Prompt Selection
|
## Shared Prompt Selection
|
||||||
|
|
||||||
The runner uses one prompt-selection and hashing boundary for ordinary
|
The internal engine applies each namespace view to slashless caller selections
|
||||||
preparation and exact prompt inspection. Preparation retains its early
|
before they reach the runner. The runner therefore selects captured prompts and
|
||||||
request-ID check before direct-session normalization; both operations select
|
profiles by the resulting exact runtime IDs. Already-qualified selections pass
|
||||||
one captured definition and its construction-time hash. Referenced message
|
through, while configured defaults and stored references are used as authored
|
||||||
content and static template syntax were captured when the engine was built.
|
and are not requalified.
|
||||||
|
|
||||||
Inspection stops after that structural lookup. It does not render templates or
|
Ordinary preparation selects the full prompt definition and construction-time
|
||||||
touch profile, artifact, schema, renderer, validator, admission, or model
|
hash. It retains its early request-ID check before direct-session
|
||||||
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
|
normalization. Referenced message content and static template syntax were
|
||||||
the public operation's exact contract.
|
captured when the engine was built.
|
||||||
|
Rendering uses those captured parses, cloning them per call to bind request
|
||||||
|
input helpers; it does not parse templates during execution.
|
||||||
|
|
||||||
|
Resource inspection bypasses the runner: the internal engine selects a captured
|
||||||
|
configuration resource, and the facade detaches its values for the caller.
|
||||||
|
Prompt resource inspection includes captured message text without rendering or
|
||||||
|
touching profile, artifact, schema, admission, or model collaborators.
|
||||||
|
|
||||||
|
`InspectRequirements` uses the same metadata selection to resolve the
|
||||||
|
request-profile/default identity, then obtains the captured requirement
|
||||||
|
snapshot by the selected prompt ID. The analyzer receives only
|
||||||
|
those identity fields rather than an execution definition. It analyzes the
|
||||||
|
snapshot with the supplied caller variables and normalized direct-session
|
||||||
|
bypass; it does not resolve target readiness or enter the preparation
|
||||||
|
pipeline. An unavailable captured snapshot is a prompt-load failure. The root
|
||||||
|
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||||
|
operation's exact contract.
|
||||||
|
|
||||||
## Shared Profile Selection
|
## Shared Profile Selection
|
||||||
|
|
||||||
The runner uses one profile-selection and target-resolution boundary for
|
The runner selects a request profile or a prompt default and uses its captured,
|
||||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
presence-aware target constructed from final profile and backend configuration.
|
||||||
request profile or a prompt default; inspection begins with its required
|
Request overrides apply afterward; preparation checks endpoint and model
|
||||||
explicit profile ID. Both use the captured, presence-aware target constructed
|
completeness only after those overrides. Resource profile inspection bypasses
|
||||||
from the final effective profile and backend configuration. Request overrides
|
this path and reports the profile before backend completion. Credential
|
||||||
apply afterward; preparation checks endpoint and model completeness only after
|
availability is checked when execution preparation requires it.
|
||||||
those overrides. Inspection can report a partial profile without requiring a
|
|
||||||
runnable target. Neither path checks credential availability or performs prompt,
|
## Smoke-Test Diagnostics
|
||||||
artifact, schema, rendering, admission, or model-client work. The root
|
|
||||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
The separate `SmokeTester` in `internal/usecase` selects from the same captured
|
||||||
exact contract.
|
configuration snapshot and backend/profile collaborators as the runner. The
|
||||||
|
internal engine applies namespace-view qualification to slashless caller IDs
|
||||||
|
before diagnostic selection; captured defaults and references remain exact.
|
||||||
|
Explicit profile selection takes precedence over prompt-default selection and a
|
||||||
|
backend diagnostic default; HTTP-only selection does not resolve a default
|
||||||
|
inference profile. Exact public selection rules belong to the root
|
||||||
|
[`Engine.SmokeTest` GoDoc](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
Diagnostics do not call `Run`, `Prepare`, or the prompt preparation pipeline.
|
||||||
|
They do not render application messages, load application artifacts, use
|
||||||
|
application schemas, or inspect session templates. Private text and structured
|
||||||
|
cases and the structured case's schema remain internal to the diagnostic
|
||||||
|
operation; they are not entries in prompt/profile listings or inspection.
|
||||||
|
`InspectConfiguration` remains a separate, explicit operator-oriented view of
|
||||||
|
effective settings and source history. A diagnostic report may include selected
|
||||||
|
effective settings and identifiers, so consumers still apply their disclosure
|
||||||
|
policy.
|
||||||
|
|
||||||
|
The engine assembles the smoke tester with the runner's generation coordinator
|
||||||
|
and capacity manager. Inference holds the ordinary logical admission lease and
|
||||||
|
uses the same active permits, retries, timeout, and request evidence path as
|
||||||
|
generation. Reachability and authentication probes bypass generation capacity
|
||||||
|
and do not require a model. An engine with an injected `LLMClient` can run
|
||||||
|
inference diagnostics but has no built-in HTTP probe collaborator; HTTP-only
|
||||||
|
diagnostics report that limitation instead of falling back to another client.
|
||||||
|
The public GoDoc owns the caller-visible timeout, cancellation, report, and
|
||||||
|
transport semantics.
|
||||||
|
|
||||||
## Shared Preparation Pipeline
|
## Shared Preparation Pipeline
|
||||||
|
|
||||||
@@ -92,8 +135,16 @@ one execution-precedence and error-ordering implementation while allowing a
|
|||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations. `Prepare` discards the plan after returning its public
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
and discards it when the operation ends. Prepared execution stores the same
|
and discards it when the operation ends. Prepared execution retains one
|
||||||
kind of plan only in its private payload.
|
immutable, redacted metadata snapshot shared by execution and internal details.
|
||||||
|
The facade detaches mutable public details. Claim completion or discard clears
|
||||||
|
execution-only payload references without modifying the retained metadata.
|
||||||
|
|
||||||
|
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||||
|
path as `Prepare` and discards the resulting internal prepared value without
|
||||||
|
converting it into a public snapshot. It does not add a runner operation or
|
||||||
|
validator component; request conversion and public error mapping remain at the
|
||||||
|
facade boundary.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
@@ -117,7 +168,7 @@ encoding of that session, every role and content value, and cache-control
|
|||||||
presence and values; its hexadecimal value is opaque.
|
presence and values; its hexadecimal value is opaque.
|
||||||
|
|
||||||
The configuration snapshot is read-only after engine construction. Concurrent
|
The configuration snapshot is read-only after engine construction. Concurrent
|
||||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
`Prepare` and `Run` calls use detached captured targets and keep invocation
|
||||||
state local.
|
state local.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
@@ -234,4 +285,8 @@ Changes to orchestration should continue to use the existing package
|
|||||||
interfaces, keep request state local to an invocation, and preserve the shared
|
interfaces, keep request state local to an invocation, and preserve the shared
|
||||||
resolution and completion pipeline. Source, renderer, validator, or
|
resolution and completion pipeline. Source, renderer, validator, or
|
||||||
model-client contract changes belong first in their owning package and
|
model-client contract changes belong first in their owning package and
|
||||||
document.
|
document. The
|
||||||
|
[smoke selection tests](../../internal/usecase/smoke_selection_test.go),
|
||||||
|
[diagnostic orchestration tests](../../internal/usecase/smoke_tester_test.go),
|
||||||
|
and [private inference-case tests](../../internal/usecase/smoke_inference_test.go)
|
||||||
|
own diagnostic selection, report assembly, and inference findings.
|
||||||
|
|||||||
@@ -11,64 +11,158 @@ source semantics. Internal packages are not consumer APIs.
|
|||||||
## Typed Configuration Loading
|
## Typed Configuration Loading
|
||||||
|
|
||||||
`internal/configuration` is the single adapter from ConfigKit's final
|
`internal/configuration` is the single adapter from ConfigKit's final
|
||||||
effective configuration to execution-owned definitions. It performs one
|
effective configuration to execution-owned definitions and resource views. It performs one
|
||||||
layered load, captures effective presence (including explicit null), records
|
layered load, captures the needed effective presence, records
|
||||||
normalization and defaults through resolution, and validates all effective
|
field-level normalization and derived defaults during resolution, and validates all effective
|
||||||
backend, profile, and prompt values. Replaced lower-layer domain values are
|
root and collection backend, profile, and prompt values. Replaced lower-layer
|
||||||
not separately validated. It builds an immutable backend index and
|
domain values are not separately validated. It indexes root resources under
|
||||||
presence-aware profile targets, applying final backend defaults only after
|
their unqualified IDs and collection resources under their full
|
||||||
profile inheritance. It retains ConfigKit's result privately for explicit
|
`namespace/local-key` IDs while retaining ConfigKit's resolved tree. It builds an immutable backend index and presence-aware profile
|
||||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
targets, applying final backend defaults only after profile inheritance. Runtime
|
||||||
depend on ConfigKit.
|
lookups use those final indexes and exact stored references; they do not rewrite
|
||||||
|
or resolve resources from ConfigKit source history. The runtime/domain packages
|
||||||
|
do not depend on ConfigKit.
|
||||||
|
|
||||||
|
ConfigKit owns source headers, imports, composition, and source evidence.
|
||||||
|
Promptkit does not preparse contributions: it validates the final effective
|
||||||
|
values once, captures the indexed identities and their original resource
|
||||||
|
contexts, and retains ConfigKit's resolved result for explicit operator inspection.
|
||||||
|
Accepted non-main source headers are consumed during composition; the main
|
||||||
|
source declaration remains available in that inspection view. Collection
|
||||||
|
release labels are descriptive metadata, not selection inputs. The
|
||||||
|
[format reference](../formats.md#source-headers-collections-and-resource-ids)
|
||||||
|
owns the exact source-header and collection contracts.
|
||||||
|
|
||||||
|
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||||
|
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||||
|
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||||
|
once to ConfigKit v1.1.0 and returns its public difference records. Comparison
|
||||||
|
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||||
|
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||||
|
both results; reports may still contain ordinary application data. See the
|
||||||
|
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
|
and ConfigKit's [v1.1.0 comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md#effective-configuration-diffs).
|
||||||
|
|
||||||
|
After resource materialization and effective-profile snapshot construction, the
|
||||||
|
adapter captures lexically ordered identity inventories for backends, profiles,
|
||||||
|
and prompts, per-kind resource namespace sets, and metadata for effective
|
||||||
|
collection declarations. Listing methods return caller-owned copies from those
|
||||||
|
captured values without consulting ConfigKit or copying resource definitions.
|
||||||
|
|
||||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||||
files under an explicit root, provide resource context, and leave imports
|
files under an explicit root, provide resource context, and leave imports
|
||||||
disabled. Each returned slice belongs to one strict-composition group; file
|
disabled. Each returned slice belongs to one strict-composition group; file
|
||||||
order does not confer priority. The engine prepends selected catalog and
|
order does not confer priority. Engine assembly prepends a dedicated embedded
|
||||||
Promptkit-owned definitions, then caller underlays and optional main. The
|
OpenRouter underlay unless disabled, without modifying caller-owned slices.
|
||||||
|
Caller underlays are composed in their given
|
||||||
|
order, followed by the optional main input. The
|
||||||
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
||||||
real loader and final-value handoff.
|
real loader and final-value handoff.
|
||||||
|
|
||||||
|
The public `CollectionInput` helper mounts one caller-selected source at
|
||||||
|
`collections/<namespace>` and enables ConfigKit imports for that input. The
|
||||||
|
helper does not read the source; ConfigKit retains responsibility for loading,
|
||||||
|
composition, imports, and source-local resource context. Imported or inherited
|
||||||
|
`content_file` values continue resolving relative to the source that contributed
|
||||||
|
the effective field, not to a flattened runtime identity. Explicit
|
||||||
|
`InspectConfiguration` views expose the retained effective configuration and
|
||||||
|
provenance for operator review; runtime selection and inventories do not derive
|
||||||
|
from source history.
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
|
|
||||||
The configuration adapter indexes each prompt by exact ID and version,
|
The configuration adapter indexes each prompt by exact ID,
|
||||||
materializes file-backed message content using the exact effective field's
|
materializes file-backed message content using the exact effective field's
|
||||||
origin and resource context, and checks static message/session template syntax
|
origin and resource context, and checks static message/session template syntax
|
||||||
at construction. It stores captured content, compiled templates, declared
|
at construction. It stores captured content, compiled templates, declared
|
||||||
input metadata, and a definition hash. It closes content-file handles before
|
input metadata, and a definition hash. Alongside the unchanged hashed
|
||||||
|
definition, it retains content-type and description presence for each declared
|
||||||
|
input and provides an internal requirement snapshot with those declarations
|
||||||
|
and the captured parses. Metadata values come from the final normalized
|
||||||
|
declaration; absent and null fields remain distinguishable from explicit
|
||||||
|
strings, including empty strings. It closes content-file handles before
|
||||||
returning and retains no source capability for public inspection.
|
returning and retains no source capability for public inspection.
|
||||||
|
|
||||||
Role and cache-control normalization is applied to execution-owned message
|
Prompt materialization shares a lazy metadata snapshot across root and collection
|
||||||
copies. The retained ConfigKit configuration keeps effective input message
|
resources during construction, avoiding a full metadata copy per collection.
|
||||||
metadata and its resource origins: recording a sequence-wide normalization in
|
Input metadata, message content and templates, and output contracts are captured
|
||||||
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
|
separately before the definition, inspection, and hash are assembled.
|
||||||
|
|
||||||
The runner selects from this index for preparation and `InspectPrompt`.
|
Normalization and defaults are recorded at individual resolved field paths,
|
||||||
Neither path rescans configuration or reopens message files. `InspectPrompt`
|
including message roles and cache controls. List indexes identify fields for
|
||||||
returns declared metadata without rendering or resolving profile, input, or
|
presence, validation, sensitivity, and resource context; full IDs select
|
||||||
schema resources. Runtime rendering still checks request variables and input
|
captured runtime indexes. Unchanged content-file fields retain their origins.
|
||||||
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
Backend-to-profile and request-time completion remain separate runtime work.
|
||||||
|
|
||||||
|
The runner selects full definitions from this index for preparation and retains
|
||||||
|
lightweight prompt metadata for `InspectRequirements`; the requirement snapshot
|
||||||
|
lookup uses the selected prompt ID alone. Separate configuration-owned resource
|
||||||
|
inspection projects the resolved record and captured messages without rendering
|
||||||
|
or resolving profile, input, or schema resources. These paths do not rescan
|
||||||
|
configuration or reopen message files. Presence metadata does not change the
|
||||||
|
opaque definition hash. Runtime rendering still checks request variables and
|
||||||
|
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||||
cover resource origins, context containment, capture, and selection.
|
cover resource origins, context containment, capture, and selection.
|
||||||
|
|
||||||
## Profiles And Built-Ins
|
The configuration adapter shares ID-only selection across its internal prompt
|
||||||
|
lookups. The execution snapshot lookup returns an owned prompt
|
||||||
|
definition and cached hash; metadata inspection returns owned input metadata
|
||||||
|
without copying execution templates. Profile selection likewise has an
|
||||||
|
identity-only lookup for consumers that do not need the execution target,
|
||||||
|
while the full profile lookup continues to return an owned target and deep copy
|
||||||
|
of provider extras. These lookups use captured indexes and do not query
|
||||||
|
ConfigKit again.
|
||||||
|
|
||||||
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
|
The `internal/prompt` requirement walker uses the captured parse trees and
|
||||||
assets as one ConfigKit underlay and PromptKit-owned profiles as the next.
|
declaration metadata without executing templates or reading artifacts. It
|
||||||
It discovers YAML inputs and supplies passive filesystem context without
|
collects literal `input` names, direct root caller keys, and literal-key
|
||||||
parsing or pre-validating catalog definitions. The configuration loader
|
`index` lookups. For `if` and `with`, it classifies branches when their
|
||||||
validates only the final effective backend and profile values, including
|
conditions are simple supported values: direct values, literals, negation, or
|
||||||
consumer overlays of maintained bases or backends. There is no catalog-only
|
two-string equality/inequality. It uses Go template truthiness, so a supplied
|
||||||
preflight, reserved source namespace, or replay of backend registration.
|
nonempty string such as `false` is true. Missing decision values and
|
||||||
|
content-dependent conditions leave dependency-bearing branches conditional
|
||||||
|
and report incomplete discovery; unresolved conditions with no branch
|
||||||
|
dependencies do not by themselves make discovery incomplete. Recognizable
|
||||||
|
references on known skipped paths remain optional inventory entries with an
|
||||||
|
inactive-use explanation, unless another applicable use establishes a
|
||||||
|
stronger classification. Skipped expressions are scanned only for
|
||||||
|
syntactically recognizable names and are not evaluated. A direct caller value
|
||||||
|
or literal-key lookup can also supply an input name, including through simple
|
||||||
|
parentheses or a direct pipeline to `input`; names are used exactly as
|
||||||
|
supplied. Unresolved names and unsupported dependency-bearing expressions
|
||||||
|
produce structured limitations while retaining recognizable references.
|
||||||
|
Where an unresolved name or scope could affect existing optional entries, those
|
||||||
|
entries are reported as conditional rather than inactive-only or definitively
|
||||||
|
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
|
||||||
|
values remain separate from unknown scopes; accesses through untracked local
|
||||||
|
maps report incomplete discovery rather than silently losing caller keys.
|
||||||
|
Unsupported name expressions reuse value analysis's conservative syntax scan
|
||||||
|
instead of recursively rescanning the same subtree.
|
||||||
|
Invoked named templates are scanned conservatively, and
|
||||||
|
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
|
||||||
|
and bypassed session templates are not traversed. This internal walker does not
|
||||||
|
change rendering behavior.
|
||||||
|
|
||||||
ConfigKit resolves profile inheritance before the handoff. The configuration
|
## Profiles
|
||||||
adapter validates effective references and field values, then builds target
|
|
||||||
snapshots with backend defaults and presence/suppression state. A profile can
|
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
The configuration adapter validates final effective resource references and
|
||||||
provider work. Preparation applies runtime overrides and then checks target
|
field values, then captures backend definitions and profile target snapshots
|
||||||
completeness. The [target tests](../../internal/configuration/targets_test.go)
|
with backend defaults and outbound-control presence. Null profile endpoint,
|
||||||
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
|
credential source, and whole provider map permit backend fallback after
|
||||||
cover selected released assets.
|
composition, subject to the required-credential rule. A profile can remain
|
||||||
|
partial until runtime overrides complete it. Resource inspection selects indexed
|
||||||
|
resolved backend and profile records without execution completion or prompt,
|
||||||
|
artifact, schema, credential lookup, or provider work. Preparation applies runtime overrides and then checks target
|
||||||
|
completeness. Identity-only profile selection can identify a partial captured
|
||||||
|
profile without copying its target or requiring readiness. The
|
||||||
|
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||||
|
target interpretation.
|
||||||
|
|
||||||
|
The internal engine delegates listing through a narrow identity-inventory
|
||||||
|
interface directly to the configuration snapshot. The execution runner has no
|
||||||
|
listing responsibility.
|
||||||
|
|
||||||
## Ordinary Artifacts
|
## Ordinary Artifacts
|
||||||
|
|
||||||
@@ -79,10 +173,15 @@ and after opening the file. It copies content into an artifact, records
|
|||||||
metadata and an opaque content-equality value, and applies a content-type
|
metadata and an opaque content-equality value, and applies a content-type
|
||||||
fallback.
|
fallback.
|
||||||
|
|
||||||
Regular files are read synchronously in bounded chunks. Cancellation is
|
Regular files are read synchronously in bounded chunks. The opened file's size
|
||||||
checked before opening, before and after every read, and before publishing the
|
is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
|
||||||
artifact, so a canceled read never publishes partial content. The ordinary
|
the returned body determines artifact size and hash. If an overreported hint
|
||||||
reader does not detach file reads into background goroutines.
|
leaves unused capacity greater than the actual body length, the completed body
|
||||||
|
is copied to release the excess backing storage, including when the body is
|
||||||
|
empty. Cancellation is checked before opening, before and after every read,
|
||||||
|
and before publishing the artifact, so a
|
||||||
|
canceled read never publishes partial content. The ordinary reader does not
|
||||||
|
detach file reads into background goroutines.
|
||||||
|
|
||||||
This ordinary reader does not implement an inbound HTTP security boundary. In
|
This ordinary reader does not implement an inbound HTTP security boundary. In
|
||||||
particular, it does not constrain files to an application root or impose an
|
particular, it does not constrain files to an application root or impose an
|
||||||
@@ -106,7 +205,9 @@ between artifact conversion chunks, around each message, and before publishing
|
|||||||
the complete prompt. It cannot interrupt template work already in progress and
|
the complete prompt. It cannot interrupt template work already in progress and
|
||||||
never publishes a partial prompt after observing cancellation. It validates and
|
never publishes a partial prompt after observing cancellation. It validates and
|
||||||
canonicalizes message roles before carrying roles, session IDs, and cache
|
canonicalizes message roles before carrying roles, session IDs, and cache
|
||||||
control into the rendered prompt. The
|
control into the rendered prompt. Message and session templates execute into
|
||||||
|
per-call string builders, so their rendered strings do not require a final
|
||||||
|
buffer-to-string copy. The
|
||||||
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
|
||||||
|
|
||||||
## Schemas And Output Validation
|
## Schemas And Output Validation
|
||||||
@@ -114,16 +215,19 @@ control into the rendered prompt. The
|
|||||||
`internal/validate` provides validators backed by an operating-system
|
`internal/validate` provides validators backed by an operating-system
|
||||||
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error.
|
error. Basic validation checks body emptiness on the byte slice directly
|
||||||
|
instead of first making a response-sized string copy.
|
||||||
|
|
||||||
Every preparation operation creates one operation-local validation plan. None,
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
basic, and JSON modes retain the effective output contract without source
|
basic, and JSON modes retain the effective output contract without source
|
||||||
access. JSON Schema mode loads the root document once, resolves and compiles
|
access. JSON Schema mode loads the root document once, compiles its declared
|
||||||
each transitive reference, and retains the compiled validator. Schema compiler
|
schema locations, and retains the compiled root validator. External resources
|
||||||
resources use canonical escaped file or private-scheme URLs; loaders decode
|
are rejected without file or network loading; in-document references remain
|
||||||
their paths once and enforce the configured source boundary. The
|
supported. Root resources use canonical escaped file or private-scheme URLs. The
|
||||||
provider-facing structured-output metadata uses the root document captured by
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
the same plan.
|
the same plan. Its detached schema document is not revalidated against the
|
||||||
|
runtime provider-value copier's traversal limits. Smoke inference uses a
|
||||||
|
separate JSON-only prepared plan before checking the prepared schema plan.
|
||||||
|
|
||||||
Schema preparation and execution remain synchronous. Promptkit checks
|
Schema preparation and execution remain synchronous. Promptkit checks
|
||||||
cancellation before and after source resolution, JSON decoding, compilation,
|
cancellation before and after source resolution, JSON decoding, compilation,
|
||||||
|
|||||||
@@ -6,59 +6,51 @@ Material: policy documents under `docs/policy/` and the Policy Reading Guide in
|
|||||||
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
|
||||||
without modification from upstream `agents/AGENTS.md`.
|
without modification from upstream `agents/AGENTS.md`.
|
||||||
|
|
||||||
|
The release policy uses upstream commit
|
||||||
|
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
|
||||||
|
`d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: CC-BY-SA-4.0
|
SPDX-License-Identifier: CC-BY-SA-4.0
|
||||||
|
|
||||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
|
||||||
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||||
|
|
||||||
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
|
||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
|
Selected module: `go.maximumdirect.net/configkit` at `v1.1.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
|
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
|
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog v3
|
## ReleaseKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3`
|
Selected module: `go.maximumdirect.net/releasekit` at `v0.9.0` (maintainer tool).
|
||||||
at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
|
[LICENSE](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
|
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4).
|
||||||
|
|
||||||
## Promptkit Rakestrawhome Catalog v3
|
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3`
|
|
||||||
at `v3.20260930.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
|
|
||||||
|
|
||||||
## jsonschema
|
## jsonschema
|
||||||
|
|
||||||
|
|||||||
@@ -51,8 +51,8 @@ The [internal component overview](../internal/overview.md) owns the inventory
|
|||||||
of implemented packages and their concrete responsibilities. This policy owns
|
of implemented packages and their concrete responsibilities. This policy owns
|
||||||
the boundaries and dependency rules those components must preserve.
|
the boundaries and dependency rules those components must preserve.
|
||||||
|
|
||||||
The internal engine composes selected catalog and caller ConfigKit inputs once,
|
The internal engine composes caller-selected ConfigKit inputs once, then
|
||||||
then assembles the captured definitions, capacity manager, renderer, validator,
|
assembles the captured definitions, capacity manager, renderer, validator,
|
||||||
outbound client, and use-case runner. The configuration adapter interprets and
|
outbound client, and use-case runner. The configuration adapter interprets and
|
||||||
validates only the final effective values, including presence, null, and
|
validates only the final effective values, including presence, null, and
|
||||||
resource context. It retains source history for explicit operator inspection,
|
resource context. It retains source history for explicit operator inspection,
|
||||||
@@ -97,7 +97,8 @@ on consumers or on Scriptorium.
|
|||||||
`internal/domain` owns source-neutral invariants for values shared across
|
`internal/domain` owns source-neutral invariants for values shared across
|
||||||
multiple input and execution boundaries, including execution-setting bounds,
|
multiple input and execution boundaries, including execution-setting bounds,
|
||||||
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
OpenAI-compatible base endpoints, session identifiers, and output-contract
|
||||||
legality. Callers retain source parsing, required-field rules, other
|
legality, plus shared resource-identity syntax and qualification. Callers retain
|
||||||
|
source parsing, required-field rules, other
|
||||||
source-specific normalization, defaulting, error classification, and policy
|
source-specific normalization, defaulting, error classification, and policy
|
||||||
specific to their own boundary.
|
specific to their own boundary.
|
||||||
|
|
||||||
|
|||||||
@@ -88,7 +88,7 @@ secret values.
|
|||||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||||
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
| Exact exported Go API | Go declarations and GoDoc, as APIs are implemented | Exported names, signatures, types, values, errors, and exact behavioral contracts. | Task-oriented consumer walkthroughs, implementation details, and future API proposals. |
|
||||||
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, built-in profiles, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
| Framework file formats | `docs/formats.md` | Prompt-definition and profile YAML fields, schema references, defaults, validation modes, credentials, and file-to-request precedence. | Exported Go declarations, outbound wire behavior, internal parsing mechanics, and application configuration. |
|
||||||
|
|
||||||
Create supporting documents for existing interfaces and responsibilities when
|
Create supporting documents for existing interfaces and responsibilities when
|
||||||
substantive content is available.
|
substantive content is available.
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.8
|
**Base Policy Version:** 1.8
|
||||||
|
|
||||||
**Last Updated**: 2026-09-27
|
**Last Updated**: 2026-11-02
|
||||||
|
|
||||||
**Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
**Upstream Source:** [oss.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
||||||
|
|
||||||
@@ -221,7 +221,8 @@ Promptkit retains its LGPL-3.0-or-later library grant in
|
|||||||
[COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate
|
[COPYRIGHT](../../COPYRIGHT). Adapted policy documentation has the separate
|
||||||
scope identified there and in [attribution](../oss/attribution.md).
|
scope identified there and in [attribution](../oss/attribution.md).
|
||||||
|
|
||||||
Promptkit publishes source-only Go module tags on Gitea, with no binaries or
|
Promptkit publishes source-only Go module tags and Gitea release pages, with no
|
||||||
separate hosted download page. The [release procedure](../release.md) owns
|
binaries. Each new hosted page reuses the complete checked-in release note,
|
||||||
publication and concrete footer URLs. The hosted-page guidance applies if such
|
including its licensing footer. The [release procedure](../release.md) owns
|
||||||
a channel is deliberately introduced; it does not require creating one.
|
source publication, hosted delivery, and concrete footer URLs. Previously
|
||||||
|
published notes and tags retain their original records.
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# Release Policy
|
# Release Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.4
|
**Base Policy Version:** 1.6
|
||||||
|
|
||||||
**Last Updated**: 2026-09-27
|
**Last Updated**: 2026-11-02
|
||||||
|
|
||||||
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
|
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
|
|||||||
intended source revision under a fresh version. Releases should require little
|
intended source revision under a fresh version. Releases should require little
|
||||||
routine intervention, avoid repeated work, and preserve published contents.
|
routine intervention, avoid repeated work, and preserve published contents.
|
||||||
|
|
||||||
|
Adopt stronger verification or reproducibility requirements through an explicit
|
||||||
|
project-specific decision justified by the intended audience and support needs.
|
||||||
|
|
||||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||||
policy and link to it rather than introduce additional release requirements.
|
policy and link to it rather than introduce additional release requirements.
|
||||||
@@ -59,7 +62,8 @@ requires a deliberate update to the policy and procedure where needed.
|
|||||||
For releases at or after `v1.0.0`:
|
For releases at or after `v1.0.0`:
|
||||||
|
|
||||||
- a major release may make intentional incompatible changes;
|
- a major release may make intentional incompatible changes;
|
||||||
- a minor release may add backward-compatible behavior; and
|
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||||
|
minimum as described below; and
|
||||||
- a patch release may make backward-compatible fixes.
|
- a patch release may make backward-compatible fixes.
|
||||||
|
|
||||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||||
@@ -71,14 +75,44 @@ Additional compatibility promises and deliberate exceptions belong in the
|
|||||||
adopted policy or canonical contract documentation. Release notes explain
|
adopted policy or canonical contract documentation. Release notes explain
|
||||||
changes; they do not redefine current compatibility contracts.
|
changes; they do not redefine current compatibility contracts.
|
||||||
|
|
||||||
## Release Documentation
|
## Consumer Minimum And Release Toolchain
|
||||||
|
|
||||||
|
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||||
|
maintainers use to build and validate a release. Record the consumer minimum in
|
||||||
|
the language's standard metadata and explain its support meaning in canonical
|
||||||
|
documentation. Define the selected release toolchain and its setup commands in
|
||||||
|
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||||
|
choices.
|
||||||
|
|
||||||
|
For Go repositories, normally declare the initial stable release of the selected
|
||||||
|
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||||
|
when a dependency or an identified correctness fix requires it, and document
|
||||||
|
the reason.
|
||||||
|
|
||||||
|
For release-policy purposes, successful build and release validation on the
|
||||||
|
selected Go patch release are sufficient to presume support across that Go
|
||||||
|
series, subject to the declared consumer minimum and concrete evidence of
|
||||||
|
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||||
|
|
||||||
|
Use the selected release toolchain for candidate validation and release builds,
|
||||||
|
including CI jobs performing those tasks. A newer validation patch version alone
|
||||||
|
leaves the consumer minimum unchanged.
|
||||||
|
|
||||||
|
A minor release may raise the Go consumer minimum, including from one Go series
|
||||||
|
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||||
|
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||||
|
the consumer minimum. Explain an increase's impact and required consumer action
|
||||||
|
in the release note. Other incompatible changes remain subject to the versioning
|
||||||
|
rules above.
|
||||||
|
|
||||||
|
## Release Documentation And Preparation
|
||||||
|
|
||||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||||
source tree. Use the complete version tag as the filename and begin with a
|
source tree. Use the complete version tag as the filename and begin with a
|
||||||
title containing the product name and that exact version.
|
title containing the product name and that exact version.
|
||||||
|
|
||||||
Include a short summary and any relevant compatibility impact or upgrade
|
Include a short summary and any relevant compatibility impact or upgrade
|
||||||
instructions. Use the Promptkit section structure below and keep the note
|
instructions. Choose sections suited to the release and keep the note
|
||||||
proportionate to the change.
|
proportionate to the change.
|
||||||
|
|
||||||
Keep exact current contracts in their canonical documents and update affected
|
Keep exact current contracts in their canonical documents and update affected
|
||||||
@@ -186,7 +220,8 @@ release without replacing its contents or reusing its version.
|
|||||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||||
identify:
|
identify:
|
||||||
|
|
||||||
1. prerequisites, primary branch, and authoritative remote;
|
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||||
|
remote;
|
||||||
2. version and release-note preparation;
|
2. version and release-note preparation;
|
||||||
3. one publication command with its selected local checks and automated guards;
|
3. one publication command with its selected local checks and automated guards;
|
||||||
4. declared deliverables, required build targets, and automation responsibilities;
|
4. declared deliverables, required build targets, and automation responsibilities;
|
||||||
@@ -206,47 +241,52 @@ navigation on 2026-09-26. Apply them prospectively; previously published notes
|
|||||||
and tags retain their original records.
|
and tags retain their original records.
|
||||||
|
|
||||||
Promptkit publishes source-only Go modules from main to origin on Gitea, using
|
Promptkit publishes source-only Go modules from main to origin on Gitea, using
|
||||||
annotated stable SemVer tags. It has no binaries, hosted download pages, or
|
annotated stable SemVer tags. The source commit is pushed through main before
|
||||||
asynchronous deliverables. The source commit is pushed through main before the
|
the tag. After source publication, maintainers use ReleaseKit locally to deliver
|
||||||
tag. The [release procedure](../release.md) owns commands and recovery.
|
a Gitea release page containing the complete checked-in note, including its
|
||||||
|
licensing footer. There are no binaries or other deliverables. Source publication
|
||||||
|
and hosted delivery have separate outcomes; delivery credentials and service
|
||||||
|
availability do not gate source-release completion. The
|
||||||
|
[release procedure](../release.md) owns commands and recovery.
|
||||||
|
|
||||||
The existing publication script requires Summary, Compatibility, Upgrade, and
|
Release preparation uses the latest Go installation available in the development
|
||||||
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State
|
environment to build the maintainer release executable and validate the
|
||||||
“No action required” or “None” where appropriate. This retained local structure
|
candidate, including the consumer examples. Keep that same local toolchain
|
||||||
matches the automated release-note guard; changing it is separate work.
|
active throughout those operations.
|
||||||
|
|
||||||
The [publication script](../../scripts/release.sh) runs the
|
For a minor release, advance Promptkit's consumer minimum to the initial stable
|
||||||
[local checks](../../scripts/release-check.sh) once for the final committed
|
release of that Go series, retaining a higher patch minimum when required by a
|
||||||
candidate: race-enabled tests, vet, three offline consumer examples, tracked
|
dependency or an identified correctness fix. Patch releases preserve the
|
||||||
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a
|
existing consumer minimum. Commit any module update before validating the final
|
||||||
credential-pattern scan. These protect concurrency, assembled library use,
|
candidate. The [release procedure](../release.md#prepare-the-candidate) owns
|
||||||
source distribution, and documentation integrity. A separate ordinary test run
|
setup commands, and the consumer guide explains
|
||||||
or standalone build is not an additional release gate. Candidate, tag, and
|
[Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
|
||||||
synchronized-main checks remain automated. No CI or module-availability wait
|
|
||||||
is required. Licensing footer URLs are preparation work, not a network gate.
|
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
Use a title containing `Promptkit` and the exact version, followed by a short
|
||||||
|
summary and any relevant compatibility impact or upgrade instructions. The
|
||||||
|
note is committed with the candidate and supplies the annotated tag message.
|
||||||
|
No fixed set of section headings is required.
|
||||||
|
|
||||||
External catalog profile data may be added, removed, renamed, or modified
|
The [publication command](../../scripts/release.sh) delegates to the released
|
||||||
without deprecation or advance notice. This exception is limited to profile
|
ReleaseKit tool selected in `go.mod`. The committed
|
||||||
data imported from the maintained catalog modules. It does not weaken ordinary
|
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
|
||||||
compatibility for Promptkit's public Go API, configuration and format semantics,
|
committed Go formatting, local Markdown links, workspace and filesystem
|
||||||
runtime behavior, catalog-module APIs, manifest schema, backend definitions, or
|
replacement inspection. The Go suite includes the offline semantic checks for
|
||||||
consumer-owned definitions.
|
the three maintained consumer examples. It also rejects the configured local
|
||||||
|
delivery token path if committed; this narrowly scoped check is not a general credential
|
||||||
|
scan. These are the selected checks for the final candidate; a separate
|
||||||
|
ordinary test run or standalone build is not an
|
||||||
|
additional release gate. The upstream tool owns candidate, remote, and tag
|
||||||
|
publication checks. It returns native status 0 for confirmed publication, 1
|
||||||
|
for a known validation or publication failure, 2 for invalid input or profile,
|
||||||
|
and 3 when the push outcome is unknown. For status 3, resolve the exact
|
||||||
|
captured publication endpoint before retrying; the repository-specific
|
||||||
|
[procedure](../release.md#failure-recovery) links to the upstream recovery
|
||||||
|
instructions. Do not wait for CI or Go module availability. Licensing footer
|
||||||
|
URLs are preparation work, not a network gate.
|
||||||
|
|
||||||
Within the catalog versions selected and tested by a Promptkit release, and
|
Hosted delivery is a separate post-publication step for new releases. It reuses
|
||||||
unless a consumer overrides the name or its base, Promptkit keeps the literal
|
the immutable tag and note without rerunning candidate validation or waiting
|
||||||
profile IDs `promptkit-openrouter-standard` and
|
for CI. Retry failed delivery against that same version after resolving any
|
||||||
`promptkit-openrouter-auto` available. The names do not promise a permanent
|
uncertain hosted write; preserve published tags and hosted content. Report
|
||||||
model, backend, routing behavior, settings, cost, output, or upstream
|
delivery status separately from the completed source release.
|
||||||
availability. The [framework format reference](../formats.md#built-in-profile-catalog)
|
|
||||||
owns their current definitions.
|
|
||||||
|
|
||||||
`v0.10.0` is the initial minor release for this compatibility transition.
|
|
||||||
Afterward, catalog-only updates and adjustments to the current selections may
|
|
||||||
ship in a patch release when all other compatibility obligations remain intact.
|
|
||||||
External profile removals can require consumer action; they are not automatic
|
|
||||||
migrations.
|
|
||||||
|
|
||||||
Catalog adoption and release validation remain local and offline. They do not
|
|
||||||
probe provider endpoints, request model lists, or make inference calls.
|
|
||||||
|
|||||||
@@ -58,17 +58,24 @@ and repository hygiene.
|
|||||||
Introducing hosted CI later would supplement, not silently redefine, this
|
Introducing hosted CI later would supplement, not silently redefine, this
|
||||||
documented validation model.
|
documented validation model.
|
||||||
|
|
||||||
Full release-candidate validation includes race-enabled package tests, explicit
|
The selected ReleaseKit profile runs the race-enabled package suite and full
|
||||||
static analysis, and execution with automated semantic checks of the three maintained
|
vet, checks committed Go formatting, local Markdown links, and module
|
||||||
offline consumer examples. The test run builds the packages; the examples
|
distribution hygiene. The Go suite includes semantic assertions for the three
|
||||||
also exercise ordinary compilation. Separate ordinary tests and a standalone
|
maintained offline consumer examples. ReleaseKit owns this shared validation
|
||||||
build are not additional release gates. Ordinary tests remain useful during
|
machinery and publication workflow. The test run builds the packages and
|
||||||
development. The preparation example protects assembled preparation and
|
exercises the examples. Separate ordinary tests and a
|
||||||
|
standalone build are not additional release gates. Ordinary tests remain useful
|
||||||
|
during development. The preparation example protects assembled preparation and
|
||||||
inspection behavior. The execution example separately protects assembled
|
inspection behavior. The execution example separately protects assembled
|
||||||
`Run`, injected-client, validation, usage, and result behavior. The observation
|
`Run`, injected-client, validation, usage, and result behavior. The observation
|
||||||
example protects assembled built-in retries, structured proposal and assessment
|
example protects assembled built-in retries, structured proposal and assessment
|
||||||
runs, and application-owned interpretation of metadata.
|
runs, and application-owned interpretation of metadata.
|
||||||
|
|
||||||
|
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
|
||||||
|
ReleaseKit wiring against disposable local repositories. It is focused
|
||||||
|
development validation when release tooling changes, not part of the default
|
||||||
|
Go test suite or a recurring release check.
|
||||||
|
|
||||||
Tests in the default suite must be deterministic, offline, and independent of
|
Tests in the default suite must be deterministic, offline, and independent of
|
||||||
real credentials. They must not invoke paid APIs, use live network
|
real credentials. They must not invoke paid APIs, use live network
|
||||||
dependencies, or depend on mutable external services. Tests that require live
|
dependencies, or depend on mutable external services. Tests that require live
|
||||||
|
|||||||
194
docs/release.md
194
docs/release.md
@@ -1,97 +1,159 @@
|
|||||||
# Release Procedure
|
# Release Procedure
|
||||||
|
|
||||||
This runbook implements the [release policy](policy/release.md). A successful
|
This runbook implements the [release policy](policy/release.md). Promptkit
|
||||||
push of the selected annotated tag to origin completes the release. Never wait
|
publishes source-only Go module tags from `main` to `origin`, followed by a
|
||||||
for CI to finish or succeed.
|
page-only Gitea release. Successful publication of the intended tag, with its
|
||||||
|
source commit available on `main`, completes the source release. Report hosted
|
||||||
|
delivery separately. Do not wait for CI or Go module availability.
|
||||||
|
|
||||||
## Prepare The Candidate
|
## Prepare The Candidate
|
||||||
|
|
||||||
Required tools are Git, Go with race-detector support, POSIX shell, and Python
|
Required tools are Git, Go with race-detector support, and POSIX shell. Work
|
||||||
3. Publication requires push access to origin. Work from the repository root.
|
from the repository root with push access to `origin`.
|
||||||
|
|
||||||
Select an unused version and create `docs/releases/<version>.md` using the
|
Use the latest Go installation available locally for the release tool and its
|
||||||
[required structure](policy/release.md#release-documentation). Describe
|
child commands. Put it first on `PATH` and keep it selected throughout:
|
||||||
compatibility impact and consumer actions and update affected canonical docs.
|
|
||||||
For catalog updates, record selected catalog versions and any consumer action,
|
|
||||||
linking to the catalog policy rather than repeating its contracts.
|
|
||||||
|
|
||||||
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
|
```sh
|
||||||
changes. Each new note must end with the following navigation, replacing
|
export GOTOOLCHAIN=local
|
||||||
`vMAJOR.MINOR.PATCH` with the selected release tag:
|
go version
|
||||||
|
|
||||||
```markdown
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Check these file paths against the candidate; a future tag need not resolve
|
For a minor release, normally advance the consumer minimum to the initial
|
||||||
remotely during preparation. The annotated tag derives from this same note,
|
stable patch of that Go series:
|
||||||
including its footer. Promptkit publishes no separate hosted release body or
|
|
||||||
download page. If such a channel is deliberately introduced, reuse the note
|
|
||||||
and preserve its footer. No extra network checks, artifacts, or approval gates
|
|
||||||
are introduced, and published notes are not retrofitted.
|
|
||||||
|
|
||||||
Retire completed roadmaps, preserving planned or deferred work and repairing
|
```sh
|
||||||
links. This is housekeeping, not a separate approval gate. Review changes
|
release_go_version=$(go env GOVERSION)
|
||||||
through the ordinary development workflow, commit them, and push main to
|
release_go_series=${release_go_version#go}
|
||||||
origin. No repeated release-specific diff review is required.
|
release_go_series=${release_go_series%.*}
|
||||||
|
go mod edit "-go=${release_go_series}.0"
|
||||||
|
```
|
||||||
|
|
||||||
|
Retain a higher patch minimum when required by a dependency or identified fix.
|
||||||
|
For a patch release, preserve the existing minimum. Explain minimum increases
|
||||||
|
under the [versioning policy](policy/release.md#versioning-and-compatibility).
|
||||||
|
Prepare dependency caches before committing the candidate:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go mod download
|
||||||
|
```
|
||||||
|
|
||||||
|
Choose an unused stable version and create `docs/releases/<version>.md` with a
|
||||||
|
title containing `Promptkit` and that exact version, a short summary, and any
|
||||||
|
compatibility impact or upgrade instructions. End the note with:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
|
||||||
|
```
|
||||||
|
|
||||||
|
Substitute the selected version and check the paths against the candidate;
|
||||||
|
future tag URLs need not resolve during preparation. Follow the
|
||||||
|
[OSS policy](policy/oss.md). Historical notes and tags remain unchanged.
|
||||||
|
|
||||||
|
Supply delivery credentials through `RELEASEKIT_GITEA_TOKEN` or the ignored
|
||||||
|
local file `.releasekit/gitea-token`, with owner-only file permissions. A
|
||||||
|
nonblank environment value takes precedence. Source publication neither reads
|
||||||
|
nor requires this token. The [profile](../releasekit.json) selects the token
|
||||||
|
path for committed-file protection; keep credentials out of commits and logs.
|
||||||
|
|
||||||
|
Retire completed roadmaps, preserve unfinished work, and repair affected links.
|
||||||
|
Review and commit the candidate on `main`, including any module updates, then
|
||||||
|
push `main` to `origin` before publishing its tag.
|
||||||
|
|
||||||
## Validate And Publish
|
## Validate And Publish
|
||||||
|
|
||||||
Run the following with the chosen unused version (replace the placeholder):
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./scripts/release.sh vMAJOR.MINOR.PATCH
|
./scripts/release.sh vMAJOR.MINOR.PATCH
|
||||||
```
|
```
|
||||||
|
|
||||||
The [publication script](../scripts/release.sh) checks a clean, synchronized
|
The launcher uses `go tool` to build or reuse the module-selected ReleaseKit
|
||||||
main candidate, the version and unused tag, and the release note's title and
|
executable in Go's cache, using the selected local toolchain. ReleaseKit runs
|
||||||
required headings. It runs the [local checks](../scripts/release-check.sh)
|
the [profile's checks](../releasekit.json) once, verifies publication state, and
|
||||||
once, rechecks candidate identity and publication state, generates the
|
publishes only the intended tag. Preserve its native exit status and diagnostic.
|
||||||
annotation from the committed note plus the validated commit, and pushes only
|
Successful publication needs no separate remote read-back or CI wait.
|
||||||
the selected annotated tag.
|
|
||||||
|
|
||||||
There is no separate tag-message authoring, example-output inspection,
|
For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`.
|
||||||
metadata inspection, or human sign-off. Compatibility and release-note content
|
It is not a required step before publication. The upstream
|
||||||
remain ordinary editorial responsibilities. Development validation is useful
|
[candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/check.md)
|
||||||
but does not require a second full run before invoking the publication script.
|
and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/cli.md)
|
||||||
|
own shared checks and exit-status definitions.
|
||||||
|
|
||||||
The script stops successfully after the tag push. CI completion, CI success,
|
## Deliver The Hosted Release
|
||||||
remote read-back, and Go module availability are not release completion gates.
|
|
||||||
Do not wait or poll for them. Provider probes and inference calls are never
|
|
||||||
part of release validation.
|
|
||||||
|
|
||||||
## Catalog Profile Compatibility
|
After source publication, keep a clean checkout at the published tag's commit:
|
||||||
|
|
||||||
The [release policy](policy/release.md#catalog-profile-compatibility) owns the
|
```sh
|
||||||
catalog compatibility boundary. This heading preserves older incoming links.
|
./scripts/releasekit.sh deliver vMAJOR.MINOR.PATCH
|
||||||
|
```
|
||||||
|
|
||||||
|
The profile selects page-only Gitea delivery for this repository. The hosted
|
||||||
|
body is the complete checked-in note, including its licensing footer. There are
|
||||||
|
no binary assets, checksum manifests, candidate revalidation, or CI waits.
|
||||||
|
ReleaseKit checks delivery credentials during delivery, not as a publication
|
||||||
|
gate. Report the hosted URL on success; delivery failure leaves the source
|
||||||
|
release complete.
|
||||||
|
|
||||||
|
For a later retry, use a clean checkout of the original tag with its recorded
|
||||||
|
module files and supply the token there. In a separate checkout, for example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
(
|
||||||
|
set -e
|
||||||
|
release_version=vMAJOR.MINOR.PATCH
|
||||||
|
release_tag_ref="refs/tags/$release_version"
|
||||||
|
git fetch origin "$release_tag_ref:$release_tag_ref"
|
||||||
|
release_tag_commit=$(git rev-parse --verify "$release_tag_ref^{commit}")
|
||||||
|
git switch --detach "$release_tag_commit"
|
||||||
|
release_worktree_status=$(git status --porcelain --untracked-files=all)
|
||||||
|
if [ -n "$release_worktree_status" ]; then
|
||||||
|
printf '%s\n' 'Delivery retry requires a clean tagged checkout.' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
./scripts/releasekit.sh deliver "$release_version"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Resolve uncertain hosted writes before retrying the same immutable version.
|
||||||
|
Matching content is accepted; conflicts are not overwritten. Do not rerun
|
||||||
|
`publish` or move tags to repair delivery. Follow the upstream
|
||||||
|
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/delivery.md)
|
||||||
|
for result interpretation and recovery. These commands apply to tags containing
|
||||||
|
this delivery configuration; older releases retain their original workflows.
|
||||||
|
|
||||||
## Failure Recovery
|
## Failure Recovery
|
||||||
|
|
||||||
If local checks fail, fix the candidate, commit and push changes, then rerun
|
For source-publication failures, use ReleaseKit's status and diagnostic to
|
||||||
the publication command. It never automatically removes a tag.
|
distinguish a candidate defect from a publication conflict. Correct and commit
|
||||||
|
candidate defects, push `main`, and rerun publication as appropriate. A failure
|
||||||
|
after local tag creation may leave a ref even when its object ID is unavailable.
|
||||||
|
|
||||||
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
|
Status 3 means the push outcome remains unknown. Do not retry or remove the local
|
||||||
attempted version and inspect origin before retrying:
|
tag until the authoritative outcome is resolved. ReleaseKit attempts to save
|
||||||
|
private publication evidence and reports a record path or a separate recording
|
||||||
|
warning. A recording failure does not change the publication outcome.
|
||||||
|
|
||||||
|
Use the recovery command reported for that attempt, or discover local records
|
||||||
|
and explicitly select the matching attempt:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
./scripts/releasekit.sh publication-attempts
|
||||||
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
|
./scripts/releasekit.sh reconcile --record /absolute/path/to/selected-record.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Matching remote tag and peeled commit values confirm publication, even if the
|
Discovery is local; reconciliation observes the original destination without
|
||||||
push command reported an error. If the remote query fails, publication status
|
rerunning validation, changing tags, or delivering hosted content. Neither is
|
||||||
is still unknown. Do not delete or recreate the local tag until absence on
|
a routine gate after successful publication. Follow the upstream
|
||||||
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
|
[recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/release.md#resolve-publication-failures)
|
||||||
local tag can be removed and the publication command rerun. Never overwrite
|
for outcomes, incomplete evidence, and older attempts without records.
|
||||||
a published tag.
|
|
||||||
|
|
||||||
CI or module-resolution problems may be investigated separately; temporary
|
Records are stored under Git's common directory by default and can contain
|
||||||
service, network, or authentication failures do not require another release.
|
credential-bearing endpoints. Keep them private, out of source control and
|
||||||
Only an actual defect requiring changes to published contents warrants a new
|
shared logs; retain them while needed for recovery and remove them when that
|
||||||
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
|
value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
|
||||||
|
for storage and retention details.
|
||||||
|
|
||||||
## Policy Changes
|
Current absence does not prove a version was never published. Remove a retained
|
||||||
|
local tag only after establishing absence and confirming it was not already
|
||||||
Update the policy and this procedure together when changing release governance.
|
published or used by another release channel. Never reuse a known published
|
||||||
Consumers own their dependency-availability and release checks; their readiness
|
version or replace a published tag or hosted page. Corrections require a new
|
||||||
does not delay completion of a Promptkit release.
|
appropriate version.
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ the existing Go API. This refactor requires no consumer code changes.
|
|||||||
|
|
||||||
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
This release selects OpenRouter `v2.20260922.1` and Rakestrawhome
|
||||||
`v2.20260922.0`; their exact inventories are linked from the
|
`v2.20260922.0`; their exact inventories are linked from the
|
||||||
[format reference](../formats.md#external-profile-inventories).
|
[migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
|
||||||
|
|
||||||
The maintained names provide continuity only within the catalog versions
|
The maintained names provide continuity only within the catalog versions
|
||||||
selected and tested by a Promptkit release and when consumers do not override
|
selected and tested by a Promptkit release and when consumers do not override
|
||||||
@@ -37,8 +37,8 @@ an available profile or supply their own complete definition; the new
|
|||||||
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
`deepseek-flash-latest` and `deepseek-pro-latest` profiles are not automatic
|
||||||
aliases and need not preserve the previous settings.
|
aliases and need not preserve the previous settings.
|
||||||
|
|
||||||
See the [release policy](../release.md#catalog-profile-compatibility) for the
|
See the [release policy](../policy/release.md#versioning-and-compatibility) for the
|
||||||
compatibility boundary, the [format reference](../formats.md#built-in-profile-catalog)
|
compatibility boundary, the [migration identity mapping](../consumers/configuration-migration.md#identity-mapping)
|
||||||
for current Promptkit-owned definitions and selected external inventories, and
|
for current resource mappings, and
|
||||||
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
the [consumer guide](../consumers/pkg-promptkit.md#choose-a-profile-ownership-model)
|
||||||
for configuration choices.
|
for configuration choices.
|
||||||
|
|||||||
@@ -72,5 +72,5 @@ for a structured proposal-and-assessment workflow.
|
|||||||
|
|
||||||
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
Backend catalog selections are unchanged from v0.10.0: OpenRouter
|
||||||
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
`v2.20260922.1` and Rakestrawhome `v2.20260922.0`. See the
|
||||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility)
|
[release compatibility policy](../policy/release.md#versioning-and-compatibility)
|
||||||
and [profile inventories](../formats.md#external-profile-inventories).
|
and [migration identity mapping](../consumers/configuration-migration.md#identity-mapping).
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ No public API, runtime behavior, or profile changes are introduced.
|
|||||||
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
Promptkit remains LGPL-3.0-or-later; the catalogs are now MIT-licensed.
|
||||||
See the [project grant](../../COPYRIGHT),
|
See the [project grant](../../COPYRIGHT),
|
||||||
[component attribution](../oss/attribution.md), and
|
[component attribution](../oss/attribution.md), and
|
||||||
[catalog compatibility policy](../policy/release.md#catalog-profile-compatibility).
|
[release compatibility policy](../policy/release.md#versioning-and-compatibility).
|
||||||
|
|
||||||
## Upgrade
|
## Upgrade
|
||||||
|
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ for API and source-selection changes.
|
|||||||
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
|
||||||
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
|
||||||
assets directly must adopt the keyed format. The maintained Promptkit profile
|
assets directly must adopt the keyed format. The maintained Promptkit profile
|
||||||
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
|
IDs remain available under the [release compatibility policy](../policy/release.md#versioning-and-compatibility),
|
||||||
which does not promise fixed profile data.
|
which does not promise fixed profile data.
|
||||||
|
|
||||||
## Upgrade
|
## Upgrade
|
||||||
|
|||||||
69
docs/releases/v0.13.0.md
Normal file
69
docs/releases/v0.13.0.md
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
# Promptkit v0.13.0
|
||||||
|
|
||||||
|
This release adds prompt requirement discovery and profile/prompt listing,
|
||||||
|
reduces avoidable configuration and execution allocations, and standardizes
|
||||||
|
maintainer publication through ReleaseKit. The minimum supported Go version
|
||||||
|
increases to 1.26.0.
|
||||||
|
|
||||||
|
## Discover Requirements And Available Definitions
|
||||||
|
|
||||||
|
- `Engine.InspectRequirements` reports recognizable input and caller-variable
|
||||||
|
dependencies for a selected prompt and supplied caller context before input
|
||||||
|
artifacts exist. It classifies dependencies as required, conditional, or
|
||||||
|
optional and reports structured limitations when discovery is incomplete.
|
||||||
|
Discovery coverage is not proof that requirements are satisfied or execution
|
||||||
|
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
|
||||||
|
and [GoDoc](../../requirements.go).
|
||||||
|
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
|
||||||
|
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
|
||||||
|
engine's captured configuration, are lexically ordered and caller-owned, and
|
||||||
|
include consumer definitions and maintained profiles that remain effective.
|
||||||
|
Listing performs no provider or execution work. See the
|
||||||
|
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections)
|
||||||
|
and [GoDoc](../../listing.go).
|
||||||
|
|
||||||
|
## Efficiency And Tooling
|
||||||
|
|
||||||
|
- Reduce response-sized copies in basic validation and rendered-message output,
|
||||||
|
and encode valid provider extra parameters only as part of the complete
|
||||||
|
request rather than serializing them twice.
|
||||||
|
- Use bounded initial file-size reservations without limiting accepted content,
|
||||||
|
releasing excess backing storage for small or empty completed artifacts.
|
||||||
|
- Reuse owned preparation values when creating prepared-execution snapshots,
|
||||||
|
while preserving independent inspection details and caller-owned data.
|
||||||
|
- Use lightweight captured profile identities and prompt metadata for inspection
|
||||||
|
instead of copying execution targets and definitions unnecessarily.
|
||||||
|
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
|
||||||
|
and inspection processing. Configuration semantics remain unchanged; these
|
||||||
|
improvements primarily benefit construction and explicit operator inspection,
|
||||||
|
not upstream inference latency.
|
||||||
|
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
|
||||||
|
publication. The [release procedure](../release.md) owns the current commands
|
||||||
|
and recovery workflow; successful tag publication remains the completion
|
||||||
|
point, without waiting for CI.
|
||||||
|
|
||||||
|
These changes preserve existing execution and ownership contracts. Benefits
|
||||||
|
depend on workload; this release introduces no fixed performance guarantee.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
|
||||||
|
previous 1.25.5 minimum. Update application build environments and any CI images
|
||||||
|
still using Go 1.25 before upgrading:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
|
||||||
|
```
|
||||||
|
|
||||||
|
No application-code or configuration migration is required from v0.12.0 beyond
|
||||||
|
the Go toolchain update. The new discovery methods are optional additions;
|
||||||
|
existing construction, inspection, preparation, and execution workflows remain
|
||||||
|
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
|
||||||
|
for the distinction between the consumer minimum and maintainer toolchain.
|
||||||
|
|
||||||
|
Maintainers should follow the current release runbook rather than the retired
|
||||||
|
`scripts/release-check.sh`. The publication entry point remains
|
||||||
|
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
|
||||||
|
mechanics.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)
|
||||||
48
docs/releases/v0.14.0.md
Normal file
48
docs/releases/v0.14.0.md
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# Promptkit v0.14.0
|
||||||
|
|
||||||
|
This release adds comparison of captured configuration inspections and updates
|
||||||
|
ConfigKit and the maintainer release tool. The minimum supported Go version
|
||||||
|
remains 1.26.0.
|
||||||
|
|
||||||
|
## Compare Captured Configurations
|
||||||
|
|
||||||
|
`ConfigurationInspection.Diff` compares the effective configuration captured by
|
||||||
|
two engines. Consumers can use it to review a candidate configuration, inspect
|
||||||
|
an overlay's effect, or present configuration changes to operators.
|
||||||
|
|
||||||
|
Comparison delegates to ConfigKit and returns its difference records, including
|
||||||
|
presence information, paired redaction, and protected-change indicators. It
|
||||||
|
does not reload sources or contact providers. Source-history differences alone
|
||||||
|
do not produce value differences.
|
||||||
|
|
||||||
|
An empty diff is not proof of equivalent execution: referenced file contents,
|
||||||
|
resolved credentials, later execution preparation, and request-specific
|
||||||
|
overrides are outside the comparison. Records can contain ordinary application
|
||||||
|
data; callers retain responsibility for their disclosure.
|
||||||
|
|
||||||
|
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
|
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
|
||||||
|
|
||||||
|
## Dependency And Tooling Updates
|
||||||
|
|
||||||
|
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
|
||||||
|
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
|
||||||
|
behavior from v0.7.0.
|
||||||
|
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
|
||||||
|
retains its existing root-module profile, publication entry point, and release
|
||||||
|
gates. See the [release procedure](../release.md) for maintainer commands.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No application-code, configuration, or Go-toolchain migration is required from
|
||||||
|
v0.13.0. Configuration comparison is an optional API addition; existing
|
||||||
|
construction, inspection, preparation, and execution workflows remain available.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
|
||||||
|
```
|
||||||
|
|
||||||
|
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
|
||||||
|
distinction between the consumer minimum and maintainer toolchain.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)
|
||||||
34
docs/releases/v0.15.0.md
Normal file
34
docs/releases/v0.15.0.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Promptkit v0.15.0
|
||||||
|
|
||||||
|
This release adds error-only validation of a concrete prompt request without
|
||||||
|
calling a model. The minimum supported Go version remains 1.26.0.
|
||||||
|
|
||||||
|
## Validate A Concrete Request
|
||||||
|
|
||||||
|
`Engine.ValidateRequest` accepts an ordinary `RunRequest` and returns nil when
|
||||||
|
it passes preparation, or the existing preparation error. Consumers can use it
|
||||||
|
for fail-fast checks, including downstream CLI validation commands, without
|
||||||
|
receiving a prepared snapshot.
|
||||||
|
|
||||||
|
The method shares `Prepare`'s request conversion and preparation path. Unlike
|
||||||
|
requirements inspection, it checks the actual supplied values and bindings.
|
||||||
|
Artifact and schema resource I/O still occurs; success does not certify provider
|
||||||
|
acceptance or later execution success. A subsequent run prepares independently.
|
||||||
|
Validation is optional, not a required step before preparation or execution.
|
||||||
|
|
||||||
|
See the [request validation guide](../consumers/pkg-promptkit.md#validate-a-concrete-request)
|
||||||
|
and [GoDoc](../../engine.go) for usage and the exact contract. Consumers that
|
||||||
|
need execution tied to a checked snapshot can use the existing
|
||||||
|
[prepared execution workflow](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No application-code, configuration, or Go-toolchain migration is required from
|
||||||
|
v0.14.0. This is an optional API addition; existing inspection, preparation, and
|
||||||
|
execution workflows remain available.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.15.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/docs/oss/attribution.md)
|
||||||
86
docs/releases/v0.16.0.md
Normal file
86
docs/releases/v0.16.0.md
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
# Promptkit v0.16.0
|
||||||
|
|
||||||
|
This release adds explicit smoke diagnostics, configuration collections,
|
||||||
|
namespace-scoped engine views, and expanded resource discovery. It also makes
|
||||||
|
intentional breaking changes to configuration ownership and prompt selection.
|
||||||
|
The minimum supported Go version remains 1.26.0.
|
||||||
|
|
||||||
|
## Configuration Collections And Discovery
|
||||||
|
|
||||||
|
Applications can compose caller-selected collections of backends, profiles,
|
||||||
|
and prompts. Collection resources have qualified identities such as
|
||||||
|
`support/answer`; root resources retain unqualified identities. `CollectionInput`
|
||||||
|
provides a convenience for mounting a selected ConfigKit input into a collection.
|
||||||
|
Promptkit now uses ConfigKit v0.8.0, including its optional source specification
|
||||||
|
header support.
|
||||||
|
|
||||||
|
`Engine.WithNamespaces` creates an immutable view with separate prompt, profile,
|
||||||
|
and backend scopes for short caller-supplied IDs. Views share the underlying
|
||||||
|
configuration, clients, capacity pools, and prepared-execution owner. Explicitly
|
||||||
|
qualified IDs and configured references keep their exact identities.
|
||||||
|
|
||||||
|
`ListProfiles` and `ListPrompts` support namespace filters. New `ListBackends`,
|
||||||
|
`ListNamespaces`, and `ListCollections` methods expose captured resource and
|
||||||
|
collection inventories. See the [collection format](../formats.md#source-headers-collections-and-resource-ids),
|
||||||
|
[scoped-view guide](../consumers/pkg-promptkit.md#use-scoped-engine-views), and
|
||||||
|
[discovery guide](../consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
|
## Explicit Smoke Diagnostics
|
||||||
|
|
||||||
|
`Engine.SmokeTest` provides backend reachability, authentication, text inference,
|
||||||
|
and structured-output inference diagnostics. Reports provide factual
|
||||||
|
observations, timing, and available usage rather than an overall readiness
|
||||||
|
verdict. Optional Boolean facts distinguish negative observations from unknown
|
||||||
|
or inapplicable checks; matching a JSON schema does not prove that a provider
|
||||||
|
enforced it.
|
||||||
|
|
||||||
|
A concluded diagnostic can report failure with a nil Go error. Invalid requests
|
||||||
|
return ordinary errors, and caller cancellation can return partial observations
|
||||||
|
with a recognizable context error. Diagnostics can contact providers and incur
|
||||||
|
inference costs only when explicitly invoked; they are not automatic execution
|
||||||
|
or release checks. See the [smoke diagnostic guide](../consumers/pkg-promptkit.md#run-explicit-smoke-diagnostics)
|
||||||
|
and [API GoDoc](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
|
## Breaking Changes And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.15.0 should follow the
|
||||||
|
[configuration and API migration guide](../consumers/configuration-migration.md):
|
||||||
|
|
||||||
|
- Supply backend and profile definitions explicitly. Promptkit no longer bundles
|
||||||
|
provider catalogs or the `promptkit-openrouter-standard` and
|
||||||
|
`promptkit-openrouter-auto` profiles. Conventional backend names do not install
|
||||||
|
definitions automatically. Applications choose and maintain their sources.
|
||||||
|
- Convert version-keyed prompt mappings into one definition per prompt ID.
|
||||||
|
Prompt `version` is now an optional descriptive label, not a selector.
|
||||||
|
Choose deliberately which former definition to install.
|
||||||
|
- Remove `PromptVersion` selectors from `RunRequest`,
|
||||||
|
`RequirementInspectionRequest`, and `SmokeTestRequest`, where applicable.
|
||||||
|
Change prompt inspection calls to `InspectPrompt(ctx, promptID)`. Update code
|
||||||
|
consuming version-bearing inspection, identity, result, or observation fields
|
||||||
|
to handle optional labels, including nil and explicitly empty labels.
|
||||||
|
- When moving resources into collections, update stored backend, profile, and
|
||||||
|
diagnostic-default references to their intended qualified IDs. Mounting a
|
||||||
|
source does not rewrite references; profile inheritance remains within its
|
||||||
|
sibling mapping. Root configuration `version`, when supplied, is now a source
|
||||||
|
specification header, distinct from descriptive resource labels.
|
||||||
|
|
||||||
|
The migration guide includes identity mappings and before/after prompt examples.
|
||||||
|
The [maintained offline example](../../examples/go-library/prepare/main.go)
|
||||||
|
demonstrates explicit composition, source-relative content, scoped preparation,
|
||||||
|
and inventory without contacting a provider.
|
||||||
|
|
||||||
|
After adapting configuration and API calls:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.16.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Correctness And Efficiency
|
||||||
|
|
||||||
|
Collection prompt materialization shares one lazy origin-metadata snapshot,
|
||||||
|
avoiding repeated full metadata copies across collections. Internal prepared
|
||||||
|
snapshots detach optional prompt labels. Smoke diagnostics preserve caller
|
||||||
|
cancellation and report actual inference destinations and known-zero usage when
|
||||||
|
no generation request was dispatched.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.16.0/docs/oss/attribution.md)
|
||||||
68
docs/releases/v0.17.0.md
Normal file
68
docs/releases/v0.17.0.md
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
# Promptkit v0.17.0
|
||||||
|
|
||||||
|
This release adopts the vanity module path `go.maximumdirect.net/promptkit`
|
||||||
|
and ConfigKit v1.0.0, standardizes configuration resources as identified lists,
|
||||||
|
and removes descriptive prompt-version metadata. These are intentional breaking
|
||||||
|
changes before v1. The minimum supported Go version increases from 1.26.0 to
|
||||||
|
1.27.0 for the ConfigKit v1 migration.
|
||||||
|
|
||||||
|
## Breaking Changes And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.16.0 should follow the
|
||||||
|
[configuration and API migration guide](../consumers/configuration-migration.md):
|
||||||
|
|
||||||
|
- Change Promptkit imports from `gitea.maximumdirect.net/eric/promptkit` to
|
||||||
|
`go.maximumdirect.net/promptkit`. Update packages that exchange Promptkit
|
||||||
|
types together; old-path and vanity-path types are distinct.
|
||||||
|
- Change ConfigKit imports to `go.maximumdirect.net/configkit` at v1.0.0 and
|
||||||
|
adopt its renamed public inspection types and fields where used.
|
||||||
|
- Convert root and collection `prompts`, `profiles`, and `backends` from keyed
|
||||||
|
mappings to lists whose records contain an explicit `id`. Collections remain
|
||||||
|
keyed by namespace. Sibling inheritance now uses `extends: {id: parent}`;
|
||||||
|
later same-ID records replace lower records in full.
|
||||||
|
- Change `ProfileSource` and `BackendSource` calls to pass slices of
|
||||||
|
`ProfileConfig` and `BackendConfig`, setting each record's `ID`.
|
||||||
|
- Change source-root `configkit:` controls to `config:`. Imports are optional
|
||||||
|
unless marked `required: true`. Optional Promptkit specification headers now
|
||||||
|
use `version: 2` or `version: "2"`; headerless sources remain valid.
|
||||||
|
- Remove descriptive prompt `version` values and code consuming `PromptVersion`,
|
||||||
|
`PromptIdentity.Version`, or corresponding `prompt_version` output members.
|
||||||
|
Collection `release_version` labels remain available. Prompt hashes may change
|
||||||
|
across this migration.
|
||||||
|
|
||||||
|
After adapting Go imports, configuration, and API calls, use Go 1.27.0 or newer:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.17.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Earlier tags retain their original module declarations. No permanent local
|
||||||
|
replacement is needed for this cutover. See the
|
||||||
|
[framework format reference](../formats.md) for current configuration semantics
|
||||||
|
and the [maintained preparation example](../../examples/go-library/prepare/main.go)
|
||||||
|
for explicit composition, source-relative imports, and scoped discovery.
|
||||||
|
|
||||||
|
## Configuration Inspection And Execution
|
||||||
|
|
||||||
|
Configuration inspection and diffs expose ConfigKit's composed values and
|
||||||
|
provenance, including list-index paths. Promptkit applies domain normalization
|
||||||
|
and execution defaults to owned interpretations without replacing the source
|
||||||
|
origins needed for relative resources. Profile inspection and preparation expose
|
||||||
|
their interpreted or effective settings. See the
|
||||||
|
[migration guide](../consumers/configuration-migration.md#select-and-compose-the-configuration-you-need)
|
||||||
|
for the distinction and inspection-path updates.
|
||||||
|
|
||||||
|
## Maintainer Tooling
|
||||||
|
|
||||||
|
Release tooling now uses the module-selected ReleaseKit v0.8.0 through `go tool`,
|
||||||
|
with page-only Gitea delivery and saved-publication-attempt recovery. The
|
||||||
|
[release procedure](../release.md) describes publication and recovery.
|
||||||
|
|
||||||
|
The three maintained offline examples now have ordinary Go workflow tests,
|
||||||
|
included in the standard suite. Their separate Python runner and custom release
|
||||||
|
hook have been removed; Python is no longer a normal release prerequisite.
|
||||||
|
The optional release-wiring diagnostic remains Python. See the
|
||||||
|
[development guide](../development.md#maintainer-validation).
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.17.0/docs/oss/attribution.md)
|
||||||
65
docs/releases/v0.18.0.md
Normal file
65
docs/releases/v0.18.0.md
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
# Promptkit v0.18.0
|
||||||
|
|
||||||
|
Configuration inspection now exposes Promptkit's recorded field normalization
|
||||||
|
and derived queue-capacity defaults through ConfigKit v1.1.0. Engines also
|
||||||
|
include a default-on OpenRouter configuration. The minimum supported Go
|
||||||
|
version remains 1.27.0.
|
||||||
|
|
||||||
|
## Bundled OpenRouter Backend And Profiles
|
||||||
|
|
||||||
|
New engines include an embedded OpenRouter backend and seventeen root profiles:
|
||||||
|
ten latest-family aliases, the auto and free routers, and five auto-router
|
||||||
|
cost-tier variants. Select a profile explicitly in a prompt or request; the
|
||||||
|
bundle does not supply an implicit execution profile or contact providers
|
||||||
|
during construction. The backend uses `OPENROUTER_API_KEY` when available under
|
||||||
|
the existing optional-credential rules.
|
||||||
|
|
||||||
|
The bundle is the lowest-priority underlay. Applications can replace its records
|
||||||
|
or derive named variants through ordinary configuration, and external collections
|
||||||
|
remain supported. Cost-tier variants inherit the effective `auto` profile, so
|
||||||
|
replacing that parent changes their inherited target while retaining plugins.
|
||||||
|
The [bundle reference](../formats.md#bundled-openrouter-configuration) owns the
|
||||||
|
inventory and composition details.
|
||||||
|
|
||||||
|
Explicit smoke diagnostics now have OpenRouter authentication metadata and an
|
||||||
|
inference fallback profile. They remain opt-in and may incur provider costs.
|
||||||
|
Latest aliases and routers can change upstream; configured slugs do not pin
|
||||||
|
the concrete model, pricing, or availability. See the
|
||||||
|
[consumer guide](../consumers/pkg-promptkit.md#construct-an-engine) for selection
|
||||||
|
and opt-out examples.
|
||||||
|
|
||||||
|
## Recorded Configuration Normalization
|
||||||
|
|
||||||
|
Inspection shows normalized backend and profile settings, prompt metadata, and
|
||||||
|
configured message roles and cache controls, with field-level resolution
|
||||||
|
history. Existing source evidence remains available, and unchanged content-file
|
||||||
|
origins continue to select their resource context.
|
||||||
|
|
||||||
|
Configuration diffs compare normalized effective values, so spelling-only
|
||||||
|
changes that normalize identically may no longer appear. Lists remain atomic in
|
||||||
|
diffs. Inspection does not include backend-to-profile completion or request
|
||||||
|
overrides; see the [configuration inspection guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
New engines install a bundled OpenRouter backend and root profiles by default.
|
||||||
|
Inventory listings and inspection sources therefore change, and references to
|
||||||
|
bundled IDs that previously failed may now resolve. Pass
|
||||||
|
`promptkit.WithBundledOpenRouter(false)` to retain application-only behavior.
|
||||||
|
Caller records can replace bundled definitions through ordinary composition;
|
||||||
|
same-ID replacements replace complete records rather than merging selected fields.
|
||||||
|
No configuration-format migration is required from v0.17.0.
|
||||||
|
Consumers that display or compare configuration inspection results should account
|
||||||
|
for normalized values and derived defaults replacing their earlier authored
|
||||||
|
representations. Request-time overrides and caller-appended messages retain
|
||||||
|
their existing processing.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.18.0
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [framework format reference](../formats.md) for current configuration
|
||||||
|
semantics. Consumers upgrading from versions before v0.17.0 should also follow
|
||||||
|
the [configuration and API migration guide](../consumers/configuration-migration.md).
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/docs/oss/attribution.md)
|
||||||
80
docs/releases/v0.19.0.md
Normal file
80
docs/releases/v0.19.0.md
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
# Promptkit v0.19.0
|
||||||
|
|
||||||
|
This release completes two rounds of audit remediation, correcting error and
|
||||||
|
diagnostic reporting and simplifying configuration, validation, and value
|
||||||
|
ownership. It includes intentional compatibility changes for custom artifact
|
||||||
|
readers, programmatic configuration sources, and schemas. The minimum supported
|
||||||
|
Go version remains 1.27.0; ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.18.0 should review these changes:
|
||||||
|
|
||||||
|
- A successful `ArtifactReader.Read` transfers ownership of the returned
|
||||||
|
artifact and its mutable body storage to Promptkit. Custom readers must stop
|
||||||
|
mutating returned values through retained aliases. Readers that reuse scratch
|
||||||
|
buffers must return independent storage. Promptkit retains its defensive
|
||||||
|
body copy, but that copy does not make concurrent buffer reuse safe. See the
|
||||||
|
[extension guidance](../consumers/pkg-promptkit.md#extension-interfaces).
|
||||||
|
- Provider maps passed through `ProfileSource` and `BackendSource` now use
|
||||||
|
ConfigKit's native Mapping input forms directly: built-in scalars, `[]any`,
|
||||||
|
and `map[string]any` recursively. Convert `json.Number` and typed containers
|
||||||
|
explicitly to supported values, choosing numeric precision appropriate to
|
||||||
|
the application. The former recursive compatibility conversion is removed.
|
||||||
|
Runtime execution overrides retain their separate accepted-value behavior.
|
||||||
|
- Schemas used for local validation and native structured output must be
|
||||||
|
self-contained. Move externally referenced definitions into the root document
|
||||||
|
and use internal references or explicit schema IDs. References requiring
|
||||||
|
another file or network resource, or relying on the source filename as a base,
|
||||||
|
fail during preparation. Promptkit does not bundle external schemas. See the
|
||||||
|
[schema reference](../formats.md#schemas).
|
||||||
|
|
||||||
|
Prepared schemas no longer inherit the depth and node limits used when copying
|
||||||
|
runtime provider values. Schemas previously rejected solely by those incidental
|
||||||
|
limits may now be accepted; decoder/compiler checks and the self-contained
|
||||||
|
schema requirement still apply.
|
||||||
|
|
||||||
|
No configuration-format migration is required from v0.18.0. After addressing
|
||||||
|
any affected integrations:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.19.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Errors And Diagnostics
|
||||||
|
|
||||||
|
Public errors and terminal observations now preserve the category of the
|
||||||
|
operation that failed even when an injected collaborator returns an error from
|
||||||
|
another Promptkit operation. Underlying causes remain available through
|
||||||
|
`errors.Is` and `errors.As`, without repeatedly duplicating nested cause trees.
|
||||||
|
|
||||||
|
Built-in generation preserves response-body read failures separately from
|
||||||
|
malformed response content, and failures during request preparation are
|
||||||
|
distinguished from dispatched attempts.
|
||||||
|
|
||||||
|
OpenRouter authentication diagnostics bound response recognition to 1 MiB.
|
||||||
|
An oversized HTTP 200 response reports
|
||||||
|
`SmokeTestReasonAuthenticationResponseLimitExceeded`, leaving credential
|
||||||
|
acceptance unknown. Cancellation during the read remains an interruption.
|
||||||
|
Consumers that enumerate diagnostic reasons should handle the new reason;
|
||||||
|
see the [diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||||
|
|
||||||
|
## Internal Simplification And Ownership
|
||||||
|
|
||||||
|
Prompt lookup and rendering now consistently use captured configuration, and
|
||||||
|
smoke syntax and schema checks share the prepared output-validation lifecycle.
|
||||||
|
Unused routing defaults, alternate private workflows, and duplicate wire
|
||||||
|
representations have been removed.
|
||||||
|
|
||||||
|
Provider-setting resolution, inspection, prepared details, and smoke reports
|
||||||
|
avoid redundant validation and copying of already accepted values. Prepared
|
||||||
|
execution shares one private redacted metadata snapshot while preserving
|
||||||
|
independent caller-visible values. Provider values are detached before custom
|
||||||
|
JSON encoders run, protecting retained metadata from encoder-side mutation.
|
||||||
|
|
||||||
|
Requirement analysis, namespace-filtered listings, and repair-feedback
|
||||||
|
construction also avoid unnecessary traversal and temporary allocation. These
|
||||||
|
changes preserve existing request isolation and execution behavior without
|
||||||
|
introducing new performance guarantees.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/docs/oss/attribution.md)
|
||||||
55
docs/releases/v0.20.0.md
Normal file
55
docs/releases/v0.20.0.md
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Promptkit v0.20.0
|
||||||
|
|
||||||
|
This minor release adds generically renderable resource inspection and typed
|
||||||
|
prompt source construction. The Go consumer minimum remains 1.27.0 and
|
||||||
|
ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Migration
|
||||||
|
|
||||||
|
`InspectProfile` now returns the resolved profile resource instead of the
|
||||||
|
backend-completed `EffectiveModelParams`. Replace accesses to that field with
|
||||||
|
the optional profile fields; use `Prepare` or `PreparedExecution.Details` for
|
||||||
|
completed request settings. `InspectPrompt` now includes captured messages and
|
||||||
|
uses optional pointers for presence-sensitive fields, including default profile,
|
||||||
|
inputs, input metadata, and output settings. Check pointers before dereferencing.
|
||||||
|
All three resource reports support ordinary JSON and YAML rendering without
|
||||||
|
per-field adapters. See the [consumer guidance](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
|
After configuration composition and inheritance, profile null endpoint,
|
||||||
|
API-key source, and whole provider map now permit backend defaults during
|
||||||
|
execution as well as appearing absent in resource inspection. Use explicit
|
||||||
|
empty strings to clear endpoint or API-key source. There is no replacement for
|
||||||
|
profile-level suppression of an entire backend provider map; edit backend
|
||||||
|
defaults through composition instead. The required-key rule still applies.
|
||||||
|
See [defaults and overrides](../formats.md#defaults-and-overrides) for the
|
||||||
|
migration example and retained ConfigKit null provenance.
|
||||||
|
|
||||||
|
## Resource Inspection And Sources
|
||||||
|
|
||||||
|
`InspectBackend` reports the resolved backend and configured diagnostics without
|
||||||
|
contacting a provider or deriving diagnostic destinations. Prompt and profile
|
||||||
|
inspection likewise report selected resources independently of request readiness;
|
||||||
|
configuration provenance, requirement analysis, and preparation remain separate.
|
||||||
|
`PromptSource` adds typed programmatic prompt definitions, and `BackendConfig`
|
||||||
|
supports sibling inheritance. ConfigKit continues to own source composition,
|
||||||
|
snapshotting, and relative message-file contexts.
|
||||||
|
|
||||||
|
The source helpers' [GoDoc](../../prompts.go) and
|
||||||
|
[backend configuration API](../../backends.go) describe programmatic construction.
|
||||||
|
|
||||||
|
## Refinements
|
||||||
|
|
||||||
|
Package GoDoc and migration guidance now consistently describe the resource
|
||||||
|
inspection contract, and the full package overview is visible in generated
|
||||||
|
documentation. Prompt inspection avoids repeating the same metadata lookup for
|
||||||
|
each declared input.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
After adapting affected inspection consumers and reviewing profile null usage:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.20.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/docs/oss/attribution.md)
|
||||||
@@ -63,8 +63,8 @@ Every engine now includes the reserved `rakestrawhome` backend, identified by
|
|||||||
selects that backend. Consumers can use the maintained endpoint, credential,
|
selects that backend. Consumers can use the maintained endpoint, credential,
|
||||||
capacity, and model defaults without registering either definition themselves.
|
capacity, and model defaults without registering either definition themselves.
|
||||||
|
|
||||||
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
|
See the [backend format reference](../formats.md#backend-definitions)
|
||||||
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
|
and the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping)
|
||||||
for the current contracts.
|
for the current contracts.
|
||||||
|
|
||||||
## Structured Generation Errors
|
## Structured Generation Errors
|
||||||
@@ -126,8 +126,8 @@ cycle-safe, and resolved afresh for ordinary operations. Prepared execution
|
|||||||
freezes the fully resolved target. The selected leaf ID remains public while
|
freezes the fully resolved target. The selected leaf ID remains public while
|
||||||
effective execution settings reflect the resolved chain.
|
effective execution settings reflect the resolved chain.
|
||||||
|
|
||||||
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
See the [profile inheritance format reference](../formats.md#resource-inheritance),
|
||||||
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
|
the [current identity migration mapping](../consumers/configuration-migration.md#identity-mapping),
|
||||||
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
||||||
behavior.
|
behavior.
|
||||||
|
|
||||||
|
|||||||
@@ -96,9 +96,8 @@ precedence rules. Promptkit no longer keeps duplicate embedded profile assets
|
|||||||
or hard-coded definitions for those maintained backends.
|
or hard-coded definitions for those maintained backends.
|
||||||
|
|
||||||
The module versions in Promptkit's `go.mod` identify the catalog releases
|
The module versions in Promptkit's `go.mod` identify the catalog releases
|
||||||
tested with this release. The [built-in backend and profile format
|
tested with this release. The [backend format reference](../formats.md#backend-definitions)
|
||||||
reference](../formats.md#built-in-backends) remains the canonical consumer
|
remains the canonical consumer contract, while the [internal source documentation](../internal/sources.md#profiles)
|
||||||
contract, while the [internal source documentation](../internal/sources.md#profiles-and-built-ins)
|
|
||||||
describes the dependency boundary.
|
describes the dependency boundary.
|
||||||
|
|
||||||
## Public API Changes
|
## Public API Changes
|
||||||
|
|||||||
@@ -38,8 +38,17 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
No ideas are currently awaiting selection. Active feature work belongs in its
|
### Schema Bundling
|
||||||
focused roadmap rather than this catalog.
|
|
||||||
|
Allow consumers to author reusable schemas across multiple files and package
|
||||||
|
them into a single self-contained schema for local validation and provider
|
||||||
|
submission. This could restore multi-file authoring convenience while retaining
|
||||||
|
the [self-contained schema requirement](../formats.md#schemas).
|
||||||
|
|
||||||
|
Bundling would need to preserve reference semantics, identifiers, anchors, and
|
||||||
|
recursion. A complete bundle would not guarantee that a provider supports every
|
||||||
|
schema feature. This is a possible future enhancement; implementation is not
|
||||||
|
currently approved.
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
181
engine.go
181
engine.go
@@ -8,11 +8,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
|
internalengine "go.maximumdirect.net/promptkit/internal/engine"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||||
@@ -24,13 +25,14 @@ var (
|
|||||||
// ErrInvalidRequest identifies a request whose required values, overrides,
|
// ErrInvalidRequest identifies a request whose required values, overrides,
|
||||||
// credentials, or effective settings are invalid.
|
// credentials, or effective settings are invalid.
|
||||||
ErrInvalidRequest = errors.New("invalid run request")
|
ErrInvalidRequest = errors.New("invalid run request")
|
||||||
// ErrPromptNotFound identifies a requested prompt ID or version that is not
|
// ErrPromptNotFound identifies a requested prompt ID that is not present in
|
||||||
// present in the selected prompt source. It does not also match
|
// the captured configuration. It does not also match ErrPromptLoad.
|
||||||
// ErrPromptLoad.
|
|
||||||
ErrPromptNotFound = errors.New("prompt not found")
|
ErrPromptNotFound = errors.New("prompt not found")
|
||||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||||
// every configured profile source. It does not also match ErrProfileLoad.
|
// every configured profile source. It does not also match ErrProfileLoad.
|
||||||
ErrProfileNotFound = errors.New("profile not found")
|
ErrProfileNotFound = errors.New("profile not found")
|
||||||
|
// ErrBackendNotFound identifies a backend absent from resource inspection.
|
||||||
|
ErrBackendNotFound = errors.New("backend not found")
|
||||||
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||||
// nor the selected prompt's default profile is present. Such an error also
|
// nor the selected prompt's default profile is present. Such an error also
|
||||||
// matches ErrInvalidRequest.
|
// matches ErrInvalidRequest.
|
||||||
@@ -82,23 +84,29 @@ var (
|
|||||||
ErrValidation = errors.New("failed to validate output")
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
|
// Engine inspects resources, enumerates prompt requirements, and
|
||||||
// requests.
|
// prepares and runs Promptkit prompt requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
|
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
||||||
// [Engine.InspectPrompt],
|
// [Engine.InspectConfiguration],
|
||||||
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.InspectPrompt], [Engine.InspectBackend],
|
||||||
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// backend-capacity pools that coordinate Run and RunPrepared admission and
|
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
||||||
// model generation. Injected collaborators may still be invoked concurrently
|
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
||||||
// across different backend pools or for unlimited backends.
|
// [Engine.ListCollections]. Engines constructed separately by [NewEngine] own
|
||||||
|
// independent backend-capacity pools that coordinate Run, RunPrepared, and
|
||||||
|
// inference SmokeTest admission and model generation. Namespace views returned
|
||||||
|
// by [Engine.WithNamespaces] share their underlying engine's pools, clients,
|
||||||
|
// and prepared-execution owner. Injected collaborators may still be invoked
|
||||||
|
// concurrently across different backend pools or for unlimited backends.
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
internal *internalengine.Engine
|
internal *internalengine.Engine
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
||||||
// highest-priority main input. Each underlay's inputs strictly compose within
|
// highest-priority main input. By default a bundled OpenRouter underlay precedes
|
||||||
// their group; later underlays overlay earlier ones. A nil Main uses
|
// the caller underlays; [WithBundledOpenRouter] can disable it. Inputs within
|
||||||
|
// each underlay strictly compose; later underlays overlay earlier ones. A nil Main uses
|
||||||
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
||||||
// may fail construction if invalid.
|
// may fail construction if invalid.
|
||||||
type ConfigurationSources struct {
|
type ConfigurationSources struct {
|
||||||
@@ -150,6 +158,15 @@ type engineOptions struct {
|
|||||||
internal internalengine.Options
|
internal internalengine.Options
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithBundledOpenRouter enables or disables the built-in OpenRouter underlay.
|
||||||
|
// It is enabled by default; when repeated, the last option wins.
|
||||||
|
func WithBundledOpenRouter(enabled bool) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
options.internal.SetBundledOpenRouter(enabled)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||||
// [Engine.RunPrepared].
|
// [Engine.RunPrepared].
|
||||||
//
|
//
|
||||||
@@ -199,7 +216,8 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
|||||||
//
|
//
|
||||||
// Prompt schema_path values refer to the file's base name. path must name an
|
// Prompt schema_path values refer to the file's base name. path must name an
|
||||||
// existing non-directory file when NewEngine applies the option. This option
|
// existing non-directory file when NewEngine applies the option. This option
|
||||||
// replaces Config.SchemaDir and earlier schema-source options.
|
// replaces Config.SchemaDir and earlier schema-source options. The selected
|
||||||
|
// schema must be self-contained; external resources are not loaded.
|
||||||
func WithSchemaFile(path string) Option {
|
func WithSchemaFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
return options.internal.SetSchemaFile(path)
|
return options.internal.SetSchemaFile(path)
|
||||||
@@ -208,11 +226,13 @@ func WithSchemaFile(path string) Option {
|
|||||||
|
|
||||||
// NewEngine constructs an Engine from configuration and options.
|
// NewEngine constructs an Engine from configuration and options.
|
||||||
//
|
//
|
||||||
// ConfigKit composes maintained catalog inputs, PromptKit-owned definitions,
|
// ConfigKit composes caller-selected underlays in their given order, followed
|
||||||
// caller underlays, and the optional main input in that order. Construction
|
// by the optional main input, after the default-on bundled OpenRouter underlay.
|
||||||
// validates the final effective backend, profile, and prompt definitions and
|
// The bundle adds a root backend and profiles to inventories, but no prompts
|
||||||
// captures referenced prompt content. A missing prompt collection is allowed.
|
// or implicit profile selection. Construction validates the final effective
|
||||||
// Schema documents and request artifacts remain runtime resources.
|
// backend, profile, and prompt definitions and captures referenced prompt
|
||||||
|
// content. A missing prompt collection is allowed. Schema documents and
|
||||||
|
// request artifacts remain runtime resources.
|
||||||
//
|
//
|
||||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||||
// configuration or options, preserving a ConfigKit LoadError when loading
|
// configuration or options, preserving a ConfigKit LoadError when loading
|
||||||
@@ -241,25 +261,25 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
return &Engine{internal: assembled}, nil
|
return &Engine{internal: assembled}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt definition without selecting a
|
// InspectPrompt resolves one prompt ID without selecting a profile or starting
|
||||||
// profile or starting execution work.
|
// execution work.
|
||||||
//
|
//
|
||||||
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
|
// InspectPrompt requires a nonblank promptID and performs an exact,
|
||||||
// promptVersion values unchanged to the engine's ordinary, case-sensitive
|
// case-sensitive lookup in the captured configuration snapshot. A
|
||||||
// prompt selection. An empty version succeeds only when that ID has one
|
// namespace-scoped view qualifies slashless caller IDs before lookup; already
|
||||||
// version; a nonempty version selects one exact ID/version pair from the
|
// qualified IDs pass through.
|
||||||
// captured configuration snapshot.
|
|
||||||
//
|
//
|
||||||
// A successful result proves that the selected definition and any referenced
|
// A successful result proves that the selected definition and any referenced
|
||||||
// message content files were structurally loaded. Inputs are returned in
|
// message content files were structurally loaded. Inputs and captured,
|
||||||
// definition order. DefaultProfileID is declared metadata only and is not
|
// unrendered messages are returned in definition order. DefaultProfileID is
|
||||||
|
// declared metadata only and is not
|
||||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||||
// Schema path when declared but without loading or compiling that schema.
|
// Schema path when declared but without loading or compiling that schema.
|
||||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||||
// the selected captured definition; its spelling, length,
|
// the selected captured definition; its spelling, length,
|
||||||
// encoding, algorithm, and security properties are not contracts.
|
// encoding, algorithm, and security properties are not contracts.
|
||||||
//
|
//
|
||||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
// This method does not return parsed templates, source paths, schemas,
|
||||||
// rendered messages, or execution settings. It does not resolve a profile or
|
// rendered messages, or execution settings. It does not resolve a profile or
|
||||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||||
// contact a provider, or generate model output. The returned PromptInspection
|
// contact a provider, or generate model output. The returned PromptInspection
|
||||||
@@ -267,39 +287,35 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
// or content files; construct another Engine to observe source changes.
|
// or content files; construct another Engine to observe source changes.
|
||||||
//
|
//
|
||||||
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
|
||||||
// matches ErrInvalidRequest. An absent exact ID or version matches
|
// matches ErrInvalidRequest. An absent exact ID matches ErrPromptNotFound and
|
||||||
// ErrPromptNotFound and not ErrPromptLoad. An ambiguous omitted version matches
|
// not ErrPromptLoad. Invalid definitions and resource failures reject
|
||||||
// ErrPromptLoad. Invalid definitions and resource failures reject construction
|
// construction instead. Cancellation during lookup preserves the context
|
||||||
// instead. Cancellation during lookup preserves the context error. InspectPrompt
|
// error. InspectPrompt returns no partial result on error.
|
||||||
// returns no partial result on error.
|
|
||||||
func (e *Engine) InspectPrompt(
|
func (e *Engine) InspectPrompt(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
promptID string,
|
promptID string,
|
||||||
promptVersion string,
|
|
||||||
) (*PromptInspection, error) {
|
) (*PromptInspection, error) {
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
}
|
}
|
||||||
|
|
||||||
inspection, err := e.internal.InspectPrompt(ctx, promptID, promptVersion)
|
inspection, err := e.internal.InspectPrompt(ctx, promptID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainPromptInspection(inspection), nil
|
return fromPromptResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||||
// starting execution work.
|
// starting execution work.
|
||||||
//
|
//
|
||||||
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
||||||
// in the captured effective configuration. It applies the framework timeout
|
// in the captured effective configuration. It reports the resolved profile
|
||||||
// baseline, selected
|
// resource without backend completion, execution defaults, or request overrides.
|
||||||
// backend, and then selected profile to EffectiveModelParams without a request
|
|
||||||
// override. BackendID is empty for an endpoint-only profile.
|
|
||||||
//
|
//
|
||||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
// APIKeyEnv in the returned resource is an environment-variable name, never its
|
||||||
// value. APIKeyRequired instead reports a direct credential requirement and is
|
// value. APIKeyRequired, when present and true, reports that a credential is
|
||||||
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
// required; an explicit profile environment source can also be present. InspectProfile neither derives
|
||||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
// absent or blank named environment variable is not an error.
|
// absent or blank named environment variable is not an error.
|
||||||
//
|
//
|
||||||
@@ -323,7 +339,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainProfileInspection(inspection), nil
|
return fromProfileResource(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectBackend selects a resolved backend by exact ID from the captured
|
||||||
|
// configuration, qualifying a slashless ID with the backend namespace. It
|
||||||
|
// reports configured resource fields and diagnostics, not credential values,
|
||||||
|
// derived diagnostic destinations, or runtime capacity occupancy. References
|
||||||
|
// are not resolved and no network request is made. The result and its mutable
|
||||||
|
// values are caller-owned. Source changes require a new engine.
|
||||||
|
//
|
||||||
|
// An invalid engine matches ErrInvalidConfig, an invalid ID matches
|
||||||
|
// ErrInvalidRequest, and an absent backend matches ErrBackendNotFound (not
|
||||||
|
// ErrInvalidRequest). Cancellation remains detectable via errors.Is. No
|
||||||
|
// partial result is returned on error.
|
||||||
|
func (e *Engine) InspectBackend(ctx context.Context, backendID string) (*BackendInspection, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
inspection, err := e.internal.InspectBackend(ctx, backendID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrBackendNotFound, err)
|
||||||
|
}
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromBackendResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
@@ -346,6 +387,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||||
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
||||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||||
|
prepared, err := e.prepareRequest(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return fromDomainPreparedRun(prepared), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateRequest reports whether req passes the same preparation path used by
|
||||||
|
// [Engine.Prepare] at the time of this call. A nil error means this concrete
|
||||||
|
// request passed preparation. It checks only this invocation and returns errors
|
||||||
|
// as encountered rather than aggregating them. All supplied inputs follow the
|
||||||
|
// same loading behavior as [Engine.Prepare], including inputs unused by the
|
||||||
|
// selected template branch. The method does not call an LLM or retain a
|
||||||
|
// prepared snapshot or executable handle, but preparation may still perform
|
||||||
|
// resource I/O and incur its associated time and memory costs.
|
||||||
|
//
|
||||||
|
// Its errors and context handling follow [Engine.Prepare], including an error
|
||||||
|
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
|
||||||
|
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
|
||||||
|
// when execution must use the snapshot produced by preparation.
|
||||||
|
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
|
||||||
|
_, err := e.prepareRequest(ctx, req)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.PreparedRun, error) {
|
||||||
ctx = observation.WithoutExecutionState(ctx)
|
ctx = observation.WithoutExecutionState(ctx)
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
@@ -360,15 +427,16 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainPreparedRun(prepared), nil
|
return prepared, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrepareExecution completely prepares a prompt request without calling the
|
// PrepareExecution completely prepares a prompt request without calling the
|
||||||
// configured LLMClient or reserving backend admission capacity. Validated
|
// configured LLMClient or reserving backend admission capacity. Validated
|
||||||
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
// RunRequest.AppendedMessages are included in the frozen effective messages.
|
||||||
//
|
//
|
||||||
// The returned opaque handle is bound to this Engine and permits one
|
// The returned opaque handle is bound to this Engine's underlying execution
|
||||||
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
// owner, shared with its namespace views, and permits one [Engine.RunPrepared]
|
||||||
|
// invocation in total. Preparation freezes the selected sources,
|
||||||
// rendered messages, effective settings, inputs, provider structured-output
|
// rendered messages, effective settings, inputs, provider structured-output
|
||||||
// metadata, and validation resources needed by that invocation. The handle
|
// metadata, and validation resources needed by that invocation. The handle
|
||||||
// retains a direct RunRequest.APIKey only in private execution state;
|
// retains a direct RunRequest.APIKey only in private execution state;
|
||||||
@@ -480,6 +548,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
// RunPrepared atomically claims and executes a handle created by
|
// RunPrepared atomically claims and executes a handle created by
|
||||||
// [Engine.PrepareExecution].
|
// [Engine.PrepareExecution].
|
||||||
//
|
//
|
||||||
|
// The originating Engine and namespace views sharing its underlying owner may
|
||||||
|
// claim the handle. Execution uses its captured selections without applying the
|
||||||
|
// invoking view's namespaces. An Engine constructed separately by [NewEngine]
|
||||||
|
// is a foreign owner.
|
||||||
|
//
|
||||||
// A valid owning-Engine invocation consumes the handle's one attempt before
|
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||||
// credential revalidation, backend admission, generation, or validation.
|
// credential revalidation, backend admission, generation, or validation.
|
||||||
// Cancellation, capacity rejection, generation failure, operational
|
// Cancellation, capacity rejection, generation failure, operational
|
||||||
|
|||||||
46
errors.go
46
errors.go
@@ -5,18 +5,52 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// publicOperationError keeps classification separate from inspectable causes.
|
||||||
|
// Cache wording so formatting nested operations does not repeatedly rebuild it.
|
||||||
|
type publicOperationError struct {
|
||||||
|
owner error
|
||||||
|
cause error
|
||||||
|
message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *publicOperationError) Error() string { return e.message }
|
||||||
|
func (e *publicOperationError) Unwrap() error { return e.cause }
|
||||||
|
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
|
||||||
|
|
||||||
func mapPublicError(err error) error {
|
func mapPublicError(err error) error {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var operation *usecase.OperationError
|
||||||
|
if errors.As(err, &operation) && operation != nil {
|
||||||
|
owner := publicErrorFor(operation.Category)
|
||||||
|
if owner != nil {
|
||||||
|
// Project the underlying specialized error separately, retaining both
|
||||||
|
// the outer operation and the original cause for Is/As callers.
|
||||||
|
mapped := mapPublicError(operation.Cause)
|
||||||
|
if errors.Is(operation.Cause, llm.ErrTransport) {
|
||||||
|
mapped = errors.Join(ErrTransport, mapped)
|
||||||
|
}
|
||||||
|
if errors.Is(operation.Cause, llm.ErrMalformedResponse) {
|
||||||
|
mapped = errors.Join(ErrMalformedResponse, mapped)
|
||||||
|
}
|
||||||
|
// The projection already retains ordinary causes; adding the original
|
||||||
|
// again would duplicate its entire subtree at each enclosing operation.
|
||||||
|
return &publicOperationError{owner: owner, cause: mapped, message: fmt.Sprintf("%v: %v", owner, mapped)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mapUnownedPublicError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapUnownedPublicError(err error) error {
|
||||||
var internalCapacityError *usecase.CapacityError
|
var internalCapacityError *usecase.CapacityError
|
||||||
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
|
||||||
@@ -80,3 +80,12 @@ func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
|||||||
t.Fatalf("request encoding category = %v", err)
|
t.Fatalf("request encoding category = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorRetainsOwnerWithSpecializedCapacity(t *testing.T) {
|
||||||
|
internal := &usecase.CapacityError{BackendID: "limited"}
|
||||||
|
err := mapPublicError(&usecase.OperationError{Category: usecase.ErrLLMGenerate, Cause: internal})
|
||||||
|
var capacity *CapacityError
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) || !errors.As(err, &capacity) || capacity.BackendID != "limited" || !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("mapped owned capacity = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed prompts/*.yaml schemas/*.json
|
//go:embed prompts/*.yaml schemas/*.json
|
||||||
@@ -84,19 +84,21 @@ func run() (summary, error) {
|
|||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
|
||||||
|
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||||
|
backend.ID = backendID
|
||||||
inputs = append(inputs,
|
inputs = append(inputs,
|
||||||
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
|
configkit.Input{Source: promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||||
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
|
backend,
|
||||||
})},
|
})},
|
||||||
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
|
configkit.Input{Source: promptkit.ProfileSource("example profiles", []promptkit.ProfileConfig{
|
||||||
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
|
{ID: "proposal-profile", Backend: &backendID, Model: &proposalModel},
|
||||||
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
|
{ID: "assessment-profile", Backend: &backendID, Model: &assessmentModel},
|
||||||
})},
|
})},
|
||||||
)
|
)
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
||||||
HTTPClient: &http.Client{Transport: transport},
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
}, promptkit.WithSchemaFS(assets, "schemas"))
|
}, promptkit.WithBundledOpenRouter(false), promptkit.WithSchemaFS(assets, "schemas")) // Keep the example's configuration self-contained.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
@@ -156,13 +158,16 @@ func run() (summary, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
if err := execute(os.Stdout); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func execute(out io.Writer) error {
|
||||||
value, err := run()
|
value, err := run()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, err)
|
return err
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if err := json.NewEncoder(os.Stdout).Encode(value); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
return json.NewEncoder(out).Encode(value)
|
||||||
}
|
}
|
||||||
|
|||||||
30
examples/go-library/observe/main_test.go
Normal file
30
examples/go-library/observe/main_test.go
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestObserveExample(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := execute(&output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got struct {
|
||||||
|
ProposalAttempts *int `json:"proposal_attempts"`
|
||||||
|
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||||
|
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||||
|
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||||
|
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||||
|
Outcome string `json:"assessment_outcome"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.ProposalAttempts == nil || *got.ProposalAttempts != 2 || got.ProposalRetries == nil || *got.ProposalRetries != 1 ||
|
||||||
|
got.UsageComplete == nil || *got.UsageComplete || got.AssessmentAttempts == nil || *got.AssessmentAttempts != 1 ||
|
||||||
|
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" {
|
||||||
|
t.Fatalf("unexpected observation summary: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.assessment:
|
- id: example.assessment
|
||||||
"1.0.0":
|
|
||||||
default_profile: assessment-profile
|
default_profile: assessment-profile
|
||||||
description: Assess a structured proposal independently of schema validity.
|
description: Assess a structured proposal independently of schema validity.
|
||||||
inputs:
|
inputs:
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.proposal:
|
- id: example.proposal
|
||||||
"1.0.0":
|
|
||||||
default_profile: proposal-profile
|
default_profile: proposal-profile
|
||||||
description: Propose a bounded improvement from a synthetic idea.
|
description: Propose a bounded improvement from a synthetic idea.
|
||||||
inputs:
|
inputs:
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
backends:
|
||||||
|
- id: offline
|
||||||
|
endpoint: https://review.example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
- id: audit
|
||||||
|
backend: review/offline
|
||||||
|
model: review-model
|
||||||
|
prompts:
|
||||||
|
- id: check
|
||||||
|
default_profile: review/audit
|
||||||
|
description: Inspect a review item without contacting a model provider.
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: Review the supplied synthetic example.
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
version: "2"
|
||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: fragments/resources.yaml
|
||||||
|
required: true
|
||||||
|
- path: fragments/prepare.yaml
|
||||||
|
destination: [prompts]
|
||||||
|
placement: sequence-item
|
||||||
|
required: true
|
||||||
|
release_version: "1.4.0"
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
id: prepare
|
||||||
|
default_profile: support/fast
|
||||||
|
description: Prepare a prompt without contacting a model provider.
|
||||||
|
inputs:
|
||||||
|
- name: note
|
||||||
|
required: true
|
||||||
|
content_type: text/plain
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: Summarize the note in one sentence.
|
||||||
|
- role: user
|
||||||
|
content_file: ../messages/note.md
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: basic
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
backends:
|
||||||
|
- id: offline
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
profiles:
|
||||||
|
- id: base
|
||||||
|
backend: support/offline
|
||||||
|
model: offline-model
|
||||||
|
timeout_seconds: 90
|
||||||
|
- id: fast
|
||||||
|
extends: {id: base}
|
||||||
|
timeout_seconds: 60
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{{input "note"}}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
# A later input replaces the profile with an inherited variant.
|
||||||
|
profiles:
|
||||||
|
- id: fast
|
||||||
|
extends: {id: base}
|
||||||
|
timeout_seconds: 30
|
||||||
@@ -4,63 +4,162 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type summary struct {
|
type summary struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version"`
|
|
||||||
SelectedProfile string `json:"selected_profile"`
|
SelectedProfile string `json:"selected_profile"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
MessageCount int `json:"message_count"`
|
MessageCount int `json:"message_count"`
|
||||||
|
ProfileTimeout int `json:"profile_timeout_seconds"`
|
||||||
|
Namespaces []string `json:"namespaces"`
|
||||||
|
Collections []collectionSummary `json:"collections"`
|
||||||
|
ScopedPrompts []string `json:"scoped_prompts"`
|
||||||
|
ReviewPrompts []string `json:"review_prompts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type collectionSummary struct {
|
||||||
|
Namespace string `json:"namespace"`
|
||||||
|
ReleaseVersion *string `json:"release_version"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
if err := run(context.Background(), "examples/go-library/prepare", os.Stdout); err != nil {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
|
||||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
|
||||||
})}}}},
|
|
||||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/prepare/prompt.yaml")},
|
|
||||||
}})
|
|
||||||
if err != nil {
|
|
||||||
exit(err)
|
|
||||||
}
|
|
||||||
view, err := engine.InspectConfiguration()
|
|
||||||
if err != nil {
|
|
||||||
exit(err)
|
|
||||||
}
|
|
||||||
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
|
|
||||||
exit(fmt.Errorf("configured profile model was not found"))
|
|
||||||
}
|
|
||||||
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
||||||
PromptID: "example.prepare",
|
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
|
||||||
"note": promptkit.Inline("Ada finished the migration review."),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
exit(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
|
||||||
encoder.SetIndent("", " ")
|
|
||||||
if err := encoder.Encode(summary{
|
|
||||||
PromptID: prepared.PromptID,
|
|
||||||
PromptVersion: prepared.PromptVersion,
|
|
||||||
SelectedProfile: prepared.SelectedProfileID,
|
|
||||||
Model: prepared.EffectiveModelParams.Model,
|
|
||||||
MessageCount: len(prepared.Messages),
|
|
||||||
}); err != nil {
|
|
||||||
exit(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func exit(err error) {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||||
|
support, err := promptkit.CollectionInput(
|
||||||
|
"support",
|
||||||
|
configkit.File(filepath.Join(exampleDir, "collections", "support", "collection.yaml")),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
review, err := promptkit.CollectionInput(
|
||||||
|
"review",
|
||||||
|
configkit.File(filepath.Join(exampleDir, "collections", "review", "collection.yaml")),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
override, err := promptkit.CollectionInput(
|
||||||
|
"support",
|
||||||
|
configkit.File(filepath.Join(exampleDir, "collections", "support", "operator-overrides.yaml")),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{
|
||||||
|
{Inputs: []configkit.Input{support, review}},
|
||||||
|
{Inputs: []configkit.Input{override}},
|
||||||
|
},
|
||||||
|
}}, promptkit.WithBundledOpenRouter(false)) // Keep this example's inventory caller-owned.
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
scoped, err := engine.WithNamespaces(promptkit.Namespaces{
|
||||||
|
Prompt: "support", Profile: "support", Backend: "support",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, found := view.Explain(configkit.Keys("collections", "support", "profiles").Index(1).Key("timeout_seconds")); !found {
|
||||||
|
return fmt.Errorf("collection profile override was not found")
|
||||||
|
}
|
||||||
|
requirements, err := scoped.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
|
||||||
|
PromptID: "prepare",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !requirements.Complete || len(requirements.Inputs) != 1 ||
|
||||||
|
requirements.Inputs[0].Name != "note" ||
|
||||||
|
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
|
||||||
|
return fmt.Errorf("required note input was not discovered before preparation")
|
||||||
|
}
|
||||||
|
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "prepare",
|
||||||
|
ProfileID: "fast",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Ada finished the migration review."),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := scoped.ValidateRequest(ctx, request); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
prepared, err := scoped.Prepare(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
profile, err := scoped.InspectProfile(ctx, "fast")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 30 {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"later profile override was not applied: timeout is %v",
|
||||||
|
profile.TimeoutSeconds,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
namespaces, err := scoped.ListNamespaces(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
collections, err := scoped.ListCollections(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
scopedPrompts, err := scoped.ListPrompts(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reviewPrompts, err := scoped.ListPrompts(ctx, promptkit.NamespaceFilter{
|
||||||
|
Mode: promptkit.NamespaceNamed, Name: "review",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
scopedPromptIDs := make([]string, len(scopedPrompts))
|
||||||
|
for index := range scopedPrompts {
|
||||||
|
scopedPromptIDs[index] = scopedPrompts[index].ID
|
||||||
|
}
|
||||||
|
reviewPromptIDs := make([]string, len(reviewPrompts))
|
||||||
|
for index := range reviewPrompts {
|
||||||
|
reviewPromptIDs[index] = reviewPrompts[index].ID
|
||||||
|
}
|
||||||
|
collectionSummaries := make([]collectionSummary, len(collections))
|
||||||
|
for index := range collections {
|
||||||
|
collectionSummaries[index] = collectionSummary{
|
||||||
|
Namespace: collections[index].Namespace, ReleaseVersion: collections[index].ReleaseVersion,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder := json.NewEncoder(out)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
return encoder.Encode(summary{
|
||||||
|
PromptID: prepared.PromptID,
|
||||||
|
SelectedProfile: prepared.SelectedProfileID,
|
||||||
|
Model: prepared.EffectiveModelParams.Model,
|
||||||
|
MessageCount: len(prepared.Messages),
|
||||||
|
ProfileTimeout: *profile.TimeoutSeconds,
|
||||||
|
Namespaces: namespaces,
|
||||||
|
Collections: collectionSummaries,
|
||||||
|
ScopedPrompts: scopedPromptIDs,
|
||||||
|
ReviewPrompts: reviewPromptIDs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
45
examples/go-library/prepare/main_test.go
Normal file
45
examples/go-library/prepare/main_test.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPrepareExample(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := run(t.Context(), ".", &output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got struct {
|
||||||
|
PromptID string `json:"prompt_id"`
|
||||||
|
Profile string `json:"selected_profile"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
MessageCount int `json:"message_count"`
|
||||||
|
Timeout int `json:"profile_timeout_seconds"`
|
||||||
|
Namespaces []string `json:"namespaces"`
|
||||||
|
Collections []map[string]json.RawMessage `json:"collections"`
|
||||||
|
ScopedPrompts []string `json:"scoped_prompts"`
|
||||||
|
ReviewPrompts []string `json:"review_prompts"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.PromptID != "support/prepare" || got.Profile != "support/fast" || got.Model != "offline-model" ||
|
||||||
|
got.MessageCount <= 0 || got.Timeout != 30 || !reflect.DeepEqual(got.Namespaces, []string{"review", "support"}) ||
|
||||||
|
!reflect.DeepEqual(got.ScopedPrompts, []string{"support/prepare"}) || !reflect.DeepEqual(got.ReviewPrompts, []string{"review/check"}) || len(got.Collections) != 2 {
|
||||||
|
t.Fatalf("unexpected preparation summary: %+v", got)
|
||||||
|
}
|
||||||
|
for index, want := range []struct{ namespace, release string }{{"review", ""}, {"support", "1.4.0"}} {
|
||||||
|
var namespace string
|
||||||
|
if err := json.Unmarshal(got.Collections[index]["namespace"], &namespace); err != nil || namespace != want.namespace {
|
||||||
|
t.Fatalf("collection namespace = %q, %v", namespace, err)
|
||||||
|
}
|
||||||
|
value, present := got.Collections[index]["release_version"]
|
||||||
|
var release *string
|
||||||
|
if !present || json.Unmarshal(value, &release) != nil || (index == 0 && release != nil) || (index == 1 && (release == nil || *release != want.release)) {
|
||||||
|
t.Fatalf("collection release = %s, present %t; want %q", value, present, want.release)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
prompts:
|
|
||||||
example.prepare:
|
|
||||||
"1.0.0":
|
|
||||||
default_profile: offline-example
|
|
||||||
description: Prepare a prompt without contacting a model provider.
|
|
||||||
inputs:
|
|
||||||
- name: note
|
|
||||||
required: true
|
|
||||||
content_type: text/plain
|
|
||||||
messages:
|
|
||||||
- role: system
|
|
||||||
content: Summarize the note in one sentence.
|
|
||||||
- role: user
|
|
||||||
content: '{{input "note"}}'
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: basic
|
|
||||||
@@ -4,10 +4,12 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type deterministicClient struct{}
|
type deterministicClient struct{}
|
||||||
@@ -39,41 +41,41 @@ type summary struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
if err := run(context.Background(), "examples/go-library/run", os.Stdout); err != nil {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
fmt.Fprintln(os.Stderr, err)
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
|
os.Exit(1)
|
||||||
"offline-example": {Endpoint: &endpoint, Model: &model},
|
}
|
||||||
})}}}},
|
|
||||||
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
|
|
||||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
|
||||||
if err != nil {
|
|
||||||
exit(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
||||||
|
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", []promptkit.ProfileConfig{
|
||||||
|
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
|
||||||
|
})}}}},
|
||||||
|
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml"))},
|
||||||
|
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(deterministicClient{})) // Use only this example's offline configuration.
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||||
PromptID: "example.run",
|
PromptID: "example.run",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"note": promptkit.Inline("Ada finished the migration review."),
|
"note": promptkit.Inline("Ada finished the migration review."),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
encoder := json.NewEncoder(out)
|
||||||
encoder.SetIndent("", " ")
|
encoder.SetIndent("", " ")
|
||||||
if err := encoder.Encode(summary{
|
return encoder.Encode(summary{
|
||||||
Output: result.RawOutput,
|
Output: result.RawOutput,
|
||||||
ValidationStatus: result.Validation.Status,
|
ValidationStatus: result.Validation.Status,
|
||||||
IsValid: result.Validation.IsValid,
|
IsValid: result.Validation.IsValid,
|
||||||
Model: result.ModelName,
|
Model: result.ModelName,
|
||||||
TotalTokens: result.Usage.TotalTokens,
|
TotalTokens: result.Usage.TotalTokens,
|
||||||
}); err != nil {
|
})
|
||||||
exit(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func exit(err error) {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
|||||||
28
examples/go-library/run/main_test.go
Normal file
28
examples/go-library/run/main_test.go
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRunExample(t *testing.T) {
|
||||||
|
var output bytes.Buffer
|
||||||
|
if err := run(t.Context(), ".", &output); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got struct {
|
||||||
|
Output string `json:"output"`
|
||||||
|
Valid *bool `json:"is_valid"`
|
||||||
|
Status string `json:"validation_status"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
Tokens *int `json:"total_tokens"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Output != "Ada finished the migration review." || got.Valid == nil || !*got.Valid ||
|
||||||
|
got.Status != "passed" || got.Model != "offline-model" || got.Tokens == nil || *got.Tokens != 18 {
|
||||||
|
t.Fatalf("unexpected execution summary: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
prompts:
|
prompts:
|
||||||
example.run:
|
- id: example.run
|
||||||
"1.0.0":
|
|
||||||
default_profile: offline-example
|
default_profile: offline-example
|
||||||
description: Run a prompt with a deterministic injected model client.
|
description: Run a prompt with a deterministic injected model client.
|
||||||
inputs:
|
inputs:
|
||||||
|
|||||||
@@ -18,9 +18,8 @@ func (r RunRequest) GoString() string {
|
|||||||
|
|
||||||
func (r RunRequest) redactedString() string {
|
func (r RunRequest) redactedString() string {
|
||||||
return fmt.Sprintf(
|
return fmt.Sprintf(
|
||||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
"promptkit.RunRequest{PromptID:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t AppendedMessages:%d}",
|
||||||
r.PromptID,
|
r.PromptID,
|
||||||
r.PromptVersion,
|
|
||||||
r.ProfileID,
|
r.ProfileID,
|
||||||
r.APIKey != "",
|
r.APIKey != "",
|
||||||
len(r.Inputs),
|
len(r.Inputs),
|
||||||
|
|||||||
@@ -12,14 +12,13 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
const capturedPromptDocument = `
|
const capturedPromptDocument = `
|
||||||
prompts:
|
prompts:
|
||||||
captured:
|
- id: captured
|
||||||
"1":
|
|
||||||
default_profile: offline
|
default_profile: offline
|
||||||
session_id: '{{.session}}'
|
session_id: '{{.session}}'
|
||||||
messages:
|
messages:
|
||||||
@@ -40,8 +39,8 @@ func capturedPromptEngine(t *testing.T, source fs.FS) *promptkit.Engine {
|
|||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{
|
Underlays: []configkit.Underlay{
|
||||||
{Inputs: inputs},
|
{Inputs: inputs},
|
||||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", []promptkit.ProfileConfig{
|
||||||
"offline": {Endpoint: &endpoint, Model: &model},
|
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||||
})}}},
|
})}}},
|
||||||
},
|
},
|
||||||
}})
|
}})
|
||||||
@@ -145,16 +144,16 @@ func TestPreparationStillReadsRequestArtifactsAndSchemasAfterConstruction(t *tes
|
|||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model := "https://example.invalid/v1", "offline-model"
|
||||||
schemas := fstest.MapFS{}
|
schemas := fstest.MapFS{}
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", []promptkit.ProfileConfig{
|
||||||
"offline": {Endpoint: &endpoint, Model: &model},
|
{ID: "offline", Endpoint: &endpoint, Model: &model},
|
||||||
})}}}},
|
})}}}},
|
||||||
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
Main: &configkit.Input{Source: configkit.Mapping("structured prompt", map[string]any{
|
||||||
"prompts": map[string]any{"structured": map[string]any{"1": map[string]any{
|
"prompts": []any{map[string]any{"id": "structured",
|
||||||
"default_profile": "offline",
|
"default_profile": "offline",
|
||||||
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
"inputs": []any{map[string]any{"name": "note", "required": true}},
|
||||||
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
|
"messages": []any{map[string]any{"role": "user", "content": `{{input "note"}}`}},
|
||||||
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"},
|
||||||
}}},
|
}},
|
||||||
})},
|
})},
|
||||||
}}, promptkit.WithSchemaFS(schemas, "."))
|
}}, promptkit.WithSchemaFS(schemas, "."))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuiltInGenerationError(t *testing.T) {
|
func TestBuiltInGenerationError(t *testing.T) {
|
||||||
|
|||||||
15
go.mod
15
go.mod
@@ -1,15 +1,18 @@
|
|||||||
module gitea.maximumdirect.net/eric/promptkit
|
module go.maximumdirect.net/promptkit
|
||||||
|
|
||||||
go 1.25.5
|
go 1.27.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
|
go.maximumdirect.net/configkit v1.1.0
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
|
go.maximumdirect.net/releasekit v0.9.0 // indirect
|
||||||
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
|
tool go.maximumdirect.net/releasekit/cmd/releasekit
|
||||||
|
|||||||
16
go.sum
16
go.sum
@@ -1,16 +1,16 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
|
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
|
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||||
|
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||||
|
go.maximumdirect.net/configkit v1.1.0 h1:UzXN2ZMnEkcL0O4YSLqD9qd9M1gb/NWCHQR/H8AaQzA=
|
||||||
|
go.maximumdirect.net/configkit v1.1.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
|
||||||
|
go.maximumdirect.net/releasekit v0.9.0 h1:sK+Aoj6gS6ATU/1reocjwBekp4YETYENwkcX5ti6mrE=
|
||||||
|
go.maximumdirect.net/releasekit v0.9.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
|
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -21,6 +21,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const fileReadChunkSize = 64 * 1024
|
const fileReadChunkSize = 64 * 1024
|
||||||
|
const fileReadInitialCapacityLimit = 1 << 20
|
||||||
|
|
||||||
// Reader resolves artifact references into actual artifacts.
|
// Reader resolves artifact references into actual artifacts.
|
||||||
type Reader interface {
|
type Reader interface {
|
||||||
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
|||||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
data := make([]byte, 0)
|
capacity := 0
|
||||||
|
if size := openedInfo.Size(); size > 0 {
|
||||||
|
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
|
||||||
|
}
|
||||||
|
data := make([]byte, 0, capacity)
|
||||||
chunk := make([]byte, fileReadChunkSize)
|
chunk := make([]byte, fileReadChunkSize)
|
||||||
for {
|
for {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
|
|||||||
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if cap(data)-len(data) > len(data) {
|
||||||
|
// A stale size hint should not make a small completed artifact retain a
|
||||||
|
// disproportionately large backing array.
|
||||||
|
compact := make([]byte, len(data))
|
||||||
|
copy(compact, data)
|
||||||
|
data = compact
|
||||||
|
}
|
||||||
|
|
||||||
contentType := mime.TypeByExtension(filepath.Ext(path))
|
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||||
if contentType == "" {
|
if contentType == "" {
|
||||||
|
|||||||
181
internal/artifact/reader_benchmark_test.go
Normal file
181
internal/artifact/reader_benchmark_test.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
package artifact
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"mime"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
var artifactFileReadBenchmarkSink *domain.Artifact
|
||||||
|
|
||||||
|
func BenchmarkArtifactFileRead(b *testing.B) {
|
||||||
|
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
|
||||||
|
b.Run(fixture.name, func(b *testing.B) {
|
||||||
|
reader := &fileReader{open: openArtifactFile}
|
||||||
|
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
|
||||||
|
checkArtifactFileRead(b, reader, ref, fixture.body)
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(fixture.body)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for range b.N {
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("read artifact: %v", err)
|
||||||
|
}
|
||||||
|
artifactFileReadBenchmarkSink = artifact
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
runtime.KeepAlive(artifactFileReadBenchmarkSink)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
|
||||||
|
// opened file reports a size larger than the content that reaches EOF.
|
||||||
|
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
|
||||||
|
for _, fixture := range []struct {
|
||||||
|
name string
|
||||||
|
body []byte
|
||||||
|
}{
|
||||||
|
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
|
||||||
|
{name: "empty", body: []byte{}},
|
||||||
|
} {
|
||||||
|
b.Run(fixture.name, func(b *testing.B) {
|
||||||
|
benchmarkRetainedArtifactFileRead(b, fixture.body)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
|
||||||
|
path := filepath.Join(b.TempDir(), "overreported.txt")
|
||||||
|
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
mime.TypeByExtension(filepath.Ext(path))
|
||||||
|
|
||||||
|
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
|
||||||
|
}}
|
||||||
|
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
||||||
|
checkArtifactFileRead(b, reader, ref, body)
|
||||||
|
|
||||||
|
const retainedBatchSize = 4
|
||||||
|
retained := make([]*domain.Artifact, retainedBatchSize)
|
||||||
|
artifactFileReadBenchmarkSink = nil
|
||||||
|
runtime.GC()
|
||||||
|
var setupMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&setupMemory)
|
||||||
|
runtime.KeepAlive(reader)
|
||||||
|
runtime.KeepAlive(ref)
|
||||||
|
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(body)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for iteration := 0; iteration < b.N; iteration++ {
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("read retained artifact: %v", err)
|
||||||
|
}
|
||||||
|
retained[iteration%len(retained)] = artifact
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
for i, artifact := range retained {
|
||||||
|
if artifact != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("fill retained artifact batch: %v", err)
|
||||||
|
}
|
||||||
|
retained[i] = artifact
|
||||||
|
}
|
||||||
|
runtime.GC()
|
||||||
|
var retainedMemory runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&retainedMemory)
|
||||||
|
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
|
||||||
|
runtime.KeepAlive(retained)
|
||||||
|
runtime.KeepAlive(reader)
|
||||||
|
runtime.KeepAlive(ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
type artifactReadBenchmarkFile struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
body []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
|
||||||
|
b.Helper()
|
||||||
|
fixtures := []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
}{
|
||||||
|
{name: "small_4KiB", size: 4 << 10},
|
||||||
|
{name: "1MiB", size: 1 << 20},
|
||||||
|
{name: "large_4MiB", size: 4 << 20},
|
||||||
|
}
|
||||||
|
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
|
||||||
|
for _, fixture := range fixtures {
|
||||||
|
body := bytes.Repeat([]byte("x"), fixture.size)
|
||||||
|
path := filepath.Join(b.TempDir(), fixture.name+".txt")
|
||||||
|
if err := os.WriteFile(path, body, 0o600); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
mime.TypeByExtension(filepath.Ext(path))
|
||||||
|
results = append(results, artifactReadBenchmarkFile{
|
||||||
|
name: fixture.name,
|
||||||
|
path: path,
|
||||||
|
body: body,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
|
||||||
|
b.Helper()
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("check artifact read: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||||
|
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||||
|
}
|
||||||
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(want)})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
b.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type sizeReportingArtifactFile struct {
|
||||||
|
artifactFile
|
||||||
|
reportedSize int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
|
||||||
|
info, err := file.artifactFile.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type reportedArtifactFileInfo struct {
|
||||||
|
os.FileInfo
|
||||||
|
size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info reportedArtifactFileInfo) Size() int64 { return info.size }
|
||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) {
|
func TestFileReaderRejectsFIFOBeforeOpen(t *testing.T) {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) {
|
func TestCompositeReaderRejectsUnsupportedReferences(t *testing.T) {
|
||||||
@@ -232,6 +232,67 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body []byte
|
||||||
|
reportedSize int64
|
||||||
|
}{
|
||||||
|
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
|
||||||
|
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
|
||||||
|
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
|
||||||
|
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
filePath := filepath.Join(t.TempDir(), "artifact.txt")
|
||||||
|
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reader := &fileReader{open: func(path string) (artifactFile, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
file.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &reportedInfoFile{
|
||||||
|
artifactFile: file,
|
||||||
|
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
|
||||||
|
}, nil
|
||||||
|
}}
|
||||||
|
|
||||||
|
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
|
||||||
|
Type: domain.ArtifactRefFile,
|
||||||
|
URI: filePath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read artifact: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(artifact.Body, tc.body) {
|
||||||
|
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
|
||||||
|
}
|
||||||
|
if artifact.Size != int64(len(tc.body)) {
|
||||||
|
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
|
||||||
|
}
|
||||||
|
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||||
|
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||||
|
}
|
||||||
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(tc.body)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
t.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
|
||||||
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
filePath := filepath.Join(t.TempDir(), "artifact.bin")
|
||||||
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
|
||||||
@@ -268,6 +329,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
|
|||||||
return f.info, nil
|
return f.info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type reportedFileSizeInfo struct {
|
||||||
|
os.FileInfo
|
||||||
|
size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info reportedFileSizeInfo) Size() int64 { return info.size }
|
||||||
|
|
||||||
type cancelAfterProgressFile struct {
|
type cancelAfterProgressFile struct {
|
||||||
artifactFile
|
artifactFile
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
|
|||||||
@@ -6,14 +6,13 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// OpenRouterID identifies the maintained OpenRouter backend.
|
// OpenRouterID is the conventional OpenRouter backend identifier.
|
||||||
OpenRouterID = "openrouter"
|
OpenRouterID = "openrouter"
|
||||||
// RakestrawHomeID identifies the maintained Rakestrawhome backend.
|
// RakestrawHomeID is the conventional Rakestrawhome backend identifier.
|
||||||
RakestrawHomeID = "rakestrawhome"
|
RakestrawHomeID = "rakestrawhome"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -28,18 +27,13 @@ type Registry struct {
|
|||||||
// NewEffectiveRegistry stores already-validated effective backend definitions
|
// NewEffectiveRegistry stores already-validated effective backend definitions
|
||||||
// under their exact configuration keys. Configuration validation and
|
// under their exact configuration keys. Configuration validation and
|
||||||
// normalization precede this ownership boundary.
|
// normalization precede this ownership boundary.
|
||||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
func NewEffectiveRegistry(definitions map[string]domain.Backend) *Registry {
|
||||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||||
for id, definition := range definitions {
|
for id, definition := range definitions {
|
||||||
params, err := jsonvalue.CopyMap(definition.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("backend %q extra parameters: %w", id, err)
|
|
||||||
}
|
|
||||||
definition.ID = id
|
definition.ID = id
|
||||||
definition.ExtraParams = params
|
registry.backends[id] = domain.CloneBackend(definition)
|
||||||
registry.backends[id] = definition
|
|
||||||
}
|
}
|
||||||
return registry, nil
|
return registry
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBackend returns a defensive copy of the backend registered with id.
|
// GetBackend returns a defensive copy of the backend registered with id.
|
||||||
@@ -51,12 +45,7 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
}
|
}
|
||||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
return domain.CloneBackend(definition), nil
|
||||||
if err != nil {
|
|
||||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
|
||||||
}
|
|
||||||
definition.ExtraParams = extraParams
|
|
||||||
return definition, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||||
|
|||||||
@@ -5,31 +5,64 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
||||||
params := map[string]any{"nested": map[string]any{"enabled": true}}
|
params := map[string]any{"nested": map[string]any{"enabled": true}}
|
||||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
defaultProfile := "catalog-profile"
|
||||||
|
reachabilityEndpoint := "https://probe.example/ready"
|
||||||
|
authenticationKind := "openrouter_key"
|
||||||
|
authenticationEndpoint := "https://probe.example/key"
|
||||||
|
smokeTests := &domain.BackendSmokeTestConfig{
|
||||||
|
DefaultProfile: &defaultProfile,
|
||||||
|
Reachability: &domain.ReachabilityProbeConfig{Endpoint: &reachabilityEndpoint},
|
||||||
|
Authentication: &domain.AuthenticationProbeConfig{
|
||||||
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
registry := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||||
" local.v1 ": {
|
" local.v1 ": {
|
||||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
ConcurrencyLimit: 2, QueueCapacity: 3,
|
||||||
|
SmokeTests: smokeTests,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
params["nested"].(map[string]any)["enabled"] = false
|
params["nested"].(map[string]any)["enabled"] = false
|
||||||
|
defaultProfile = "mutated-profile"
|
||||||
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
|
authenticationKind = "none"
|
||||||
|
authenticationEndpoint = "https://mutated.example/key"
|
||||||
selected, err := registry.GetBackend(" local.v1 ")
|
selected, err := registry.GetBackend(" local.v1 ")
|
||||||
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||||
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
|
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
|
||||||
}
|
}
|
||||||
|
if selected.SmokeTests == nil || selected.SmokeTests.DefaultProfile == nil ||
|
||||||
|
*selected.SmokeTests.DefaultProfile != "catalog-profile" || selected.SmokeTests.Reachability == nil ||
|
||||||
|
selected.SmokeTests.Reachability.Endpoint == nil || *selected.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
|
||||||
|
selected.SmokeTests.Authentication == nil || selected.SmokeTests.Authentication.Kind == nil ||
|
||||||
|
*selected.SmokeTests.Authentication.Kind != "openrouter_key" || selected.SmokeTests.Authentication.Endpoint == nil ||
|
||||||
|
*selected.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
|
||||||
|
t.Fatalf("registry did not capture nested diagnostic metadata: %+v", selected.SmokeTests)
|
||||||
|
}
|
||||||
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
|
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
|
||||||
|
*selected.SmokeTests.DefaultProfile = "mutated-return"
|
||||||
|
*selected.SmokeTests.Reachability.Endpoint = "https://mutated-return.example/ready"
|
||||||
|
*selected.SmokeTests.Authentication.Kind = "none"
|
||||||
|
*selected.SmokeTests.Authentication.Endpoint = "https://mutated-return.example/key"
|
||||||
again, err := registry.GetBackend(" local.v1 ")
|
again, err := registry.GetBackend(" local.v1 ")
|
||||||
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
|
||||||
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
|
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
|
||||||
}
|
}
|
||||||
|
if again.SmokeTests == nil || again.SmokeTests.DefaultProfile == nil ||
|
||||||
|
*again.SmokeTests.DefaultProfile != "catalog-profile" || again.SmokeTests.Reachability == nil ||
|
||||||
|
again.SmokeTests.Reachability.Endpoint == nil || *again.SmokeTests.Reachability.Endpoint != "https://probe.example/ready" ||
|
||||||
|
again.SmokeTests.Authentication == nil || again.SmokeTests.Authentication.Kind == nil ||
|
||||||
|
*again.SmokeTests.Authentication.Kind != "openrouter_key" || again.SmokeTests.Authentication.Endpoint == nil ||
|
||||||
|
*again.SmokeTests.Authentication.Endpoint != "https://probe.example/key" {
|
||||||
|
t.Fatalf("lookup exposed nested diagnostic metadata: %+v", again.SmokeTests)
|
||||||
|
}
|
||||||
policies := registry.CapacityPolicies()
|
policies := registry.CapacityPolicies()
|
||||||
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
|
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
|
||||||
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
|
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
|
||||||
@@ -43,10 +76,7 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
||||||
registry, err := backend.NewEffectiveRegistry(nil)
|
registry := backend.NewEffectiveRegistry(nil)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
t.Fatalf("missing backend error = %v", err)
|
t.Fatalf("missing backend error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
package capacity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
|
||||||
)
|
|
||||||
|
|
||||||
type client struct {
|
|
||||||
manager *Manager
|
|
||||||
next llm.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient wraps next with configured active-generation limits. A nil manager
|
|
||||||
// leaves next unchanged.
|
|
||||||
func NewClient(manager *Manager, next llm.Client) llm.Client {
|
|
||||||
if manager == nil {
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
return &client{
|
|
||||||
manager: manager,
|
|
||||||
next: next,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *client) Generate(
|
|
||||||
ctx context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
if c.manager.getPool(req.Target.BackendID) == nil {
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
@@ -3,15 +3,14 @@ package capacity
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
type generateResult struct {
|
type generateResult struct {
|
||||||
@@ -24,6 +23,18 @@ type clientFunc func(
|
|||||||
domain.GenerateRequest,
|
domain.GenerateRequest,
|
||||||
) (*domain.GenerateResponse, error)
|
) (*domain.GenerateResponse, error)
|
||||||
|
|
||||||
|
// newTestClient exercises one acquisition around a controlled call.
|
||||||
|
func newTestClient(manager *Manager, next llm.Client) llm.Client {
|
||||||
|
return clientFunc(func(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
release, err := manager.Acquire(ctx, req.Target.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
return next.Generate(ctx, req)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (f clientFunc) Generate(
|
func (f clientFunc) Generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -167,7 +178,7 @@ func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
|
|||||||
"second": secondRelease,
|
"second": secondRelease,
|
||||||
"third": thirdRelease,
|
"third": thirdRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
first := generateAsync(client, context.Background(), "limited", "first")
|
first := generateAsync(client, context.Background(), "limited", "first")
|
||||||
if got := receiveStarted(t, next.started); got != "first" {
|
if got := receiveStarted(t, next.started); got != "first" {
|
||||||
@@ -212,7 +223,7 @@ func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
|
|||||||
releases[string(rune('a'+i))] = gate
|
releases[string(rune('a'+i))] = gate
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
results := make([]<-chan generateResult, 0, len(releases))
|
results := make([]<-chan generateResult, 0, len(releases))
|
||||||
for id := range releases {
|
for id := range releases {
|
||||||
@@ -257,7 +268,7 @@ func TestClientRemovesCanceledWaiters(t *testing.T) {
|
|||||||
"three": make(chan struct{}),
|
"three": make(chan struct{}),
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
if got := receiveStarted(t, next.started); got != "holder" {
|
if got := receiveStarted(t, next.started); got != "holder" {
|
||||||
@@ -321,7 +332,7 @@ func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
waitForActiveCount(t, manager, "limited", 1)
|
waitForActiveCount(t, manager, "limited", 1)
|
||||||
@@ -394,7 +405,7 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
"alpha": alphaRelease,
|
"alpha": alphaRelease,
|
||||||
"beta": betaRelease,
|
"beta": betaRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
||||||
beta := generateAsync(client, context.Background(), "beta", "beta")
|
beta := generateAsync(client, context.Background(), "beta", "beta")
|
||||||
@@ -423,63 +434,6 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if got := NewClient(nil, next); got != next {
|
|
||||||
t.Fatal("nil manager did not return the wrapped client unchanged")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
|
|
||||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
|
||||||
"limited": {ConcurrencyLimit: 1},
|
|
||||||
})
|
|
||||||
request := domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{
|
|
||||||
SessionID: "session",
|
|
||||||
Messages: []domain.RenderedMessage{
|
|
||||||
{Role: "user", Content: "content"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Target: domain.ExecutionTarget{
|
|
||||||
BackendID: "limited",
|
|
||||||
Model: "model",
|
|
||||||
ExtraParams: map[string]any{"key": "value"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
response := &domain.GenerateResponse{
|
|
||||||
Content: "output",
|
|
||||||
Usage: domain.TokenUsage{TotalTokens: 7},
|
|
||||||
}
|
|
||||||
collaboratorErr := errors.New("collaborator failure")
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
response *domain.GenerateResponse
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
{name: "successful response", response: response},
|
|
||||||
{name: "nil response"},
|
|
||||||
{name: "collaborator error", response: response, err: collaboratorErr},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
var captured domain.GenerateRequest
|
|
||||||
next := clientFunc(func(
|
|
||||||
_ context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
captured = req
|
|
||||||
return tc.response, tc.err
|
|
||||||
})
|
|
||||||
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
|
|
||||||
if !reflect.DeepEqual(captured, request) {
|
|
||||||
t.Fatalf("request changed: %#v", captured)
|
|
||||||
}
|
|
||||||
if gotResponse != tc.response || gotErr != tc.err {
|
|
||||||
t.Fatalf("response=(%p, %v), want (%p, %v)",
|
|
||||||
gotResponse, gotErr, tc.response, tc.err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||||
@@ -496,7 +450,7 @@ func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: "recovered"}, nil
|
return &domain.GenerateResponse{Content: "recovered"}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
request := domain.GenerateRequest{
|
request := domain.GenerateRequest{
|
||||||
Target: domain.ExecutionTarget{BackendID: "limited"},
|
Target: domain.ExecutionTarget{BackendID: "limited"},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrCapacityExceeded identifies an admission rejected because a backend's
|
// ErrCapacityExceeded identifies an admission rejected because a backend's
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
|
func TestNewManagerRejectsInvalidPolicies(t *testing.T) {
|
||||||
|
|||||||
@@ -1,52 +0,0 @@
|
|||||||
// Package catalog selects the maintained configuration assets supplied with
|
|
||||||
// PromptKit. ConfigKit owns their composition and effective validation.
|
|
||||||
package catalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3"
|
|
||||||
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Underlays selects external catalogs followed by PromptKit-owned profiles.
|
|
||||||
// Neither group is interpreted before the final effective configuration loads.
|
|
||||||
func Underlays() ([]configkit.Underlay, error) {
|
|
||||||
openrouterInputs, err := assetInputs("OpenRouter catalog", openrouter.FS(), openrouter.Root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
rakestrawhomeInputs, err := assetInputs("Rakestrawhome catalog", rakestrawhome.FS(), rakestrawhome.Root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
catalogInputs := append(openrouterInputs, rakestrawhomeInputs...)
|
|
||||||
owned := configkit.Input{Source: configkit.Mapping("PromptKit-owned profiles", map[string]any{
|
|
||||||
"profiles": map[string]any{
|
|
||||||
"promptkit-openrouter-standard": map[string]any{"extends": "gemini-flash-latest"},
|
|
||||||
"promptkit-openrouter-auto": map[string]any{"backend": backend.OpenRouterID, "model": "openrouter/auto"},
|
|
||||||
},
|
|
||||||
})}
|
|
||||||
return []configkit.Underlay{{Inputs: catalogInputs}, {Inputs: []configkit.Input{owned}}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func assetInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
|
|
||||||
files, err := filecatalog.FindFSYAMLFiles(context.Background(), filesystem, root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("discover %s configuration: %w", label, err)
|
|
||||||
}
|
|
||||||
namespace := configkit.NewFilesystem(label, filesystem)
|
|
||||||
inputs := make([]configkit.Input, 0, len(files))
|
|
||||||
for _, name := range files {
|
|
||||||
inputs = append(inputs, configkit.Input{
|
|
||||||
Source: namespace.File(name),
|
|
||||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: root},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return inputs, nil
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
package catalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPublishedCatalogsResolveThroughEffectiveConfiguration(t *testing.T) {
|
|
||||||
underlays, err := Underlays()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
state, err := configuration.Load(configkit.Input{Source: configkit.NoSource()}, underlays)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
openrouter, err := state.GetBackend("openrouter")
|
|
||||||
if err != nil || openrouter.Endpoint != "https://openrouter.ai/api/v1" || openrouter.ConcurrencyLimit != 16 || openrouter.QueueCapacity != 1024 {
|
|
||||||
t.Fatalf("OpenRouter backend: %+v, %v", openrouter, err)
|
|
||||||
}
|
|
||||||
rakestrawhome, err := state.GetBackend("rakestrawhome")
|
|
||||||
if err != nil || rakestrawhome.Endpoint != "https://inference.ai.rakestrawhome.com/v1" || rakestrawhome.ConcurrencyLimit != 4 {
|
|
||||||
t.Fatalf("Rakestrawhome backend: %+v, %v", rakestrawhome, err)
|
|
||||||
}
|
|
||||||
for id, want := range map[string]string{
|
|
||||||
"promptkit-openrouter-standard": "~google/gemini-flash-latest",
|
|
||||||
"promptkit-openrouter-auto": "openrouter/auto",
|
|
||||||
"rakestrawhome-gemma-4-31b": "google/gemma-4-31b-it",
|
|
||||||
} {
|
|
||||||
profile, err := state.GetProfile(context.Background(), id)
|
|
||||||
if err != nil || profile.ResolvedTarget.Model != want {
|
|
||||||
t.Fatalf("profile %s: %+v, %v", id, profile, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConsumerOverlayReplacesCatalogDefaultsBeforeValidation(t *testing.T) {
|
|
||||||
underlays, err := Underlays()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
|
||||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid lower value"}},
|
|
||||||
})}}})
|
|
||||||
main := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
|
||||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "https://alternate.example/v1"}},
|
|
||||||
"profiles": map[string]any{"gemini-flash-latest": map[string]any{"model": "consumer-model"}},
|
|
||||||
})}
|
|
||||||
state, err := configuration.Load(main, underlays)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("replaced lower-layer value failed final validation: %v", err)
|
|
||||||
}
|
|
||||||
profile, err := state.GetProfile(context.Background(), "promptkit-openrouter-standard")
|
|
||||||
if err != nil || profile.ResolvedTarget.Model != "consumer-model" || profile.ResolvedTarget.Endpoint != "https://alternate.example/v1" {
|
|
||||||
t.Fatalf("maintained profile did not observe final configuration: %+v, %v", profile, err)
|
|
||||||
}
|
|
||||||
_, err = configuration.Load(configkit.Input{Source: configkit.Mapping("invalid final", map[string]any{
|
|
||||||
"backends": map[string]any{"openrouter": map[string]any{"endpoint": "invalid final value"}},
|
|
||||||
})}, underlays[:2])
|
|
||||||
var loadError *configkit.LoadError
|
|
||||||
if !errors.As(err, &loadError) {
|
|
||||||
t.Fatalf("invalid final backend did not produce a ConfigKit load error: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
const benchmarkDefinitionCount = 128
|
const benchmarkDefinitionCount = 128
|
||||||
@@ -15,7 +15,10 @@ const benchmarkDefinitionCount = 128
|
|||||||
var (
|
var (
|
||||||
constructionResult *loaded
|
constructionResult *loaded
|
||||||
selectedPrompt *domain.PromptDefinition
|
selectedPrompt *domain.PromptDefinition
|
||||||
|
selectedPromptHash string
|
||||||
|
selectedPromptInspection *domain.PromptInspection
|
||||||
selectedProfile *domain.ExecutionProfile
|
selectedProfile *domain.ExecutionProfile
|
||||||
|
selectedProfileIdentity string
|
||||||
)
|
)
|
||||||
|
|
||||||
type configurationShape struct {
|
type configurationShape struct {
|
||||||
@@ -34,6 +37,20 @@ func benchmarkPrompt(profileID string) map[string]any {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func benchmarkRecords(values map[string]any) []any {
|
||||||
|
records := make([]any, 0, len(values))
|
||||||
|
for _, id := range sortedKeys(values) {
|
||||||
|
value := values[id].(map[string]any)
|
||||||
|
record := make(map[string]any, len(value)+1)
|
||||||
|
for key, field := range value {
|
||||||
|
record[key] = field
|
||||||
|
}
|
||||||
|
record["id"] = id
|
||||||
|
records = append(records, record)
|
||||||
|
}
|
||||||
|
return records
|
||||||
|
}
|
||||||
|
|
||||||
func independentDefinitions(count int) configurationShape {
|
func independentDefinitions(count int) configurationShape {
|
||||||
profiles := make(map[string]any, count)
|
profiles := make(map[string]any, count)
|
||||||
prompts := make(map[string]any, count)
|
prompts := make(map[string]any, count)
|
||||||
@@ -41,10 +58,10 @@ func independentDefinitions(count int) configurationShape {
|
|||||||
profileID := fmt.Sprintf("profile-%03d", index)
|
profileID := fmt.Sprintf("profile-%03d", index)
|
||||||
promptID := fmt.Sprintf("prompt-%03d", index)
|
promptID := fmt.Sprintf("prompt-%03d", index)
|
||||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||||
prompts[promptID] = map[string]any{"1": benchmarkPrompt(profileID)}
|
prompts[promptID] = benchmarkPrompt(profileID)
|
||||||
}
|
}
|
||||||
return configurationShape{
|
return configurationShape{
|
||||||
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": profiles, "prompts": prompts})},
|
main: configkit.Input{Source: configkit.Mapping("independent definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)})},
|
||||||
inputEntries: count * 2,
|
inputEntries: count * 2,
|
||||||
effectiveEntries: count * 2,
|
effectiveEntries: count * 2,
|
||||||
tiers: 1,
|
tiers: 1,
|
||||||
@@ -60,14 +77,14 @@ func fileBackedDefinitions(count int) configurationShape {
|
|||||||
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
profiles[profileID] = map[string]any{"endpoint": "https://example.test/v1", "model": profileID}
|
||||||
prompt := benchmarkPrompt(profileID)
|
prompt := benchmarkPrompt(profileID)
|
||||||
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
prompt["messages"] = []any{map[string]any{"role": "user", "content_file": "body.md"}}
|
||||||
prompts[promptID] = map[string]any{"1": prompt}
|
prompts[promptID] = prompt
|
||||||
}
|
}
|
||||||
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
namespace := configkit.NewFilesystem("benchmark content", fstest.MapFS{
|
||||||
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
"assets/body.md": &fstest.MapFile{Data: []byte("Hello {{.name}}")},
|
||||||
})
|
})
|
||||||
return configurationShape{
|
return configurationShape{
|
||||||
main: configkit.Input{
|
main: configkit.Input{
|
||||||
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": profiles, "prompts": prompts}),
|
Source: configkit.Mapping("file-backed definitions", map[string]any{"profiles": benchmarkRecords(profiles), "prompts": benchmarkRecords(prompts)}),
|
||||||
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "assets"},
|
||||||
},
|
},
|
||||||
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
|
inputEntries: count * 2, effectiveEntries: count * 2, tiers: 1,
|
||||||
@@ -81,12 +98,12 @@ func sharedBaseDefinitions(count int) configurationShape {
|
|||||||
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
|
"extra_params": map[string]any{"provider": map[string]any{"mode": "shared", "limit": 4}},
|
||||||
}
|
}
|
||||||
for index := range count {
|
for index := range count {
|
||||||
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": "base", "temperature": float64(index%10) / 10}
|
profiles[fmt.Sprintf("child-%03d", index)] = map[string]any{"extends": map[string]any{"id": "base"}, "temperature": float64(index%10) / 10}
|
||||||
}
|
}
|
||||||
return configurationShape{
|
return configurationShape{
|
||||||
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
main: configkit.Input{Source: configkit.Mapping("inherited definitions", map[string]any{
|
||||||
"profiles": profiles,
|
"profiles": benchmarkRecords(profiles),
|
||||||
"prompts": map[string]any{"shared": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}},
|
"prompts": benchmarkRecords(map[string]any{"shared": benchmarkPrompt(fmt.Sprintf("child-%03d", count-1))}),
|
||||||
})},
|
})},
|
||||||
inputEntries: count + 2,
|
inputEntries: count + 2,
|
||||||
effectiveEntries: count + 2,
|
effectiveEntries: count + 2,
|
||||||
@@ -102,19 +119,19 @@ func layeredDefinitions(count int) configurationShape {
|
|||||||
for index := range count {
|
for index := range count {
|
||||||
id := fmt.Sprintf("profile-%03d", index)
|
id := fmt.Sprintf("profile-%03d", index)
|
||||||
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
|
base[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"}
|
||||||
operator[id] = map[string]any{"model": "operator-model"}
|
operator[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model"}
|
||||||
programmatic[id] = map[string]any{"extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
programmatic[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}}
|
||||||
final[id] = map[string]any{"temperature": float64(index%10) / 10}
|
final[id] = map[string]any{"endpoint": "https://example.test/v1", "model": "operator-model", "extra_params": map[string]any{"provider": map[string]any{"mode": "programmatic"}}, "temperature": float64(index%10) / 10}
|
||||||
}
|
}
|
||||||
return configurationShape{
|
return configurationShape{
|
||||||
underlays: []configkit.Underlay{
|
underlays: []configkit.Underlay{
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": base})}}},
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("fallback", map[string]any{"profiles": benchmarkRecords(base)})}}},
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": operator})}}},
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{"profiles": benchmarkRecords(operator)})}}},
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": programmatic})}}},
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("programmatic", map[string]any{"profiles": benchmarkRecords(programmatic)})}}},
|
||||||
},
|
},
|
||||||
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
main: configkit.Input{Source: configkit.Mapping("main", map[string]any{
|
||||||
"profiles": final,
|
"profiles": benchmarkRecords(final),
|
||||||
"prompts": map[string]any{"layered": map[string]any{"1": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}},
|
"prompts": benchmarkRecords(map[string]any{"layered": benchmarkPrompt(fmt.Sprintf("profile-%03d", count-1))}),
|
||||||
})},
|
})},
|
||||||
inputEntries: count*4 + 1,
|
inputEntries: count*4 + 1,
|
||||||
effectiveEntries: count + 1,
|
effectiveEntries: count + 1,
|
||||||
@@ -122,6 +139,40 @@ func layeredDefinitions(count int) configurationShape {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func BenchmarkCollectionFileBackedConstruction(b *testing.B) {
|
||||||
|
for _, count := range []int{4, 16, 64} {
|
||||||
|
b.Run(fmt.Sprintf("collections_%d", count), func(b *testing.B) {
|
||||||
|
collections := make(map[string]any, count)
|
||||||
|
for index := range count {
|
||||||
|
collections[fmt.Sprintf("collection-%03d", index)] = map[string]any{
|
||||||
|
"prompts": []any{
|
||||||
|
map[string]any{"id": "example",
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content_file": "body.md"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
namespace := configkit.NewFilesystem("collection benchmark", fstest.MapFS{
|
||||||
|
"body.md": {Data: []byte("Hello")},
|
||||||
|
})
|
||||||
|
main := configkit.Input{
|
||||||
|
Source: configkit.Mapping("collections", map[string]any{"collections": collections}),
|
||||||
|
Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: ".", BaseDir: "."},
|
||||||
|
}
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
result, err := load(main, nil)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
constructionResult = result
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(count), "collections/op")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func BenchmarkConfigurationConstruction(b *testing.B) {
|
func BenchmarkConfigurationConstruction(b *testing.B) {
|
||||||
for _, scenario := range []struct {
|
for _, scenario := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -157,7 +208,7 @@ func BenchmarkConfigurationSelection(b *testing.B) {
|
|||||||
b.Run("prompt", func(b *testing.B) {
|
b.Run("prompt", func(b *testing.B) {
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
for b.Loop() {
|
for b.Loop() {
|
||||||
definition, _, err := result.prompt("prompt-127", "1")
|
definition, _, err := result.prompt("prompt-127")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -177,3 +228,116 @@ func BenchmarkConfigurationSelection(b *testing.B) {
|
|||||||
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func inspectionExtraParams(entries int) map[string]any {
|
||||||
|
providers := make(map[string]any, entries)
|
||||||
|
for index := range entries {
|
||||||
|
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
|
||||||
|
"limits": map[string]any{"burst": index + 1, "period": "minute"},
|
||||||
|
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map[string]any{"provider_options": providers}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectionDefinitions(extraParams map[string]any) configurationShape {
|
||||||
|
profiles := map[string]any{
|
||||||
|
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
|
||||||
|
}
|
||||||
|
prompt := benchmarkPrompt("small")
|
||||||
|
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
|
||||||
|
prompts := map[string]any{
|
||||||
|
"inspection": prompt,
|
||||||
|
}
|
||||||
|
return configurationShape{
|
||||||
|
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
|
||||||
|
"profiles": benchmarkRecords(profiles),
|
||||||
|
"prompts": benchmarkRecords(prompts),
|
||||||
|
})},
|
||||||
|
inputEntries: 2, effectiveEntries: 2, tiers: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkInspectionSelection(b *testing.B) {
|
||||||
|
for _, scenario := range []struct {
|
||||||
|
name string
|
||||||
|
entries int
|
||||||
|
}{
|
||||||
|
{name: "small_extra", entries: 2},
|
||||||
|
{name: "nested_extra", entries: 64},
|
||||||
|
} {
|
||||||
|
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
|
||||||
|
result, err := load(shape.main, nil)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
||||||
|
if err != nil || definition.ID != "inspection" || hash == "" ||
|
||||||
|
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
|
||||||
|
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
|
||||||
|
}
|
||||||
|
profile, err := result.GetProfile(ctx, "small")
|
||||||
|
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
|
||||||
|
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
|
||||||
|
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
|
||||||
|
}
|
||||||
|
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
||||||
|
if err != nil || inspection.PromptID != "inspection" ||
|
||||||
|
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
|
||||||
|
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
|
||||||
|
}
|
||||||
|
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||||
|
if err != nil || identity != "small" {
|
||||||
|
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
b.Run("prompt/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedPrompt = definition
|
||||||
|
selectedPromptHash = hash
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
inspection, err := result.GetPromptInspection(ctx, "inspection")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedPromptInspection = inspection
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
b.Run("profile/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
profile, err := result.GetProfile(ctx, "small")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedProfile = profile
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
identity, err := result.GetProfileIdentity(ctx, "small")
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
selectedProfileIdentity = identity
|
||||||
|
}
|
||||||
|
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
16
internal/configuration/bundled.go
Normal file
16
internal/configuration/bundled.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed bundled/openrouter.yaml
|
||||||
|
var bundledFiles embed.FS
|
||||||
|
|
||||||
|
// BundledOpenRouterUnderlay returns the built-in OpenRouter configuration source.
|
||||||
|
func BundledOpenRouterUnderlay() configkit.Underlay {
|
||||||
|
filesystem := configkit.NewFilesystem("Promptkit bundled OpenRouter", bundledFiles)
|
||||||
|
return configkit.Underlay{Inputs: []configkit.Input{{Source: filesystem.File("bundled/openrouter.yaml")}}}
|
||||||
|
}
|
||||||
60
internal/configuration/bundled/openrouter.yaml
Normal file
60
internal/configuration/bundled/openrouter.yaml
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
backends:
|
||||||
|
- id: openrouter
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
smoke_tests:
|
||||||
|
default_profile: gpt-luna-latest
|
||||||
|
authentication:
|
||||||
|
kind: openrouter_key
|
||||||
|
profiles:
|
||||||
|
- id: gpt-astra-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-astra-latest
|
||||||
|
- id: gpt-sol-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-sol-latest
|
||||||
|
- id: gpt-terra-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-terra-latest
|
||||||
|
- id: gpt-luna-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-luna-latest
|
||||||
|
- id: claude-fable-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-fable-latest
|
||||||
|
- id: claude-opus-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-opus-latest
|
||||||
|
- id: claude-sonnet-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-sonnet-latest
|
||||||
|
- id: claude-haiku-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-haiku-latest
|
||||||
|
- id: gemini-pro-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~google/gemini-pro-latest
|
||||||
|
- id: gemini-flash-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~google/gemini-flash-latest
|
||||||
|
- id: auto
|
||||||
|
backend: openrouter
|
||||||
|
model: openrouter/auto
|
||||||
|
- id: free
|
||||||
|
backend: openrouter
|
||||||
|
model: openrouter/free
|
||||||
|
- id: auto-low
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: low}]}
|
||||||
|
- id: auto-medium
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: medium}]}
|
||||||
|
- id: auto-high
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: high}]}
|
||||||
|
- id: auto-xhigh
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: xhigh}]}
|
||||||
|
- id: auto-max
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: max}]}
|
||||||
54
internal/configuration/bundled_test.go
Normal file
54
internal/configuration/bundled_test.go
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||||
|
state, err := Load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{BundledOpenRouterUnderlay()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backend, err := state.GetBackend("openrouter")
|
||||||
|
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||||
|
t.Fatalf("backend = %+v, %v", backend, err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
|
||||||
|
t.Fatalf("bundle inventories: backends=%v prompts=%v collections=%v", state.backendIDs, state.promptIDs, state.collections)
|
||||||
|
}
|
||||||
|
models := map[string]string{
|
||||||
|
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
|
||||||
|
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
|
||||||
|
"gemini-pro-latest": "~google/gemini-pro-latest", "gemini-flash-latest": "~google/gemini-flash-latest", "auto": "openrouter/auto", "free": "openrouter/free",
|
||||||
|
}
|
||||||
|
for _, tier := range []string{"low", "medium", "high", "xhigh", "max"} {
|
||||||
|
models["auto-"+tier] = "openrouter/auto"
|
||||||
|
}
|
||||||
|
if len(state.profileIDs) != len(models) {
|
||||||
|
t.Fatalf("profiles = %v", state.profileIDs)
|
||||||
|
}
|
||||||
|
for id, model := range models {
|
||||||
|
profile, err := state.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
||||||
|
}
|
||||||
|
want := defaults.ExecutionTargetDefault()
|
||||||
|
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
|
||||||
|
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
||||||
|
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(*profile.ResolvedTarget, want) {
|
||||||
|
t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
|
t.Fatalf("source evidence = %+v", state.result.Sources())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,9 +2,10 @@ package configuration
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/ConfigKit"
|
"go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Inspection is the read-only subset of the retained ConfigKit result used by
|
// Inspection is the read-only subset of the retained ConfigKit result used by
|
||||||
@@ -16,6 +17,15 @@ type Inspection interface {
|
|||||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||||
WriteEffective(io.Writer) error
|
WriteEffective(io.Writer) error
|
||||||
WriteExplanation(io.Writer, configkit.Path) error
|
WriteExplanation(io.Writer, configkit.Path) error
|
||||||
|
capturedResult() *configkit.Result[document]
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectionResult struct {
|
||||||
|
*configkit.Result[document]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
|
||||||
|
return inspection.Result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Inspection returns only the retained result's inspection operations.
|
// Inspection returns only the retained result's inspection operations.
|
||||||
@@ -23,7 +33,19 @@ func (state *loaded) Inspection() Inspection {
|
|||||||
if state == nil {
|
if state == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return state.result
|
return inspectionResult{Result: state.result}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Diff compares two captured ConfigKit results without reconstructing them.
|
||||||
|
func Diff(left, right Inspection) ([]configkit.Difference, error) {
|
||||||
|
var leftResult, rightResult *configkit.Result[document]
|
||||||
|
if left != nil {
|
||||||
|
leftResult = left.capturedResult()
|
||||||
|
}
|
||||||
|
if right != nil {
|
||||||
|
rightResult = right.capturedResult()
|
||||||
|
}
|
||||||
|
return configkit.Diff(leftResult, rightResult)
|
||||||
}
|
}
|
||||||
|
|
||||||
func secretProviderPath(path configkit.Path) bool {
|
func secretProviderPath(path configkit.Path) bool {
|
||||||
@@ -32,7 +54,7 @@ func secretProviderPath(path configkit.Path) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
|
parts := strings.Split(strings.TrimPrefix(pointer, "/"), "/")
|
||||||
if len(parts) < 4 || (parts[0] != "backends" && parts[0] != "profiles") || parts[2] != "extra_params" {
|
if !providerExtraParamsPath(parts) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
|
key := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(parts[len(parts)-1], "~1", "/"), "~0", "~"))
|
||||||
@@ -43,3 +65,18 @@ func secretProviderPath(path configkit.Path) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func providerExtraParamsPath(parts []string) bool {
|
||||||
|
if len(parts) >= 4 && sequenceIndex(parts[1]) &&
|
||||||
|
(parts[0] == "backends" || parts[0] == "profiles") && parts[2] == "extra_params" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return len(parts) >= 6 && parts[0] == "collections" && parts[1] != "" &&
|
||||||
|
(parts[2] == "backends" || parts[2] == "profiles") &&
|
||||||
|
sequenceIndex(parts[3]) && parts[4] == "extra_params"
|
||||||
|
}
|
||||||
|
|
||||||
|
func sequenceIndex(segment string) bool {
|
||||||
|
index, err := strconv.Atoi(segment)
|
||||||
|
return err == nil && index >= 0 && strconv.Itoa(index) == segment
|
||||||
|
}
|
||||||
|
|||||||
32
internal/configuration/inspection_test.go
Normal file
32
internal/configuration/inspection_test.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path configkit.Path
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "backend nested secret", path: configkit.Keys("backends").Index(0).Key("extra_params").Key("nested").Key("Token"), want: true},
|
||||||
|
{name: "profile nested secret", path: configkit.Keys("profiles").Index(1).Key("extra_params").Key("authorization"), want: true},
|
||||||
|
{name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends").Index(2).Key("extra_params").Key("nested").Key("Token"), want: true},
|
||||||
|
{name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles").Index(0).Key("extra_params").Key("authorization"), want: true},
|
||||||
|
{name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false},
|
||||||
|
{name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false},
|
||||||
|
{name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false},
|
||||||
|
{name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false},
|
||||||
|
{name: "ordinary provider value", path: configkit.Keys("profiles").Index(0).Key("extra_params").Key("provider").Key("mode"), want: false},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
if got := secretProviderPath(test.path); got != test.want {
|
||||||
|
t.Fatalf("secretProviderPath(%q) = %t, want %t", test.path, got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
152
internal/configuration/interpretation_test.go
Normal file
152
internal/configuration/interpretation_test.go
Normal file
@@ -0,0 +1,152 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
|
||||||
|
state, err := load(mapping("padded target", map[string]any{
|
||||||
|
"backends": []any{map[string]any{
|
||||||
|
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 1,
|
||||||
|
"smoke_tests": map[string]any{
|
||||||
|
"reachability": map[string]any{"endpoint": " https://example.invalid/health "},
|
||||||
|
"authentication": map[string]any{"kind": " NONE ", "endpoint": " https://example.invalid/key "},
|
||||||
|
},
|
||||||
|
}},
|
||||||
|
"profiles": []any{map[string]any{"id": "partial", "backend": "local", "service_tier": " standard ", "api_key_env": " TOKEN "}},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backend, err := state.GetBackend("local")
|
||||||
|
if err != nil || backend.Endpoint != "https://example.invalid/v1" || backend.QueueCapacity != 1024 ||
|
||||||
|
backend.SmokeTests == nil || *backend.SmokeTests.Reachability.Endpoint != "https://example.invalid/health" ||
|
||||||
|
*backend.SmokeTests.Authentication.Kind != "none" || *backend.SmokeTests.Authentication.Endpoint != "https://example.invalid/key" {
|
||||||
|
t.Fatalf("interpreted backend = %+v, %v", backend, err)
|
||||||
|
}
|
||||||
|
profile, err := state.GetProfile(context.Background(), "partial")
|
||||||
|
if err != nil || profile.ResolvedTarget.ServiceTier != "standard" || profile.ResolvedTarget.APIKeyEnv != "TOKEN" {
|
||||||
|
t.Fatalf("interpreted profile = %+v, %v", profile, err)
|
||||||
|
}
|
||||||
|
composed := state.result.Config.Backends[0]
|
||||||
|
if *composed.Endpoint != "https://example.invalid/v1" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 ||
|
||||||
|
*composed.SmokeTests.Reachability.Endpoint != "https://example.invalid/health" ||
|
||||||
|
*composed.SmokeTests.Authentication.Kind != "none" || *composed.SmokeTests.Authentication.Endpoint != "https://example.invalid/key" ||
|
||||||
|
*state.result.Config.Profiles[0].ServiceTier != "standard" || *state.result.Config.Profiles[0].APIKeyEnv != "TOKEN" {
|
||||||
|
t.Fatalf("resolved values were not normalized: %+v", state.result.Config)
|
||||||
|
}
|
||||||
|
path := configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint")
|
||||||
|
field, found := state.result.Metadata().Lookup(path)
|
||||||
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
|
t.Fatalf("nested field origin changed: %+v", field)
|
||||||
|
}
|
||||||
|
for _, path := range []configkit.Path{
|
||||||
|
configkit.Keys("backends").Index(0).Key("endpoint"),
|
||||||
|
configkit.Keys("backends").Index(0).Key("smoke_tests").Key("reachability").Key("endpoint"),
|
||||||
|
configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("kind"),
|
||||||
|
configkit.Keys("profiles").Index(0).Key("service_tier"),
|
||||||
|
configkit.Keys("profiles").Index(0).Key("api_key_env"),
|
||||||
|
} {
|
||||||
|
field, found := state.result.Metadata().Lookup(path)
|
||||||
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
|
t.Errorf("normalization at %v: %+v", path, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
|
||||||
|
definition := map[string]any{
|
||||||
|
"id": "example", "inputs": []any{map[string]any{"name": " document "}},
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
|
"output": map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": " schema.json "},
|
||||||
|
}
|
||||||
|
state, err := load(mapping("prompt metadata", map[string]any{"prompts": []any{definition}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
selected, inspection, err := state.prompt("example")
|
||||||
|
if err != nil || selected.Validation.SchemaPath != "schema.json" || inspection.OutputContract.SchemaPath != "schema.json" || selected.Inputs[0].Name != "document" {
|
||||||
|
t.Fatalf("interpreted prompt = %+v, %+v, %v", selected, inspection, err)
|
||||||
|
}
|
||||||
|
composed := state.result.Config.Prompts[0]
|
||||||
|
if *composed.Output.SchemaPath != "schema.json" || composed.Inputs[0].Name != "document" {
|
||||||
|
t.Fatalf("prompt was not normalized: %+v", composed)
|
||||||
|
}
|
||||||
|
definition["inputs"] = []any{map[string]any{"name": " document "}, map[string]any{"name": "document"}}
|
||||||
|
_, err = load(mapping("duplicate inputs", map[string]any{"prompts": []any{definition}}), nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "/prompts/0/inputs/1") || !strings.Contains(err.Error(), "duplicated") {
|
||||||
|
t.Fatalf("duplicate normalized names were not rejected: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptResolutionRecordsMessageFields(t *testing.T) {
|
||||||
|
loadPrompt := func(role, cacheType, ttl string) *loaded {
|
||||||
|
t.Helper()
|
||||||
|
state, err := load(mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||||
|
"id": "example", "inputs": []any{map[string]any{"name": " input ", "content_type": nil, "description": " text "}},
|
||||||
|
"messages": []any{map[string]any{"role": role, "content": "", "cache_control": map[string]any{"type": cacheType, "ttl": ttl}}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
padded := loadPrompt(" USER ", " ephemeral ", " 1h ")
|
||||||
|
canonical := loadPrompt("user", "ephemeral", "1h")
|
||||||
|
definition, inspection, err := padded.prompt("example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
other, otherInspection, err := canonical.prompt("example")
|
||||||
|
if err != nil || inspection.PromptHash != otherInspection.PromptHash || definition.Templates[0].Role != other.Templates[0].Role || definition.Templates[0].Content != "" {
|
||||||
|
t.Fatalf("equivalent prompt mismatch: %+v %+v %v", definition, other, err)
|
||||||
|
}
|
||||||
|
base := configkit.Keys("prompts").Index(0)
|
||||||
|
for _, path := range []configkit.Path{base.Key("inputs").Index(0).Key("name"), base.Key("inputs").Index(0).Key("description"), base.Key("messages").Index(0).Key("role"), base.Key("messages").Index(0).Key("cache_control").Key("type"), base.Key("messages").Index(0).Key("cache_control").Key("ttl")} {
|
||||||
|
field, found := padded.result.Metadata().Lookup(path)
|
||||||
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
|
t.Errorf("normalization at %v: %+v", path, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if padded.promptMetadata["example"][0].contentType != fieldNull {
|
||||||
|
t.Fatal("null input metadata was not retained")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolutionRecordsTargetDependencies(t *testing.T) {
|
||||||
|
backend := func(id string) map[string]any {
|
||||||
|
return map[string]any{"id": id, "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 2, "queue_capacity": nil}
|
||||||
|
}
|
||||||
|
state, err := load(mapping("targets", map[string]any{
|
||||||
|
"backends": []any{backend("root")},
|
||||||
|
"collections": map[string]any{"scoped": map[string]any{"backends": []any{backend("local")}}},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, base := range []configkit.Path{configkit.Keys("backends").Index(0), configkit.Keys("collections", "scoped", "backends").Index(0)} {
|
||||||
|
endpoint, ok := state.result.Explain(base.Key("endpoint"))
|
||||||
|
if !ok || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") {
|
||||||
|
t.Fatalf("endpoint explanation: %+v", endpoint)
|
||||||
|
}
|
||||||
|
history := state.result.Metadata().History(base.Key("queue_capacity"))
|
||||||
|
found := false
|
||||||
|
for _, change := range history {
|
||||||
|
if change.Origin.Operation == configkit.ChangeDerivedDefault {
|
||||||
|
for _, dependency := range change.Dependencies {
|
||||||
|
if dependency.String() == base.Key("concurrency_limit").String() {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("missing queue dependency: %+v", history)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user