Compare commits
116 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4f3212bfb6 | |||
| 4faf43cc29 | |||
| c9663ec63c | |||
| d8c98ac575 | |||
| 5ee602cc4b | |||
| 96c619f7d4 | |||
| b3b5ec3820 | |||
| 5dd1418953 | |||
| f5c900cb0a | |||
| c5f3de13f3 | |||
| 88725e79ab | |||
| 6c015a17fa | |||
| 9463c9463a | |||
| 1fbd175e53 | |||
| 4b4ce43796 | |||
| d78474a3bf | |||
| acaf2ccd8b | |||
| 97e5739bfb | |||
| 90f796aef7 | |||
| 0dec3dcec0 | |||
| 448e64dc40 | |||
| de46935a0e | |||
| c69703b2b6 | |||
| f660b6e0fc | |||
| 5f56c65427 | |||
| cfa3a85308 | |||
| 1ff6d778af | |||
| 3f93763c9c | |||
| 4aeb58f913 | |||
| ac1e150168 | |||
| ff57dbf8b3 | |||
| beb18cc9d6 | |||
| 7203e5d6a1 | |||
| 356563a6e5 | |||
| 6673284312 | |||
| 6f42e1e007 | |||
| cee9bbc7b2 | |||
| 7db5572b62 | |||
| a91bc1608b | |||
| e7a3665443 | |||
| 59e8010623 | |||
| ada5377f32 | |||
| df208e132b | |||
| 8c0fb15727 | |||
| 2e2fba4993 | |||
| 68668cee56 | |||
| f745b14b71 | |||
| dbba1485a7 | |||
| 7c435032b6 | |||
| e834e7d806 | |||
| a4901cbb0e | |||
| 5dcfc2e953 | |||
| a02561d40f | |||
| 6112ee8fb4 | |||
| 0aceff8c08 | |||
| cddb2b389c | |||
| e056cb1607 | |||
| 1e122f0473 | |||
| 2f0f5fa607 | |||
| 7d500ee62f | |||
| 7be677c80b | |||
| 1c075bd510 | |||
| b8dc8a49a9 | |||
| cd373ee46b | |||
| e881bbdcf4 | |||
| 106c808442 | |||
| 4ec35f5ca6 | |||
| 6a1a32ae49 | |||
| e85ea2db35 | |||
| 3ef2a9e18e | |||
| a85a6788b4 | |||
| 21fea46fb3 | |||
| 62bf82a56b | |||
| bd3dfa992c | |||
| 425f53a62e | |||
| fcc94eb9e9 | |||
| 95730a3c11 | |||
| c542b81666 | |||
| e8e112186e | |||
| aa08d46ef6 | |||
| 668d09997a | |||
| 69203dcb12 | |||
| 3b760310a6 | |||
| b39460a507 | |||
| 5177db0611 | |||
| 31f1e6a89d | |||
| f7085d76f6 | |||
| cf0bfd4d76 | |||
| 6b1323d1fe | |||
| fb87fabc2e | |||
| 5adc1c1e64 | |||
| 0d27252999 | |||
| d9f6aad305 | |||
| fda0338134 | |||
| 2b3d165ce2 | |||
| 43f55edb98 | |||
| 4950e1dee2 | |||
| d240364776 | |||
| 6f57f2f30a | |||
| 2c50eca033 | |||
| ac44a4f325 | |||
| 9e05c3ac8b | |||
| 2c83925c88 | |||
| e0cd63923d | |||
| 75204d6134 | |||
| 5ce6e01f27 | |||
| 5cec25a40c | |||
| 855aae6daa | |||
| 43abbcb414 | |||
| 9f33fed823 | |||
| 7a98feb399 | |||
| 626134e5e1 | |||
| e7ed315671 | |||
| d7d9a7dd65 | |||
| 18f6a474e0 | |||
| fb7ebe6a6c |
@@ -26,6 +26,8 @@ source-relative prompt content, supplies an inline input, and prepares through
|
|||||||
a scoped view. The example also demonstrates collection inventories and
|
a scoped view. The example also demonstrates collection inventories and
|
||||||
namespace-filtered discovery, without contacting a provider or requiring
|
namespace-filtered discovery, without contacting a provider or requiring
|
||||||
credentials.
|
credentials.
|
||||||
|
The example disables the bundled OpenRouter configuration to keep its
|
||||||
|
inventories limited to its own offline fixtures.
|
||||||
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
The [consumer guide](docs/consumers/pkg-promptkit.md) also shows a local backend,
|
||||||
ordered file/embedded sources, programmatic mappings, and how to replace an
|
ordered file/embedded sources, programmatic mappings, and how to replace an
|
||||||
engine when source files change. Applications adapting the prior keyed
|
engine when source files change. Applications adapting the prior keyed
|
||||||
@@ -43,6 +45,11 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
||||||
|
OpenRouter backend and profiles, recorded configuration normalization, and
|
||||||
|
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
||||||
|
the Go 1.27 minimum remains unchanged.
|
||||||
|
|
||||||
The [v0.17.0 release notes](docs/releases/v0.17.0.md) cover the vanity module
|
The [v0.17.0 release notes](docs/releases/v0.17.0.md) cover the vanity module
|
||||||
path, ConfigKit v1, identified resource lists, removal of prompt-version
|
path, ConfigKit v1, identified resource lists, removal of prompt-version
|
||||||
metadata, and the Go 1.27 minimum. This is a breaking pre-v1 migration.
|
metadata, and the Go 1.27 minimum. This is a breaking pre-v1 migration.
|
||||||
|
|||||||
43
artifact_ownership_contract_test.go
Normal file
43
artifact_ownership_contract_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type independentArtifactReader struct{}
|
||||||
|
|
||||||
|
func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
return &promptkit.Artifact{Body: []byte(ref.Body), Hash: ref.Body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("ownership", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithArtifactReader(independentArtifactReader{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var group sync.WaitGroup
|
||||||
|
for i := range 24 {
|
||||||
|
group.Go(func() {
|
||||||
|
value := fmt.Sprintf("body-%d", i)
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"document": promptkit.Inline(value)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("prepare: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != value || prepared.InputHashes["document"] != value {
|
||||||
|
t.Errorf("prepared = %+v", prepared)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
}
|
||||||
@@ -78,7 +78,8 @@ type AuthenticationProbeConfig struct {
|
|||||||
// It encodes fields without validating them; NewEngine validates the final
|
// It encodes fields without validating them; NewEngine validates the final
|
||||||
// effective configuration after all selected source layers are combined.
|
// effective configuration after all selected source layers are combined.
|
||||||
// Mapping captures a copy of supported input values at creation; unsupported
|
// Mapping captures a copy of supported input values at creation; unsupported
|
||||||
// values fail when the source is loaded.
|
// values fail when the source is loaded. ExtraParams must use ConfigKit Mapping
|
||||||
|
// input forms; callers must explicitly convert numbers or containers as needed.
|
||||||
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||||
definitions := make([]any, 0, len(backends))
|
definitions := make([]any, 0, len(backends))
|
||||||
for _, config := range backends {
|
for _, config := range backends {
|
||||||
@@ -96,7 +97,7 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
|||||||
fields["queue_capacity"] = *config.QueueCapacity
|
fields["queue_capacity"] = *config.QueueCapacity
|
||||||
}
|
}
|
||||||
if config.ExtraParams != nil {
|
if config.ExtraParams != nil {
|
||||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
fields["extra_params"] = config.ExtraParams
|
||||||
}
|
}
|
||||||
if config.SmokeTests != nil {
|
if config.SmokeTests != nil {
|
||||||
smokeTests := make(map[string]any)
|
smokeTests := make(map[string]any)
|
||||||
|
|||||||
114
bundled_composition_contract_test.go
Normal file
114
bundled_composition_contract_test.go
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
||||||
|
underlay := configkit.Input{Source: configkit.Mapping("caller underlay", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "gpt-sol-latest", "model": "underlay-model"}},
|
||||||
|
})}
|
||||||
|
underlayEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
||||||
|
if err != nil || underlayProfile.EffectiveModelParams.Model != "underlay-model" || underlayProfile.EffectiveModelParams.BackendID != "" {
|
||||||
|
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
||||||
|
}
|
||||||
|
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}, map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "gpt-sol-latest", "model": "main-model"},
|
||||||
|
map[string]any{"id": "custom", "extends": map[string]any{"id": "gpt-luna-latest"}, "model": "custom-model"},
|
||||||
|
map[string]any{"id": "auto", "backend": "other", "model": "other-model"},
|
||||||
|
},
|
||||||
|
"collections": map[string]any{"openrouter": map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://collection.invalid/v1"}},
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "backend": "openrouter/local", "model": "collection-model"}},
|
||||||
|
}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}, Main: &main}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inspect := func(id string) promptkit.ExecutionTarget {
|
||||||
|
t.Helper()
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return profile.EffectiveModelParams
|
||||||
|
}
|
||||||
|
if target := inspect("gpt-sol-latest"); target.Model != "main-model" || target.BackendID != "" {
|
||||||
|
t.Fatalf("replacement = %+v", target)
|
||||||
|
}
|
||||||
|
if target := inspect("custom"); target.Model != "custom-model" || target.BackendID != "openrouter" || target.Endpoint != "https://local.invalid/v1" {
|
||||||
|
t.Fatalf("inherited = %+v", target)
|
||||||
|
}
|
||||||
|
if target := inspect("gpt-luna-latest"); target.Endpoint != "https://local.invalid/v1" || target.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("backend replacement = %+v", target)
|
||||||
|
}
|
||||||
|
if target := inspect("openrouter/profile"); target.BackendID != "openrouter/local" || target.Model != "collection-model" {
|
||||||
|
t.Fatalf("collection = %+v", target)
|
||||||
|
}
|
||||||
|
target := inspect("auto-low")
|
||||||
|
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
||||||
|
if target.BackendID != "other" || target.Model != "other-model" || !reflect.DeepEqual(target.ExtraParams, wantPlugins) {
|
||||||
|
t.Errorf("inherited tier = %+v", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledCompositionSequenceModes(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, mode, id string
|
||||||
|
wantErr bool
|
||||||
|
hasAuto bool
|
||||||
|
}{
|
||||||
|
{"duplicate append", "append", "auto", true, false},
|
||||||
|
{"distinct append", "append", "custom", false, true},
|
||||||
|
{"replace", "replace", "custom", false, false},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
|
||||||
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"profiles"}, "mode": test.mode}}},
|
||||||
|
"profiles": []any{map[string]any{"id": test.id, "model": "custom-model"}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
|
if test.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate rejection")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil || !slices.Contains(profiles, "custom") || slices.Contains(profiles, "auto") != test.hasAuto {
|
||||||
|
t.Fatalf("profiles = %v, %v", profiles, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledCompositionRejectsStrictConflictsAndMissingBackends(t *testing.T) {
|
||||||
|
profile := configkit.Input{Source: configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "auto", "model": "caller"}}})}
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile, profile}}}}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("strict group accepted duplicate records")
|
||||||
|
}
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("replace backends", map[string]any{
|
||||||
|
"config": map[string]any{"sequences": []any{map[string]any{"destination": []any{"backends"}, "mode": "replace"}}},
|
||||||
|
"backends": []any{map[string]any{"id": "other", "endpoint": "https://other.invalid/v1"}},
|
||||||
|
})}
|
||||||
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}); err == nil {
|
||||||
|
t.Fatal("missing bundled backend was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
89
bundled_execution_contract_test.go
Normal file
89
bundled_execution_contract_test.go
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func bundledPromptConfig(client *http.Client) promptkit.Config {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||||
|
"id": "prompt", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
}}})}
|
||||||
|
return promptkit.Config{HTTPClient: client, Sources: promptkit.ConfigurationSources{Main: &input}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledProfilePreparationAndExecution(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
engine, err := promptkit.NewEngine(bundledPromptConfig(nil), promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileRequired) {
|
||||||
|
t.Fatalf("missing profile = %v", err)
|
||||||
|
}
|
||||||
|
temperature := 0.3
|
||||||
|
request := promptkit.RunRequest{PromptID: "prompt", ProfileID: "gpt-luna-latest", Execution: &promptkit.ExecutionTargetOverride{Temperature: &temperature}}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
details := prepared.Details()
|
||||||
|
if details.SelectedProfileID != "gpt-luna-latest" || details.EffectiveModelParams.Model != "~openai/gpt-luna-latest" || details.EffectiveModelParams.BackendID != "openrouter" || details.EffectiveModelParams.Temperature != temperature {
|
||||||
|
t.Fatalf("prepared target = %+v", details.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(context.Background(), request); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 || client.requests[0].Target.Model != "~openai/gpt-luna-latest" || client.requests[0].Target.BackendID != "openrouter" {
|
||||||
|
t.Fatalf("requests = %+v", client.requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
|
for _, test := range []struct{ id, model, tier string }{
|
||||||
|
{"auto-low", "openrouter/auto", "low"}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
||||||
|
} {
|
||||||
|
t.Run(test.id, func(t *testing.T) {
|
||||||
|
var body map[string]any
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
if request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||||
|
t.Errorf("authorization = %q", request.Header.Get("Authorization"))
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(bundledPromptConfig(&http.Client{Transport: transport}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", ProfileID: test.id}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if body["model"] != test.model || body["extra_params"] != nil {
|
||||||
|
t.Fatalf("request = %+v", body)
|
||||||
|
}
|
||||||
|
if test.tier == "" {
|
||||||
|
if _, exists := body["plugins"]; exists {
|
||||||
|
t.Fatalf("unexpected plugins = %+v", body)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
want := []any{map[string]any{"id": "auto-router", "cost_tier": test.tier}}
|
||||||
|
if !reflect.DeepEqual(body["plugins"], want) {
|
||||||
|
t.Fatalf("plugins = %#v, want %#v", body["plugins"], want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
117
bundled_openrouter_contract_test.go
Normal file
117
bundled_openrouter_contract_test.go
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type rejectBundleTransport struct{}
|
||||||
|
|
||||||
|
func (rejectBundleTransport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||||
|
return nil, errors.New("unexpected HTTP request")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
options []promptkit.Option
|
||||||
|
enabled bool
|
||||||
|
}{
|
||||||
|
{"default", nil, true}, {"enabled", []promptkit.Option{promptkit.WithBundledOpenRouter(true)}, true},
|
||||||
|
{"disabled", []promptkit.Option{promptkit.WithBundledOpenRouter(false)}, false},
|
||||||
|
{"last disables", []promptkit.Option{promptkit.WithBundledOpenRouter(true), promptkit.WithBundledOpenRouter(false)}, false},
|
||||||
|
{"last enables", []promptkit.Option{promptkit.WithBundledOpenRouter(false), promptkit.WithBundledOpenRouter(true)}, true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: rejectBundleTransport{}}}, test.options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
collections, err := engine.ListCollections(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if (len(profiles) > 0) != test.enabled || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
||||||
|
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
||||||
|
}
|
||||||
|
if test.enabled {
|
||||||
|
if !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
|
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
||||||
|
}
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), "auto")
|
||||||
|
if err != nil || profile.EffectiveModelParams.BackendID != "openrouter" {
|
||||||
|
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(fmtSources(view.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
|
t.Fatal("bundle source missing")
|
||||||
|
}
|
||||||
|
if explanation, found := view.Explain(configkit.Keys("backends").Index(0).Key("endpoint")); !found || !strings.Contains(explanation.Effective.YAML, "https://openrouter.ai/api/v1") || len(explanation.History) == 0 {
|
||||||
|
t.Fatalf("bundle explanation = %+v", explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolCount(value bool) int {
|
||||||
|
if value {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
func fmtSources(sources []configkit.SourceInfo) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, source := range sources {
|
||||||
|
b.WriteString(source.Label)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundlePreservesCallerUnderlayBackingArray(t *testing.T) {
|
||||||
|
source := configkit.Mapping("caller", map[string]any{"profiles": []any{map[string]any{"id": "caller", "model": "model"}}})
|
||||||
|
groups := make([]configkit.Underlay, 1, 2)
|
||||||
|
groups[0] = configkit.Underlay{Inputs: []configkit.Input{{Source: source}}}
|
||||||
|
before := append([]configkit.Underlay(nil), groups...)
|
||||||
|
if _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: groups}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(groups, before) || !reflect.DeepEqual(groups[:2][1], configkit.Underlay{}) {
|
||||||
|
t.Fatal("caller underlays were modified")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDisabledBundleKeepsCallerOpenRouter(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("caller", map[string]any{"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://local.invalid/v1"}}})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil || !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
|
t.Fatalf("backends = %v, %v", backends, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
181
bundled_smoke_contract_test.go
Normal file
181
bundled_smoke_contract_test.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type cancelingAuthenticationBody struct {
|
||||||
|
remaining int
|
||||||
|
cancel context.CancelFunc
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *cancelingAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n := min(len(p), b.remaining)
|
||||||
|
for i := range p[:n] {
|
||||||
|
p[i] = 'x'
|
||||||
|
}
|
||||||
|
b.remaining -= n
|
||||||
|
if b.remaining == 0 {
|
||||||
|
b.cancel()
|
||||||
|
return n, context.Canceled
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
func (b *cancelingAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestAuthenticationReadCancellationWinsOverOverflow(t *testing.T) {
|
||||||
|
for _, size := range []int{1024, (1 << 20) + 1} {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
body := &cancelingAuthenticationBody{remaining: size, cancel: cancel}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter"})
|
||||||
|
cancel()
|
||||||
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != promptkit.ExecutionFailureCanceled || report.Facts.CredentialsAccepted != nil || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || !body.closed {
|
||||||
|
t.Fatalf("size %d: report=%+v error=%v closed=%v", size, report, err, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
|
var urls []string
|
||||||
|
var models []string
|
||||||
|
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
|
urls = append(urls, request.URL.String())
|
||||||
|
body := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
|
||||||
|
if request.Method == http.MethodPost {
|
||||||
|
var payload map[string]any
|
||||||
|
if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
models = append(models, payload["model"].(string))
|
||||||
|
content := "PROMPTKIT_OK"
|
||||||
|
if _, structured := payload["response_format"]; structured {
|
||||||
|
content = `{"status":"ok"}`
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body = string(encoded)
|
||||||
|
} else if strings.HasSuffix(request.URL.Path, "/key") && request.Header.Get("Authorization") != "Bearer synthetic-key" {
|
||||||
|
t.Errorf("authentication header missing")
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, _ := engine.ListProfiles(context.Background())
|
||||||
|
inspectionBefore, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, kind := range []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput} {
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: kind, BackendID: "openrouter"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Failure != "" {
|
||||||
|
t.Fatalf("%s report = %+v, %v", kind, report, err)
|
||||||
|
}
|
||||||
|
if kind == promptkit.SmokeTestAuthentication && (report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted) {
|
||||||
|
t.Fatalf("authentication facts = %+v", report.Facts)
|
||||||
|
}
|
||||||
|
if kind == promptkit.SmokeTestTextInference || kind == promptkit.SmokeTestStructuredOutput {
|
||||||
|
if report.Target.ProfileID != "gpt-luna-latest" || report.Target.EffectiveSettings.Model != "~openai/gpt-luna-latest" {
|
||||||
|
t.Fatalf("target = %+v", report.Target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wantURLs := []string{"https://openrouter.ai/api/v1", "https://openrouter.ai/api/v1/key", "https://openrouter.ai/api/v1/chat/completions", "https://openrouter.ai/api/v1/chat/completions"}
|
||||||
|
if !reflect.DeepEqual(urls, wantURLs) || !reflect.DeepEqual(models, []string{"~openai/gpt-luna-latest", "~openai/gpt-luna-latest"}) {
|
||||||
|
t.Fatalf("urls %v, models %v", urls, models)
|
||||||
|
}
|
||||||
|
after, _ := engine.ListProfiles(context.Background())
|
||||||
|
if !reflect.DeepEqual(before, after) {
|
||||||
|
t.Fatalf("inventory changed: %v %v", before, after)
|
||||||
|
}
|
||||||
|
inspectionAfter, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
diff, err := inspectionBefore.Diff(inspectionAfter)
|
||||||
|
if err != nil || len(diff) != 0 {
|
||||||
|
t.Fatalf("inspection changed: %+v, %v", diff, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
||||||
|
replacement := configkit.Input{Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
|
||||||
|
replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||||
|
replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
replacementReport, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||||
|
if err != nil || replacementReport.Target.ProfileID != "gpt-luna-latest" || replacementReport.Target.EffectiveSettings.Model != "replacement" {
|
||||||
|
t.Fatalf("replaced default = %+v, %v", replacementReport, err)
|
||||||
|
}
|
||||||
|
replacementClient.response.Content = "unexpected"
|
||||||
|
mismatch, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
|
||||||
|
if err != nil || mismatch == nil || !mismatch.Completed {
|
||||||
|
t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
|
||||||
|
}
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
request promptkit.SmokeTestRequest
|
||||||
|
profile, model string
|
||||||
|
}{
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter"}, "custom", "custom-model"},
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt"}, "free", "openrouter/free"},
|
||||||
|
{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt", ProfileID: "gpt-luna-latest"}, "gpt-luna-latest", "replacement"},
|
||||||
|
} {
|
||||||
|
test.request.Kind = promptkit.SmokeTestTextInference
|
||||||
|
report, err := engine.SmokeTest(context.Background(), test.request)
|
||||||
|
if err != nil || report.Target.ProfileID != test.profile || report.Target.EffectiveSettings.Model != test.model {
|
||||||
|
t.Fatalf("report = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(client.requests[len(client.requests)-1].Prompt.Messages[0].Content, "private application marker") {
|
||||||
|
t.Fatal("application content used by diagnostic")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeAuthenticationResponseOverflow(t *testing.T) {
|
||||||
|
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(strings.Repeat("x", (1<<20)+1)))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter", APIKey: "private-key-marker"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Reason != promptkit.SmokeTestReasonAuthenticationResponseLimitExceeded || report.Failure != promptkit.ExecutionFailureOther || report.Facts.CredentialsAccepted != nil {
|
||||||
|
t.Fatalf("overflow = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -81,6 +81,34 @@ func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffUsesNormalizedValues(t *testing.T) {
|
||||||
|
inspect := func(endpoint string) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("backend", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": endpoint}},
|
||||||
|
})}}}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return view
|
||||||
|
}
|
||||||
|
left := inspect(" HTTPS://example.invalid/v1 ")
|
||||||
|
equal := inspect("https://example.invalid/v1")
|
||||||
|
differences, err := left.Diff(equal)
|
||||||
|
if err != nil || len(differences) != 0 {
|
||||||
|
t.Fatalf("normalized diff = %+v, %v", differences, err)
|
||||||
|
}
|
||||||
|
changed := inspect("https://other.invalid/v1")
|
||||||
|
differences, err = left.Diff(changed)
|
||||||
|
if err != nil || len(differences) != 1 || differences[0].Path.String() != "/backends" {
|
||||||
|
t.Fatalf("changed diff = %+v, %v", differences, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||||
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
"sequence": []any{"earlier", "unchanged"},
|
"sequence": []any{"earlier", "unchanged"},
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
Underlays: []configkit.Underlay{
|
Underlays: []configkit.Underlay{
|
||||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
||||||
"backends": []any{map[string]any{
|
"backends": []any{map[string]any{
|
||||||
"id": "local", "endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY",
|
||||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||||
}},
|
}},
|
||||||
"collections": map[string]any{"literal": map[string]any{
|
"collections": map[string]any{"literal": map[string]any{
|
||||||
@@ -42,7 +42,7 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
}},
|
}},
|
||||||
})}}},
|
})}}},
|
||||||
},
|
},
|
||||||
}})
|
}}, promptkit.WithBundledOpenRouter(false))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct layered engine: %v", err)
|
t.Fatalf("construct layered engine: %v", err)
|
||||||
}
|
}
|
||||||
@@ -64,6 +64,11 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
|||||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||||
}
|
}
|
||||||
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token")
|
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token")
|
||||||
|
endpointPath := configkit.Keys("backends").Index(0).Key("endpoint")
|
||||||
|
endpoint, found := view.Explain(endpointPath)
|
||||||
|
if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") {
|
||||||
|
t.Fatalf("normalized endpoint explanation = %+v", endpoint)
|
||||||
|
}
|
||||||
secret, found := view.Explain(secretPath)
|
secret, found := view.Explain(secretPath)
|
||||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||||
|
|||||||
@@ -291,7 +291,7 @@ func TestBackendSourceOwnsSmokeTestValuesAndDefersDefaultLookup(t *testing.T) {
|
|||||||
"https://backend.example/v1",
|
"https://backend.example/v1",
|
||||||
"not-yet-listed",
|
"not-yet-listed",
|
||||||
"https://probe.example/ready?check=1",
|
"https://probe.example/ready?check=1",
|
||||||
" OPENROUTER_KEY ",
|
"openrouter_key",
|
||||||
"https://probe.example/key?scope=current",
|
"https://probe.example/key?scope=current",
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(contents, want) {
|
if !strings.Contains(contents, want) {
|
||||||
|
|||||||
83
convert.go
83
convert.go
@@ -2,7 +2,6 @@ package promptkit
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
@@ -395,88 +394,10 @@ func copyAnyMap(src map[string]any) map[string]any {
|
|||||||
if src == nil {
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make(map[string]any, len(src))
|
return jsonvalue.CloneTrusted(src).(map[string]any)
|
||||||
for k, v := range src {
|
|
||||||
out[k] = copyAny(v)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyAny(value any) any {
|
func copyAny(value any) any { return jsonvalue.CloneTrusted(value) }
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
switch v := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
return copyAnyMap(v)
|
|
||||||
case []any:
|
|
||||||
out := make([]any, len(v))
|
|
||||||
for i, item := range v {
|
|
||||||
out[i] = copyAny(item)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case []string:
|
|
||||||
return copyStringSlice(v)
|
|
||||||
case []byte:
|
|
||||||
return copyBytes(v)
|
|
||||||
default:
|
|
||||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
|
||||||
if !value.IsValid() {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
switch value.Kind() {
|
|
||||||
case reflect.Interface:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
copied := copyReflectValue(value.Elem())
|
|
||||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
|
||||||
return copied
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
out.Set(copied)
|
|
||||||
return out
|
|
||||||
case reflect.Pointer:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type().Elem())
|
|
||||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
|
||||||
return out
|
|
||||||
case reflect.Map:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
|
||||||
iter := value.MapRange()
|
|
||||||
for iter.Next() {
|
|
||||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Slice:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Array:
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
default:
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyStringSlice(src []string) []string {
|
func copyStringSlice(src []string) []string {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
This breaking pre-v1 minor migration adopts Promptkit's vanity module path,
|
This breaking pre-v1 minor migration adopts Promptkit's vanity module path,
|
||||||
changes the v0.16-era keyed resources to identified lists, adopts ConfigKit v1,
|
changes the v0.16-era keyed resources to identified lists, adopts ConfigKit v1,
|
||||||
and removes prompt-version labels.
|
and removes prompt-version labels.
|
||||||
Promptkit does not install a provider catalog; applications choose configuration
|
Promptkit installs a default-on bundled OpenRouter catalog; applications choose configuration
|
||||||
inputs explicitly, and execution selects one prompt by its exact ID. This guide
|
inputs explicitly, and execution selects one prompt by its exact ID. This guide
|
||||||
explains the shape changes and decisions an application must make; it is not an
|
explains the shape changes and decisions an application must make; it is not an
|
||||||
automatic migration tool. The [framework format reference](../formats.md) owns
|
automatic migration tool. The [framework format reference](../formats.md) owns
|
||||||
@@ -134,13 +134,15 @@ Replace former maps passed to `BackendSource` in the same way, setting each
|
|||||||
`BackendConfig.ID`. Duplicates are retained for loading to reject. Resource
|
`BackendConfig.ID`. Duplicates are retained for loading to reject. Resource
|
||||||
inspection paths now address list indexes, for example
|
inspection paths now address list indexes, for example
|
||||||
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
||||||
and diffs expose composed values and provenance, while `InspectProfile` and
|
and diffs expose resolved values, recorded normalization and provenance.
|
||||||
other interpreted views expose Promptkit normalization and defaults; the views
|
`InspectProfile` additionally applies backend completion; request preparation
|
||||||
are not promised to be identical.
|
can apply further overrides, so the views are not promised to be identical.
|
||||||
|
|
||||||
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
||||||
`prompt_version` JSON members. Prompt hashes are opaque equality values and may
|
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
||||||
change when the removed field no longer participates in hashing.
|
values and may also change because redundant captured fields were removed.
|
||||||
|
Invalidate persisted caches or comparisons that span this change; hashes from
|
||||||
|
inspection and preparation of the same captured definition still agree.
|
||||||
|
|
||||||
## Identity Mapping
|
## Identity Mapping
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import "go.maximumdirect.net/promptkit"
|
|||||||
For migration from the former module path, see
|
For migration from the former module path, see
|
||||||
[Update Go Imports](configuration-migration.md#update-go-imports).
|
[Update Go Imports](configuration-migration.md#update-go-imports).
|
||||||
|
|
||||||
Configuration APIs use `go.maximumdirect.net/configkit` v1.0.0. Its types are
|
Configuration APIs use `go.maximumdirect.net/configkit` v1.1.0. Its types are
|
||||||
not interchangeable with the old ConfigKit module path.
|
not interchangeable with the old ConfigKit module path.
|
||||||
|
|
||||||
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
The maintained [offline preparation example](../../examples/go-library/prepare/main.go)
|
||||||
@@ -52,6 +52,32 @@ Create an engine with [`NewEngine`](../../engine.go). Select each directory
|
|||||||
explicitly and place its inputs in an ordered ConfigKit underlay. A group
|
explicitly and place its inputs in an ordered ConfigKit underlay. A group
|
||||||
strictly composes its files; later groups can overlay earlier ones:
|
strictly composes its files; later groups can overlay earlier ones:
|
||||||
|
|
||||||
|
The bundled OpenRouter backend and profiles are installed in a lower underlay
|
||||||
|
by default. Select a profile explicitly, for example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("application", map[string]any{
|
||||||
|
"prompts": []any{map[string]any{
|
||||||
|
"id": "my-prompt",
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "none"},
|
||||||
|
}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}})
|
||||||
|
if err != nil { return err }
|
||||||
|
result, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "my-prompt", ProfileID: "gpt-luna-latest"})
|
||||||
|
```
|
||||||
|
|
||||||
|
The backend looks up
|
||||||
|
`OPENROUTER_API_KEY` when available; credentials are not required at engine
|
||||||
|
construction. For the complete inventory and diagnostic defaults see the
|
||||||
|
[format reference](../formats.md#bundled-openrouter-configuration). Use
|
||||||
|
`promptkit.WithBundledOpenRouter(false)` for application-only inventories.
|
||||||
|
Caller records replace matching bundled records in full; use `extends` with a
|
||||||
|
distinct ID to inherit. Replacing `auto` also changes its cost-tier children's
|
||||||
|
inherited target, while their plugins remain. External collections remain
|
||||||
|
supported.
|
||||||
|
|
||||||
```go
|
```go
|
||||||
promptInputs, err := promptkit.DirectoryInputs("prompts")
|
promptInputs, err := promptkit.DirectoryInputs("prompts")
|
||||||
if err != nil { return err }
|
if err != nil { return err }
|
||||||
@@ -152,7 +178,12 @@ input := configkit.Input{Source: document}
|
|||||||
```
|
```
|
||||||
|
|
||||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
||||||
is not needed. Backend and profile definitions are supplied by the caller;
|
is not needed. Their provider maps use ConfigKit Mapping input forms: built-in
|
||||||
|
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
|
||||||
|
typed containers explicitly to an appropriate supported value, choosing numeric
|
||||||
|
precision for your application. Mapping captures supported values immediately;
|
||||||
|
unsupported or cyclic values are reported when loaded. Runtime override values
|
||||||
|
follow their separate rules. Backend and profile definitions are supplied by the caller;
|
||||||
there is no registration-only backend namespace.
|
there is no registration-only backend namespace.
|
||||||
|
|
||||||
When configuration or message files change, build and validate a new engine,
|
When configuration or message files change, build and validate a new engine,
|
||||||
@@ -211,7 +242,7 @@ sensitive application data.
|
|||||||
## Inspect The Captured Configuration
|
## Inspect The Captured Configuration
|
||||||
|
|
||||||
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
|
||||||
operator-selected report of composed configuration, source descriptions,
|
operator-selected report of resolved configuration, source descriptions,
|
||||||
diagnostics, and path history. Configuration inspection uses list indexes in
|
diagnostics, and path history. Configuration inspection uses list indexes in
|
||||||
the collection's nested path rather than qualified runtime IDs. For a profile
|
the collection's nested path rather than qualified runtime IDs. For a profile
|
||||||
at index `i` in collection `team`, construct the path with `configkit.Keys`:
|
at index `i` in collection `team`, construct the path with `configkit.Keys`:
|
||||||
@@ -230,8 +261,9 @@ if found && explanation.Effective.Redacted {
|
|||||||
the application decides whether those reports may be disclosed. Credential-
|
the application decides whether those reports may be disclosed. Credential-
|
||||||
shaped provider keys are redacted, but arbitrary prompt and provider content
|
shaped provider keys are redacted, but arbitrary prompt and provider content
|
||||||
is not necessarily safe to share. Inspection uses the engine's captured result
|
is not necessarily safe to share. Inspection uses the engine's captured result
|
||||||
and does not reopen sources or resolve credentials. It retains composed values
|
and does not reopen sources or resolve credentials. It retains effective values,
|
||||||
and provenance rather than Promptkit's normalized/defaulted interpretation.
|
including recorded field normalization and queue defaulting, alongside source
|
||||||
|
provenance.
|
||||||
`InspectProfile` instead shows backend-completed execution settings for one profile, and `Prepare` or
|
`InspectProfile` instead shows backend-completed execution settings for one profile, and `Prepare` or
|
||||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||||
and validation resources. None of those views is a substitute for the others.
|
and validation resources. None of those views is a substitute for the others.
|
||||||
@@ -263,11 +295,13 @@ for _, difference := range differences {
|
|||||||
```
|
```
|
||||||
|
|
||||||
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||||
and ConfigKit's [v1.0.0 effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md#effective-configuration-diffs)
|
and ConfigKit's [v1.1.0 effective-configuration diff guidance](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md#effective-configuration-diffs)
|
||||||
for the exact comparison contract. The operation compares only frozen effective
|
for the exact comparison contract. The operation compares only frozen effective
|
||||||
configuration values: it does not reread external files or compare their
|
configuration values: it does not reread external files or compare their
|
||||||
contents, resolved credential contents, later execution preparation, or
|
contents, resolved credential contents, later execution preparation, or
|
||||||
request-specific overrides. An empty diff is not proof of equivalent execution.
|
request-specific overrides. Inputs normalizing to the same effective value do
|
||||||
|
not create a value diff; lists are compared atomically. An empty diff is not
|
||||||
|
proof of equivalent execution.
|
||||||
Records can contain ordinary application data, so use the application's
|
Records can contain ordinary application data, so use the application's
|
||||||
disclosure policy when presenting them.
|
disclosure policy when presenting them.
|
||||||
|
|
||||||
@@ -662,9 +696,9 @@ the engine to pick up changed definitions or message files.
|
|||||||
### Choose A Profile Ownership Model
|
### Choose A Profile Ownership Model
|
||||||
|
|
||||||
Supply backend and profile definitions through the configuration sources your
|
Supply backend and profile definitions through the configuration sources your
|
||||||
application selects. PromptKit does not install provider catalogs or built-in
|
application selects, alongside the default-on bundled OpenRouter inventory.
|
||||||
profile inventories; conventional backend ID constants are names only. Define
|
Conventional backend ID constants are names only. Define or replace the model,
|
||||||
the model, endpoint, credentials lookup, and settings that your application
|
endpoint, credentials lookup, and settings that your application
|
||||||
intends to support. A profile may inherit a sibling profile by its local key in
|
intends to support. A profile may inherit a sibling profile by its local key in
|
||||||
the same root or collection list; its backend reference uses the exact
|
the same root or collection list; its backend reference uses the exact
|
||||||
configured identity. Caller-selected source layers determine the final values. The
|
configured identity. Caller-selected source layers determine the final values. The
|
||||||
@@ -685,18 +719,19 @@ if err != nil {
|
|||||||
}
|
}
|
||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
target := inspection.EffectiveModelParams
|
||||||
|
if inspection.APIKeyRequired {
|
||||||
|
// Arrange a usable direct or configured environment credential.
|
||||||
|
}
|
||||||
if target.APIKeyEnv != "" {
|
if target.APIKeyEnv != "" {
|
||||||
// This is a configured optional environment lookup source.
|
// This is the configured environment lookup source, not its value.
|
||||||
} else if inspection.APIKeyRequired {
|
|
||||||
// Arrange a direct credential before later execution.
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Use this configuration-time boundary when only the profile and its target need
|
Use this configuration-time boundary when only the profile and its target need
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
or rendering work; use prepared execution when that work must remain tied to a
|
||||||
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
later execution. A reported `APIKeyEnv` identifies a configured source, while
|
||||||
`APIKeyRequired` is the explicit local requirement. The
|
`APIKeyRequired` is the explicit requirement; both may be present. The
|
||||||
[credential format reference](../formats.md#credentials) and the method's
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
contracts.
|
contracts.
|
||||||
@@ -850,6 +885,12 @@ secret-handling policies.
|
|||||||
|
|
||||||
## Extension Interfaces
|
## Extension Interfaces
|
||||||
|
|
||||||
|
JSON Schema validation and native structured output use a self-contained root
|
||||||
|
document. Internal references are supported; references to other files or
|
||||||
|
network resources fail during preparation. `WithSchemaFile` selects one root
|
||||||
|
file, while `WithSchemaFS` and `SchemaDir` select a root source. Prepared
|
||||||
|
executions retain their captured schema even if the source later changes.
|
||||||
|
|
||||||
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||||
behavior does not fit the application. Their GoDoc defines concurrent use,
|
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||||
context handling, ownership of copied values, nil responses, and preservation
|
context handling, ownership of copied values, nil responses, and preservation
|
||||||
@@ -857,6 +898,11 @@ of collaborator errors. Implementations must honor cancellation, safely manage
|
|||||||
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||||
the application policy that motivated the injection.
|
the application policy that motivated the injection.
|
||||||
|
|
||||||
|
A successful `ArtifactReader.Read` transfers the returned artifact and mutable
|
||||||
|
body storage to Promptkit. Readers that previously reused a returned buffer
|
||||||
|
must instead return independent storage and stop mutating it after return;
|
||||||
|
private scratch buffers may still be reused when the returned bytes are copied.
|
||||||
|
|
||||||
## Handle Errors
|
## Handle Errors
|
||||||
|
|
||||||
Use `errors.Is` with the
|
Use `errors.Is` with the
|
||||||
|
|||||||
@@ -58,6 +58,14 @@ exist.
|
|||||||
|
|
||||||
## Maintainer Validation
|
## Maintainer Validation
|
||||||
|
|
||||||
|
Use the newest Go installation already available locally for ordinary
|
||||||
|
development checks. Put its `bin` directory first on `PATH` and set
|
||||||
|
`GOTOOLCHAIN=local` to keep that toolchain selected. For repository-isolated
|
||||||
|
checks, set `GOWORK=off`; consumers may use their own workspaces. The selected
|
||||||
|
validation patch does not by itself change the module's consumer minimum. See
|
||||||
|
the [release toolchain policy](policy/release.md#consumer-minimum-and-release-toolchain)
|
||||||
|
and [release procedure](release.md#prepare-the-candidate) for release selection.
|
||||||
|
|
||||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||||
successful checks for unchanged content merely to create a commit. During
|
successful checks for unchanged content merely to create a commit. During
|
||||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ This is the canonical reference for Promptkit's backend, profile, prompt, and
|
|||||||
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
schema definitions. The [Go consumer guide](consumers/pkg-promptkit.md) shows
|
||||||
how to select sources and use an engine; the
|
how to select sources and use an engine; the
|
||||||
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md)
|
||||||
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md)
|
owns outbound wire behavior. [ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md)
|
||||||
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
owns the generic YAML grammar, imports, layering, inheritance, and diagnostics.
|
||||||
This reference describes what Promptkit does with the final effective values.
|
This reference describes what Promptkit does with the final effective values.
|
||||||
|
|
||||||
@@ -14,8 +14,8 @@ A configuration document may contain top-level `backends`, `profiles`, and
|
|||||||
`prompts` lists, plus a `collections` mapping of namespaced resource groups.
|
`prompts` lists, plus a `collections` mapping of namespaced resource groups.
|
||||||
Each resource has an explicit literal `id`, not a filename-derived identity.
|
Each resource has an explicit literal `id`, not a filename-derived identity.
|
||||||
IDs and references must be nonblank and remain exact: Promptkit does not trim or
|
IDs and references must be nonblank and remain exact: Promptkit does not trim or
|
||||||
re-key them. Unknown fields are errors. Promptkit installs no provider or
|
re-key them. Unknown fields are errors. Promptkit installs a default-on bundled
|
||||||
profile catalog; callers select the definitions they need.
|
OpenRouter underlay; callers still explicitly select a profile.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
backends:
|
backends:
|
||||||
@@ -86,9 +86,10 @@ list. It is not a qualified or cross-collection reference.
|
|||||||
no label; an explicitly empty string is retained. It does not select a resource
|
no label; an explicitly empty string is retained. It does not select a resource
|
||||||
or change execution. Runtime lookups
|
or change execution. Runtime lookups
|
||||||
use the captured effective resource indexes. `InspectConfiguration` is an
|
use the captured effective resource indexes. `InspectConfiguration` is an
|
||||||
explicit operator view of composed settings and provenance. Normalization and
|
explicit operator view of resolved settings and provenance. Selected field
|
||||||
defaults appear in Promptkit's interpreted execution views, not as changes to
|
normalizations and the derived queue default are recorded in that tree, with
|
||||||
that composed tree. Source history is not a second runtime resolver, and the
|
source history retained. Request-time completion remains separate. Source
|
||||||
|
history is not a second runtime resolver, and the
|
||||||
retained effective shape is not flattened or rewritten into those indexes.
|
retained effective shape is not flattened or rewritten into those indexes.
|
||||||
|
|
||||||
For examples of explicitly selecting native collection inputs and inspecting
|
For examples of explicitly selecting native collection inputs and inspecting
|
||||||
@@ -331,9 +332,58 @@ For example, a runtime `extra_params["extends"]` key is ordinary provider
|
|||||||
data, not a configuration inheritance instruction. An explicit zero timeout
|
data, not a configuration inheritance instruction. An explicit zero timeout
|
||||||
does not disable the caller context or HTTP transport timeout.
|
does not disable the caller context or HTTP transport timeout.
|
||||||
|
|
||||||
|
### Bundled OpenRouter Configuration
|
||||||
|
|
||||||
|
The default bundle installs root backend `openrouter`, endpoint
|
||||||
|
`https://openrouter.ai/api/v1`, optional credential environment name
|
||||||
|
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
|
||||||
|
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
|
||||||
|
applies. No prompt, implicit profile selection, or tuning defaults are installed.
|
||||||
|
Disable the entire bundle with `WithBundledOpenRouter(false)`.
|
||||||
|
|
||||||
|
The root profile IDs and their models are:
|
||||||
|
|
||||||
|
| Profile | Model |
|
||||||
|
| --- | --- |
|
||||||
|
| `gpt-astra-latest` | `~openai/gpt-astra-latest` |
|
||||||
|
| `gpt-sol-latest` | `~openai/gpt-sol-latest` |
|
||||||
|
| `gpt-terra-latest` | `~openai/gpt-terra-latest` |
|
||||||
|
| `gpt-luna-latest` | `~openai/gpt-luna-latest` |
|
||||||
|
| `claude-fable-latest` | `~anthropic/claude-fable-latest` |
|
||||||
|
| `claude-opus-latest` | `~anthropic/claude-opus-latest` |
|
||||||
|
| `claude-sonnet-latest` | `~anthropic/claude-sonnet-latest` |
|
||||||
|
| `claude-haiku-latest` | `~anthropic/claude-haiku-latest` |
|
||||||
|
| `gemini-pro-latest` | `~google/gemini-pro-latest` |
|
||||||
|
| `gemini-flash-latest` | `~google/gemini-flash-latest` |
|
||||||
|
| `auto` | `openrouter/auto` |
|
||||||
|
| `free` | `openrouter/free` |
|
||||||
|
|
||||||
|
The profiles `auto-low`, `auto-medium`, `auto-high`, `auto-xhigh`, and
|
||||||
|
`auto-max` extend `auto` and add an `auto-router` plugin with the corresponding
|
||||||
|
`cost_tier`. Replacing `auto` changes their inherited backend and model while
|
||||||
|
retaining their plugins. These are root IDs even when a model contains `/`.
|
||||||
|
For example, a caller can replace `auto` as a complete record and extend another
|
||||||
|
bundled profile under a new ID:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
profiles:
|
||||||
|
- id: auto
|
||||||
|
backend: openrouter
|
||||||
|
model: openrouter/free
|
||||||
|
- id: my-luna
|
||||||
|
extends: {id: gpt-luna-latest}
|
||||||
|
timeout_seconds: 60
|
||||||
|
```
|
||||||
|
|
||||||
|
See OpenRouter's [alias examples](https://openrouter.ai/~openai/gpt-luna-latest),
|
||||||
|
[auto router and cost tiers](https://openrouter.ai/docs/guides/routing/routers/auto-router),
|
||||||
|
and [free router](https://openrouter.ai/docs/guides/routing/routers/free-router)
|
||||||
|
for provider behavior and availability.
|
||||||
|
|
||||||
### Source And Profile Precedence
|
### Source And Profile Precedence
|
||||||
|
|
||||||
`NewEngine` composes caller underlays in their given order, followed by an
|
`NewEngine` composes the bundled OpenRouter underlay first by default, then
|
||||||
|
caller underlays in their given order, followed by an
|
||||||
optional main input. Later layers overlay earlier ones. Inputs within one
|
optional main input. Later layers overlay earlier ones. Inputs within one
|
||||||
underlay group are strictly composed: conflicting contributions are errors,
|
underlay group are strictly composed: conflicting contributions are errors,
|
||||||
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
not priority signals. The directory helpers discover `.yaml` and `.yml` files recursively
|
||||||
@@ -363,16 +413,22 @@ references are exact IDs and use their qualified form when they select a
|
|||||||
collection resource. There is no Promptkit-specific depth limit or second
|
collection resource. There is no Promptkit-specific depth limit or second
|
||||||
base-chain resolver. Missing references and cycles fail at construction. For
|
base-chain resolver. Missing references and cycles fail at construction. For
|
||||||
generic inheritance and layer mechanics, consult
|
generic inheritance and layer mechanics, consult
|
||||||
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md).
|
[ConfigKit](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md).
|
||||||
|
|
||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
Directory and `fs.FS` schema paths resolve within the configured root. Each
|
||||||
referenced nested schemas resolve relative to the owning schema document.
|
schema must be self-contained: in-document references, IDs, and anchors are
|
||||||
|
supported, but references requiring another file or network resource fail.
|
||||||
|
References must also work without the source filename as a base; use fragment
|
||||||
|
references or explicit schema IDs rather than relying on that filename.
|
||||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
||||||
invalid, or unresolvable schema causes an operational validation error.
|
invalid, or unresolvable schema causes an operational validation error.
|
||||||
Valid JSON that fails the schema instead yields a failed validation result.
|
Valid JSON that fails the schema instead yields a failed validation result.
|
||||||
|
Schema documents are detached without applying the provider extra-parameter
|
||||||
|
copy limits; this does not change the schema decoder, compiler, or
|
||||||
|
self-contained-resource rules.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
|
|||||||
@@ -16,14 +16,13 @@ Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
|||||||
Before the client is called, the engine resolves framework, backend, profile,
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
and request values into one execution target. Endpoint configuration is trimmed
|
and request values into one execution target. Endpoint configuration is trimmed
|
||||||
and must be an absolute HTTP or HTTPS URL with a host and without user
|
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||||
information, a query, or a fragment. A non-empty endpoint from the target
|
information, a query, or a fragment. The target's endpoint is validated again
|
||||||
overrides the client's configured base URL. The final selected endpoint is
|
before transport.
|
||||||
validated again before transport.
|
|
||||||
|
|
||||||
The completion URL is composed through parsed URL path operations. Nested base
|
The completion URL is composed through parsed URL path operations. Nested base
|
||||||
paths are retained, repeated trailing slashes are normalized, and the result
|
paths are retained, repeated trailing slashes are normalized, and the result
|
||||||
has exactly one appended `/chat/completions` suffix. Generation fails before
|
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||||
sending when neither source supplies a valid endpoint.
|
sending when the target does not supply a valid endpoint.
|
||||||
|
|
||||||
The target's backend ID is routing metadata for prepared values, results, and
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
injected clients. The built-in client does not derive the URL from that ID and
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
@@ -70,7 +69,9 @@ and numeric `usage`; unrelated fields and account limits are ignored. This is
|
|||||||
only a minimal identifying subset of OpenRouter's documented
|
only a minimal identifying subset of OpenRouter's documented
|
||||||
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||||
not validation of the full account document. Other statuses and unrecognized
|
not validation of the full account document. Other statuses and unrecognized
|
||||||
200 responses do not establish acceptance.
|
200 responses do not establish acceptance. A 200 response exceeding the bounded
|
||||||
|
recognition allowance reports `authentication_response_limit_exceeded`, leaving
|
||||||
|
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||||
RoundTripper boundary and any response associated with it. It includes the
|
RoundTripper boundary and any response associated with it. It includes the
|
||||||
@@ -87,8 +88,7 @@ transport for HTTP-only diagnostics.
|
|||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
target's model takes precedence over the client's configured model, and one
|
target's model must be available.
|
||||||
must be available.
|
|
||||||
|
|
||||||
Each ordinary message contains its `role` and string `content`. A
|
Each ordinary message contains its `role` and string `content`. A
|
||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
@@ -115,9 +115,10 @@ The client conditionally includes:
|
|||||||
strict flag, and schema document. Plain JSON validation does not add an
|
strict flag, and schema document. Plain JSON validation does not add an
|
||||||
object-only response constraint.
|
object-only response constraint.
|
||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
Configured profile maps recursively overlay backend defaults, while a nonempty
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
runtime override replaces the complete configured map. The [format reference](../formats.md#defaults-and-overrides)
|
||||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
owns these resolution rules. The resulting effective map is merged into the
|
||||||
|
top-level body. Empty keys, values that cannot be encoded, and
|
||||||
collisions with these reserved fields are rejected before any provider call:
|
collisions with these reserved fields are rejected before any provider call:
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
|
|||||||
@@ -68,9 +68,8 @@ Preparation and validation never hold an active permit.
|
|||||||
|
|
||||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||||
returns an idempotent release function. The coordinator uses it around
|
returns an idempotent release function. The coordinator uses it around
|
||||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
individual attempts. There is one queue and one active limit per limited backend
|
||||||
operation for focused internal callers and tests, but engine assembly no
|
pool.
|
||||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
|
||||||
|
|
||||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||||
@@ -121,7 +120,7 @@ validation, bounded admission, idempotent release, context handling, and
|
|||||||
unlimited admission. The
|
unlimited admission. The
|
||||||
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
unlimited calls, passthrough behavior, and panic release.
|
unlimited calls, and panic release using test-only one-shot calls.
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
admission, lease lifetime, failure release, and shared initial/repair
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
|
|||||||
@@ -26,10 +26,9 @@ client uses its endpoint, credential metadata, generation fields, and extra
|
|||||||
parameters. `BackendID` remains routing metadata
|
parameters. `BackendID` remains routing metadata
|
||||||
for the generation boundary and is not mapped into the provider payload.
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
Construction trims and validates a nonempty configured base URL and clones any
|
Construction clones any supplied `http.Client` so Promptkit can apply its
|
||||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
timeout default without mutating the caller's client. Generation preparation
|
||||||
mutating the caller's client. An empty configured base remains valid because a
|
then:
|
||||||
resolved request target may supply the endpoint. Generation preparation then:
|
|
||||||
|
|
||||||
1. validates shared execution-setting invariants and the final selected base
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
endpoint;
|
endpoint;
|
||||||
@@ -42,6 +41,10 @@ resolved request target may supply the endpoint. Generation preparation then:
|
|||||||
the complete-request failure remains a request failure;
|
the complete-request failure remains a request failure;
|
||||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||||
|
|
||||||
|
Provider values are detached with trusted copying before encoding. Custom JSON
|
||||||
|
encoding methods and per-value encoding-failure diagnostics operate on those
|
||||||
|
private copies rather than retained prepared or diagnostic metadata.
|
||||||
|
|
||||||
Each execution of that prepared request creates a fresh HTTP request, body
|
Each execution of that prepared request creates a fresh HTTP request, body
|
||||||
reader, and attempt timeout context. It closes the response body and returns
|
reader, and attempt timeout context. It closes the response body and returns
|
||||||
the HTTP status, decoded response or error, reported token values and presence,
|
the HTTP status, decoded response or error, reported token values and presence,
|
||||||
@@ -77,7 +80,8 @@ not retain headers, credentials, raw request or response bodies, or a redirect
|
|||||||
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||||
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||||
Reachability and named authentication probes use the same configured built-in
|
Reachability and named authentication probes use the same configured built-in
|
||||||
client but discard their response bodies after local recognition. The
|
client. The OpenRouter adapter owns endpoint derivation, bounded account-response
|
||||||
|
recognition, and body closure; the smoke coordinator interprets its evidence. The
|
||||||
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||||
owns their observable HTTP semantics.
|
owns their observable HTTP semantics.
|
||||||
|
|
||||||
@@ -105,14 +109,13 @@ public ownership and redaction semantics belong to the
|
|||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid generation
|
||||||
configuration, invalid generation requests, request execution failures,
|
requests, request execution failures, non-success provider statuses, and
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
malformed successful responses. Provider response bodies are never exposed in
|
||||||
response bodies are never exposed in raw form through non-success errors.
|
raw form through non-success errors.
|
||||||
|
|
||||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
A missing or invalid final selected endpoint is an invalid generation request
|
||||||
invalid final selected endpoint is an invalid generation request and is
|
and is rejected before transport.
|
||||||
rejected before transport.
|
|
||||||
|
|
||||||
Authentication resolves a trimmed direct key before a trimmed configured
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ lifecycle instrumentation spans the facade and runner boundaries.
|
|||||||
Prepared execution starts only after its private payload is atomically claimed;
|
Prepared execution starts only after its private payload is atomically claimed;
|
||||||
the facade receives that invocation's recorder privately and emits the terminal
|
the facade receives that invocation's recorder privately and emits the terminal
|
||||||
event after public error mapping. Failed claims and preparation emit nothing.
|
event after public error mapping. Failed claims and preparation emit nothing.
|
||||||
|
The public projection retains private operation-owner metadata separately from
|
||||||
|
inspectable child causes. Terminal classification uses that owner, preserving
|
||||||
|
context precedence and generation-specific transport/provider details rather
|
||||||
|
than mistaking a collaborator's nested lookup error for the parent's operation.
|
||||||
|
|
||||||
The facade masks inherited recorder and prepared-claim state when entering
|
The facade masks inherited recorder and prepared-claim state when entering
|
||||||
preparation or execution. Delegated calls from an injected collaborator retain
|
preparation or execution. Delegated calls from an injected collaborator retain
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ contributor workflow and validation.
|
|||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
|
|||||||
@@ -42,10 +42,11 @@ Ordinary preparation selects the full prompt definition and construction-time
|
|||||||
hash. It retains its early request-ID check before direct-session
|
hash. It retains its early request-ID check before direct-session
|
||||||
normalization. Referenced message content and static template syntax were
|
normalization. Referenced message content and static template syntax were
|
||||||
captured when the engine was built.
|
captured when the engine was built.
|
||||||
|
Rendering uses those captured parses, cloning them per call to bind request
|
||||||
|
input helpers; it does not parse templates during execution.
|
||||||
|
|
||||||
`InspectPrompt` selects captured metadata through an optional repository
|
`InspectPrompt` selects captured metadata directly from the repository. It
|
||||||
capability, falling back to the full definition and existing hash resolution
|
returns owned metadata without
|
||||||
for internal repositories without it. It returns owned metadata without
|
|
||||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
rendering templates or touching profile, artifact, schema, renderer, validator,
|
||||||
admission, or model collaborators. The root
|
admission, or model collaborators. The root
|
||||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
||||||
@@ -57,8 +58,7 @@ snapshot by the selected prompt ID. The analyzer receives only
|
|||||||
those identity fields rather than an execution definition. It analyzes the
|
those identity fields rather than an execution definition. It analyzes the
|
||||||
snapshot with the supplied caller variables and normalized direct-session
|
snapshot with the supplied caller variables and normalized direct-session
|
||||||
bypass; it does not resolve target readiness or enter the preparation
|
bypass; it does not resolve target readiness or enter the preparation
|
||||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
pipeline. An unavailable captured snapshot is a prompt-load failure. The root
|
||||||
source fallback. The root
|
|
||||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||||
operation's exact contract.
|
operation's exact contract.
|
||||||
|
|
||||||
@@ -142,11 +142,10 @@ one execution-precedence and error-ordering implementation while allowing a
|
|||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations. `Prepare` discards the plan after returning its public
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
and discards it when the operation ends. Prepared execution transfers the
|
and discards it when the operation ends. Prepared execution retains one
|
||||||
completed preparation's owned message, input-hash, and copied-schema values
|
immutable, redacted metadata snapshot shared by execution and internal details.
|
||||||
into its private payload, while deep-copying mutable provider extras and
|
The facade detaches mutable public details. Claim completion or discard clears
|
||||||
creating an independent details snapshot. Claim completion or discard clears
|
execution-only payload references without modifying the retained metadata.
|
||||||
the execution payload without removing the details snapshot.
|
|
||||||
|
|
||||||
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||||
path as `Prepare` and discards the resulting internal prepared value without
|
path as `Prepare` and discards the resulting internal prepared value without
|
||||||
@@ -176,7 +175,7 @@ encoding of that session, every role and content value, and cache-control
|
|||||||
presence and values; its hexadecimal value is opaque.
|
presence and values; its hexadecimal value is opaque.
|
||||||
|
|
||||||
The configuration snapshot is read-only after engine construction. Concurrent
|
The configuration snapshot is read-only after engine construction. Concurrent
|
||||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
`Prepare` and `Run` calls use detached captured targets and keep invocation
|
||||||
state local.
|
state local.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
|
|||||||
@@ -12,13 +12,12 @@ source semantics. Internal packages are not consumer APIs.
|
|||||||
|
|
||||||
`internal/configuration` is the single adapter from ConfigKit's final
|
`internal/configuration` is the single adapter from ConfigKit's final
|
||||||
effective configuration to execution-owned definitions. It performs one
|
effective configuration to execution-owned definitions. It performs one
|
||||||
layered load, captures effective presence (including explicit null), builds
|
layered load, captures effective presence (including explicit null), records
|
||||||
detached normalized interpretations during validation, and validates all effective
|
field-level normalization and derived defaults during resolution, and validates all effective
|
||||||
root and collection backend, profile, and prompt values. Replaced lower-layer
|
root and collection backend, profile, and prompt values. Replaced lower-layer
|
||||||
domain values are not separately validated. It indexes root resources under
|
domain values are not separately validated. It indexes root resources under
|
||||||
their unqualified IDs and collection resources under their full
|
their unqualified IDs and collection resources under their full
|
||||||
`namespace/local-key` IDs while retaining ConfigKit's effective tree in its
|
`namespace/local-key` IDs while retaining ConfigKit's resolved tree. It builds an immutable backend index and presence-aware profile
|
||||||
authored shape. It builds an immutable backend index and presence-aware profile
|
|
||||||
targets, applying final backend defaults only after profile inheritance. Runtime
|
targets, applying final backend defaults only after profile inheritance. Runtime
|
||||||
lookups use those final indexes and exact stored references; they do not rewrite
|
lookups use those final indexes and exact stored references; they do not rewrite
|
||||||
or resolve resources from ConfigKit source history. The runtime/domain packages
|
or resolve resources from ConfigKit source history. The runtime/domain packages
|
||||||
@@ -27,7 +26,7 @@ do not depend on ConfigKit.
|
|||||||
ConfigKit owns source headers, imports, composition, and source evidence.
|
ConfigKit owns source headers, imports, composition, and source evidence.
|
||||||
Promptkit does not preparse contributions: it validates the final effective
|
Promptkit does not preparse contributions: it validates the final effective
|
||||||
values once, captures the indexed identities and their original resource
|
values once, captures the indexed identities and their original resource
|
||||||
contexts, and retains ConfigKit's composed result for explicit operator inspection.
|
contexts, and retains ConfigKit's resolved result for explicit operator inspection.
|
||||||
Accepted non-main source headers are consumed during composition; the main
|
Accepted non-main source headers are consumed during composition; the main
|
||||||
source declaration remains available in that inspection view. Collection
|
source declaration remains available in that inspection view. Collection
|
||||||
release labels are descriptive metadata, not selection inputs. The
|
release labels are descriptive metadata, not selection inputs. The
|
||||||
@@ -37,12 +36,12 @@ owns the exact source-header and collection contracts.
|
|||||||
The `ConfigurationInspection.Diff` facade method compares two retained results
|
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||||
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||||
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||||
once to ConfigKit v1.0.0 and returns its public difference records. Comparison
|
once to ConfigKit v1.1.0 and returns its public difference records. Comparison
|
||||||
uses only frozen effective values and does not reopen sources, resolve runtime
|
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||||
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||||
both results; reports may still contain ordinary application data. See the
|
both results; reports may still contain ordinary application data. See the
|
||||||
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
and ConfigKit's [v1.0.0 comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.0.0/docs/config.md#effective-configuration-diffs).
|
and ConfigKit's [v1.1.0 comparison contract](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.1.0/docs/config.md#effective-configuration-diffs).
|
||||||
|
|
||||||
After resource materialization and effective-profile snapshot construction, the
|
After resource materialization and effective-profile snapshot construction, the
|
||||||
adapter captures lexically ordered identity inventories for backends, profiles,
|
adapter captures lexically ordered identity inventories for backends, profiles,
|
||||||
@@ -53,7 +52,9 @@ captured values without consulting ConfigKit or copying resource definitions.
|
|||||||
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
|
||||||
files under an explicit root, provide resource context, and leave imports
|
files under an explicit root, provide resource context, and leave imports
|
||||||
disabled. Each returned slice belongs to one strict-composition group; file
|
disabled. Each returned slice belongs to one strict-composition group; file
|
||||||
order does not confer priority. Caller underlays are composed in their given
|
order does not confer priority. Engine assembly prepends a dedicated embedded
|
||||||
|
OpenRouter underlay unless disabled, without modifying caller-owned slices.
|
||||||
|
Caller underlays are composed in their given
|
||||||
order, followed by the optional main input. The
|
order, followed by the optional main input. The
|
||||||
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
[configuration tests](../../internal/configuration/load_test.go) exercise the
|
||||||
real loader and final-value handoff.
|
real loader and final-value handoff.
|
||||||
@@ -84,13 +85,14 @@ returning and retains no source capability for public inspection.
|
|||||||
|
|
||||||
Prompt materialization shares a lazy metadata snapshot across root and collection
|
Prompt materialization shares a lazy metadata snapshot across root and collection
|
||||||
resources during construction, avoiding a full metadata copy per collection.
|
resources during construction, avoiding a full metadata copy per collection.
|
||||||
|
Input metadata, message content and templates, and output contracts are captured
|
||||||
|
separately before the definition, inspection, and hash are assembled.
|
||||||
|
|
||||||
Normalization and defaults are applied to execution-owned interpretations,
|
Normalization and defaults are recorded at individual resolved field paths,
|
||||||
including message roles and cache controls. ConfigKit retains the composed
|
including message roles and cache controls. List indexes identify fields for
|
||||||
values and origins, rather than synthetic Promptkit normalization events. List
|
presence, validation, sensitivity, and resource context; full IDs select
|
||||||
indexes identify fields for presence, validation, sensitivity, and resource
|
captured runtime indexes. Unchanged content-file fields retain their origins.
|
||||||
context; full IDs select captured runtime indexes. Recording a sequence-wide
|
Backend-to-profile and request-time completion remain separate runtime work.
|
||||||
normalization would replace origins of unchanged content-file fields.
|
|
||||||
|
|
||||||
The runner selects full definitions from this index for preparation and uses
|
The runner selects full definitions from this index for preparation and uses
|
||||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||||
@@ -146,8 +148,8 @@ change rendering behavior.
|
|||||||
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||||
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
||||||
The configuration adapter validates final effective resource references and
|
The configuration adapter validates final effective resource references and
|
||||||
field values, then builds target
|
field values, then captures backend definitions and profile target snapshots
|
||||||
snapshots with backend defaults and presence/suppression state. A profile can
|
with backend defaults and presence/suppression state. A profile can
|
||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
that captured target without prompt, artifact, schema, credential lookup, or
|
||||||
provider work. Preparation applies runtime overrides and then checks target
|
provider work. Preparation applies runtime overrides and then checks target
|
||||||
@@ -216,12 +218,14 @@ instead of first making a response-sized string copy.
|
|||||||
|
|
||||||
Every preparation operation creates one operation-local validation plan. None,
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
basic, and JSON modes retain the effective output contract without source
|
basic, and JSON modes retain the effective output contract without source
|
||||||
access. JSON Schema mode loads the root document once, resolves and compiles
|
access. JSON Schema mode loads the root document once, compiles its declared
|
||||||
each transitive reference, and retains the compiled validator. Schema compiler
|
schema locations, and retains the compiled root validator. External resources
|
||||||
resources use canonical escaped file or private-scheme URLs; loaders decode
|
are rejected without file or network loading; in-document references remain
|
||||||
their paths once and enforce the configured source boundary. The
|
supported. Root resources use canonical escaped file or private-scheme URLs. The
|
||||||
provider-facing structured-output metadata uses the root document captured by
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
the same plan.
|
the same plan. Its detached schema document is not revalidated against the
|
||||||
|
runtime provider-value copier's traversal limits. Smoke inference uses a
|
||||||
|
separate JSON-only prepared plan before checking the prepared schema plan.
|
||||||
|
|
||||||
Schema preparation and execution remain synchronous. Promptkit checks
|
Schema preparation and execution remain synchronous. Promptkit checks
|
||||||
cancellation before and after source resolution, JSON decoding, compilation,
|
cancellation before and after source resolution, JSON decoding, compilation,
|
||||||
|
|||||||
@@ -26,17 +26,31 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
|||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `go.maximumdirect.net/configkit` at `v1.0.0`.
|
Selected module: `go.maximumdirect.net/configkit` at `v1.1.0`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53/COPYRIGHT) |
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9/COPYRIGHT) |
|
||||||
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53/LICENSE).
|
[LICENSE](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9/LICENSE).
|
||||||
|
|
||||||
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c88aac990b23f48a31f9046b830d4d0954f57f53).
|
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/191015e289d7f2b91abb9fdd9c4f2f5cf75b4de9).
|
||||||
|
|
||||||
|
## ReleaseKit
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/releasekit` at `v0.9.0` (maintainer tool).
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/COPYRIGHT) |
|
||||||
|
[LICENSE](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/LICENSE).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4).
|
||||||
|
|
||||||
## jsonschema
|
## jsonschema
|
||||||
|
|
||||||
|
|||||||
@@ -74,8 +74,8 @@ Successful publication needs no separate remote read-back or CI wait.
|
|||||||
|
|
||||||
For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`.
|
For optional local diagnosis, use `./scripts/releasekit.sh check VERSION`.
|
||||||
It is not a required step before publication. The upstream
|
It is not a required step before publication. The upstream
|
||||||
[candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/check.md)
|
[candidate-check guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/check.md)
|
||||||
and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/cli.md)
|
and [CLI contract](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/cli.md)
|
||||||
own shared checks and exit-status definitions.
|
own shared checks and exit-status definitions.
|
||||||
|
|
||||||
## Deliver The Hosted Release
|
## Deliver The Hosted Release
|
||||||
@@ -97,14 +97,26 @@ For a later retry, use a clean checkout of the original tag with its recorded
|
|||||||
module files and supply the token there. In a separate checkout, for example:
|
module files and supply the token there. In a separate checkout, for example:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git switch --detach vMAJOR.MINOR.PATCH
|
(
|
||||||
./scripts/releasekit.sh deliver vMAJOR.MINOR.PATCH
|
set -e
|
||||||
|
release_version=vMAJOR.MINOR.PATCH
|
||||||
|
release_tag_ref="refs/tags/$release_version"
|
||||||
|
git fetch origin "$release_tag_ref:$release_tag_ref"
|
||||||
|
release_tag_commit=$(git rev-parse --verify "$release_tag_ref^{commit}")
|
||||||
|
git switch --detach "$release_tag_commit"
|
||||||
|
release_worktree_status=$(git status --porcelain --untracked-files=all)
|
||||||
|
if [ -n "$release_worktree_status" ]; then
|
||||||
|
printf '%s\n' 'Delivery retry requires a clean tagged checkout.' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
./scripts/releasekit.sh deliver "$release_version"
|
||||||
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
Resolve uncertain hosted writes before retrying the same immutable version.
|
Resolve uncertain hosted writes before retrying the same immutable version.
|
||||||
Matching content is accepted; conflicts are not overwritten. Do not rerun
|
Matching content is accepted; conflicts are not overwritten. Do not rerun
|
||||||
`publish` or move tags to repair delivery. Follow the upstream
|
`publish` or move tags to repair delivery. Follow the upstream
|
||||||
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/delivery.md)
|
[delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/delivery.md)
|
||||||
for result interpretation and recovery. These commands apply to tags containing
|
for result interpretation and recovery. These commands apply to tags containing
|
||||||
this delivery configuration; older releases retain their original workflows.
|
this delivery configuration; older releases retain their original workflows.
|
||||||
|
|
||||||
@@ -131,13 +143,13 @@ and explicitly select the matching attempt:
|
|||||||
Discovery is local; reconciliation observes the original destination without
|
Discovery is local; reconciliation observes the original destination without
|
||||||
rerunning validation, changing tags, or delivering hosted content. Neither is
|
rerunning validation, changing tags, or delivering hosted content. Neither is
|
||||||
a routine gate after successful publication. Follow the upstream
|
a routine gate after successful publication. Follow the upstream
|
||||||
[recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/release.md#resolve-publication-failures)
|
[recovery procedure](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/release.md#resolve-publication-failures)
|
||||||
for outcomes, incomplete evidence, and older attempts without records.
|
for outcomes, incomplete evidence, and older attempts without records.
|
||||||
|
|
||||||
Records are stored under Git's common directory by default and can contain
|
Records are stored under Git's common directory by default and can contain
|
||||||
credential-bearing endpoints. Keep them private, out of source control and
|
credential-bearing endpoints. Keep them private, out of source control and
|
||||||
shared logs; retain them while needed for recovery and remove them when that
|
shared logs; retain them while needed for recovery and remove them when that
|
||||||
value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.8.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
|
value expires. See the [evidence guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/publish.md#retain-and-reconcile-publication-evidence)
|
||||||
for storage and retention details.
|
for storage and retention details.
|
||||||
|
|
||||||
Current absence does not prove a version was never published. Remove a retained
|
Current absence does not prove a version was never published. Remove a retained
|
||||||
|
|||||||
65
docs/releases/v0.18.0.md
Normal file
65
docs/releases/v0.18.0.md
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
# Promptkit v0.18.0
|
||||||
|
|
||||||
|
Configuration inspection now exposes Promptkit's recorded field normalization
|
||||||
|
and derived queue-capacity defaults through ConfigKit v1.1.0. Engines also
|
||||||
|
include a default-on OpenRouter configuration. The minimum supported Go
|
||||||
|
version remains 1.27.0.
|
||||||
|
|
||||||
|
## Bundled OpenRouter Backend And Profiles
|
||||||
|
|
||||||
|
New engines include an embedded OpenRouter backend and seventeen root profiles:
|
||||||
|
ten latest-family aliases, the auto and free routers, and five auto-router
|
||||||
|
cost-tier variants. Select a profile explicitly in a prompt or request; the
|
||||||
|
bundle does not supply an implicit execution profile or contact providers
|
||||||
|
during construction. The backend uses `OPENROUTER_API_KEY` when available under
|
||||||
|
the existing optional-credential rules.
|
||||||
|
|
||||||
|
The bundle is the lowest-priority underlay. Applications can replace its records
|
||||||
|
or derive named variants through ordinary configuration, and external collections
|
||||||
|
remain supported. Cost-tier variants inherit the effective `auto` profile, so
|
||||||
|
replacing that parent changes their inherited target while retaining plugins.
|
||||||
|
The [bundle reference](../formats.md#bundled-openrouter-configuration) owns the
|
||||||
|
inventory and composition details.
|
||||||
|
|
||||||
|
Explicit smoke diagnostics now have OpenRouter authentication metadata and an
|
||||||
|
inference fallback profile. They remain opt-in and may incur provider costs.
|
||||||
|
Latest aliases and routers can change upstream; configured slugs do not pin
|
||||||
|
the concrete model, pricing, or availability. See the
|
||||||
|
[consumer guide](../consumers/pkg-promptkit.md#construct-an-engine) for selection
|
||||||
|
and opt-out examples.
|
||||||
|
|
||||||
|
## Recorded Configuration Normalization
|
||||||
|
|
||||||
|
Inspection shows normalized backend and profile settings, prompt metadata, and
|
||||||
|
configured message roles and cache controls, with field-level resolution
|
||||||
|
history. Existing source evidence remains available, and unchanged content-file
|
||||||
|
origins continue to select their resource context.
|
||||||
|
|
||||||
|
Configuration diffs compare normalized effective values, so spelling-only
|
||||||
|
changes that normalize identically may no longer appear. Lists remain atomic in
|
||||||
|
diffs. Inspection does not include backend-to-profile completion or request
|
||||||
|
overrides; see the [configuration inspection guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
New engines install a bundled OpenRouter backend and root profiles by default.
|
||||||
|
Inventory listings and inspection sources therefore change, and references to
|
||||||
|
bundled IDs that previously failed may now resolve. Pass
|
||||||
|
`promptkit.WithBundledOpenRouter(false)` to retain application-only behavior.
|
||||||
|
Caller records can replace bundled definitions through ordinary composition;
|
||||||
|
same-ID replacements replace complete records rather than merging selected fields.
|
||||||
|
No configuration-format migration is required from v0.17.0.
|
||||||
|
Consumers that display or compare configuration inspection results should account
|
||||||
|
for normalized values and derived defaults replacing their earlier authored
|
||||||
|
representations. Request-time overrides and caller-appended messages retain
|
||||||
|
their existing processing.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.18.0
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [framework format reference](../formats.md) for current configuration
|
||||||
|
semantics. Consumers upgrading from versions before v0.17.0 should also follow
|
||||||
|
the [configuration and API migration guide](../consumers/configuration-migration.md).
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.18.0/docs/oss/attribution.md)
|
||||||
80
docs/releases/v0.19.0.md
Normal file
80
docs/releases/v0.19.0.md
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
# Promptkit v0.19.0
|
||||||
|
|
||||||
|
This release completes two rounds of audit remediation, correcting error and
|
||||||
|
diagnostic reporting and simplifying configuration, validation, and value
|
||||||
|
ownership. It includes intentional compatibility changes for custom artifact
|
||||||
|
readers, programmatic configuration sources, and schemas. The minimum supported
|
||||||
|
Go version remains 1.27.0; ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.18.0 should review these changes:
|
||||||
|
|
||||||
|
- A successful `ArtifactReader.Read` transfers ownership of the returned
|
||||||
|
artifact and its mutable body storage to Promptkit. Custom readers must stop
|
||||||
|
mutating returned values through retained aliases. Readers that reuse scratch
|
||||||
|
buffers must return independent storage. Promptkit retains its defensive
|
||||||
|
body copy, but that copy does not make concurrent buffer reuse safe. See the
|
||||||
|
[extension guidance](../consumers/pkg-promptkit.md#extension-interfaces).
|
||||||
|
- Provider maps passed through `ProfileSource` and `BackendSource` now use
|
||||||
|
ConfigKit's native Mapping input forms directly: built-in scalars, `[]any`,
|
||||||
|
and `map[string]any` recursively. Convert `json.Number` and typed containers
|
||||||
|
explicitly to supported values, choosing numeric precision appropriate to
|
||||||
|
the application. The former recursive compatibility conversion is removed.
|
||||||
|
Runtime execution overrides retain their separate accepted-value behavior.
|
||||||
|
- Schemas used for local validation and native structured output must be
|
||||||
|
self-contained. Move externally referenced definitions into the root document
|
||||||
|
and use internal references or explicit schema IDs. References requiring
|
||||||
|
another file or network resource, or relying on the source filename as a base,
|
||||||
|
fail during preparation. Promptkit does not bundle external schemas. See the
|
||||||
|
[schema reference](../formats.md#schemas).
|
||||||
|
|
||||||
|
Prepared schemas no longer inherit the depth and node limits used when copying
|
||||||
|
runtime provider values. Schemas previously rejected solely by those incidental
|
||||||
|
limits may now be accepted; decoder/compiler checks and the self-contained
|
||||||
|
schema requirement still apply.
|
||||||
|
|
||||||
|
No configuration-format migration is required from v0.18.0. After addressing
|
||||||
|
any affected integrations:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.19.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Errors And Diagnostics
|
||||||
|
|
||||||
|
Public errors and terminal observations now preserve the category of the
|
||||||
|
operation that failed even when an injected collaborator returns an error from
|
||||||
|
another Promptkit operation. Underlying causes remain available through
|
||||||
|
`errors.Is` and `errors.As`, without repeatedly duplicating nested cause trees.
|
||||||
|
|
||||||
|
Built-in generation preserves response-body read failures separately from
|
||||||
|
malformed response content, and failures during request preparation are
|
||||||
|
distinguished from dispatched attempts.
|
||||||
|
|
||||||
|
OpenRouter authentication diagnostics bound response recognition to 1 MiB.
|
||||||
|
An oversized HTTP 200 response reports
|
||||||
|
`SmokeTestReasonAuthenticationResponseLimitExceeded`, leaving credential
|
||||||
|
acceptance unknown. Cancellation during the read remains an interruption.
|
||||||
|
Consumers that enumerate diagnostic reasons should handle the new reason;
|
||||||
|
see the [diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||||
|
|
||||||
|
## Internal Simplification And Ownership
|
||||||
|
|
||||||
|
Prompt lookup and rendering now consistently use captured configuration, and
|
||||||
|
smoke syntax and schema checks share the prepared output-validation lifecycle.
|
||||||
|
Unused routing defaults, alternate private workflows, and duplicate wire
|
||||||
|
representations have been removed.
|
||||||
|
|
||||||
|
Provider-setting resolution, inspection, prepared details, and smoke reports
|
||||||
|
avoid redundant validation and copying of already accepted values. Prepared
|
||||||
|
execution shares one private redacted metadata snapshot while preserving
|
||||||
|
independent caller-visible values. Provider values are detached before custom
|
||||||
|
JSON encoders run, protecting retained metadata from encoder-side mutation.
|
||||||
|
|
||||||
|
Requirement analysis, namespace-filtered listings, and repair-feedback
|
||||||
|
construction also avoid unnecessary traversal and temporary allocation. These
|
||||||
|
changes preserve existing request isolation and execution behavior without
|
||||||
|
introducing new performance guarantees.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/docs/oss/attribution.md)
|
||||||
@@ -38,30 +38,17 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
### Field-Level Normalization With Preserved List Provenance
|
### Schema Bundling
|
||||||
|
|
||||||
Reevaluate normalizing resource fields directly in ConfigKit's effective
|
Allow consumers to author reusable schemas across multiple files and package
|
||||||
configuration after upstream ConfigKit supports recording changes within a
|
them into a single self-contained schema for local validation and provider
|
||||||
list without replacing the source attribution of unchanged fields.
|
submission. This could restore multi-file authoring convenience while retaining
|
||||||
|
the [self-contained schema requirement](../formats.md#schemas).
|
||||||
|
|
||||||
This could let configuration inspection show PromptKit-normalized values and
|
Bundling would need to preserve reference semantics, identifiers, anchors, and
|
||||||
field-level change history while preserving the origins needed to resolve
|
recursion. A complete bundle would not guarantee that a provider supports every
|
||||||
relative resources such as `content_file`. Assess whether that inspection
|
schema feature. This is a possible future enhancement; implementation is not
|
||||||
benefit justifies changing the execution-owned normalization approach.
|
currently approved.
|
||||||
|
|
||||||
Include normalization of roles and cache-control values in configured messages
|
|
||||||
in this reevaluation. This could make inspection reflect the values used for
|
|
||||||
execution and reduce separate normalization paths. Upstream support would need
|
|
||||||
to preserve untouched field origins within nested message lists as well as outer
|
|
||||||
resource lists. PromptKit would continue owning the normalization rules;
|
|
||||||
ConfigKit would record their effects and provenance. Caller-appended messages
|
|
||||||
would still need request-time normalization because they do not pass through
|
|
||||||
ConfigKit.
|
|
||||||
Consider moving these transformations only if it improves inspection consistency
|
|
||||||
and simplifies implementation without changing execution behavior.
|
|
||||||
|
|
||||||
Upstream support is the reconsideration trigger, not a dependency of the
|
|
||||||
current ConfigKit migration or a commitment to adopt the enhancement.
|
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
25
engine.go
25
engine.go
@@ -101,8 +101,9 @@ type Engine struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
// ConfigurationSources selects ordered ConfigKit underlays and an optional
|
||||||
// highest-priority main input. Each underlay's inputs strictly compose within
|
// highest-priority main input. By default a bundled OpenRouter underlay precedes
|
||||||
// their group; later underlays overlay earlier ones. A nil Main uses
|
// the caller underlays; [WithBundledOpenRouter] can disable it. Inputs within
|
||||||
|
// each underlay strictly compose; later underlays overlay earlier ones. A nil Main uses
|
||||||
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
|
||||||
// may fail construction if invalid.
|
// may fail construction if invalid.
|
||||||
type ConfigurationSources struct {
|
type ConfigurationSources struct {
|
||||||
@@ -154,6 +155,15 @@ type engineOptions struct {
|
|||||||
internal internalengine.Options
|
internal internalengine.Options
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithBundledOpenRouter enables or disables the built-in OpenRouter underlay.
|
||||||
|
// It is enabled by default; when repeated, the last option wins.
|
||||||
|
func WithBundledOpenRouter(enabled bool) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
options.internal.SetBundledOpenRouter(enabled)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||||
// [Engine.RunPrepared].
|
// [Engine.RunPrepared].
|
||||||
//
|
//
|
||||||
@@ -203,7 +213,8 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
|||||||
//
|
//
|
||||||
// Prompt schema_path values refer to the file's base name. path must name an
|
// Prompt schema_path values refer to the file's base name. path must name an
|
||||||
// existing non-directory file when NewEngine applies the option. This option
|
// existing non-directory file when NewEngine applies the option. This option
|
||||||
// replaces Config.SchemaDir and earlier schema-source options.
|
// replaces Config.SchemaDir and earlier schema-source options. The selected
|
||||||
|
// schema must be self-contained; external resources are not loaded.
|
||||||
func WithSchemaFile(path string) Option {
|
func WithSchemaFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
return options.internal.SetSchemaFile(path)
|
return options.internal.SetSchemaFile(path)
|
||||||
@@ -213,7 +224,9 @@ func WithSchemaFile(path string) Option {
|
|||||||
// NewEngine constructs an Engine from configuration and options.
|
// NewEngine constructs an Engine from configuration and options.
|
||||||
//
|
//
|
||||||
// ConfigKit composes caller-selected underlays in their given order, followed
|
// ConfigKit composes caller-selected underlays in their given order, followed
|
||||||
// by the optional main input. Construction validates the final effective
|
// by the optional main input, after the default-on bundled OpenRouter underlay.
|
||||||
|
// The bundle adds a root backend and profiles to inventories, but no prompts
|
||||||
|
// or implicit profile selection. Construction validates the final effective
|
||||||
// backend, profile, and prompt definitions and captures referenced prompt
|
// backend, profile, and prompt definitions and captures referenced prompt
|
||||||
// content. A missing prompt collection is allowed. Schema documents and
|
// content. A missing prompt collection is allowed. Schema documents and
|
||||||
// request artifacts remain runtime resources.
|
// request artifacts remain runtime resources.
|
||||||
@@ -299,8 +312,8 @@ func (e *Engine) InspectPrompt(
|
|||||||
// override. BackendID is empty for an endpoint-only profile.
|
// override. BackendID is empty for an endpoint-only profile.
|
||||||
//
|
//
|
||||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
// APIKeyEnv in the returned target is an environment-variable name, never its
|
||||||
// value. APIKeyRequired instead reports a direct credential requirement and is
|
// value. APIKeyRequired reports that a credential is required; an explicit
|
||||||
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
// profile environment source can also be present. InspectProfile neither derives
|
||||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
// absent or blank named environment variable is not an error.
|
// absent or blank named environment variable is not an error.
|
||||||
//
|
//
|
||||||
|
|||||||
34
errors.go
34
errors.go
@@ -13,10 +13,44 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// publicOperationError keeps classification separate from inspectable causes.
|
||||||
|
// Cache wording so formatting nested operations does not repeatedly rebuild it.
|
||||||
|
type publicOperationError struct {
|
||||||
|
owner error
|
||||||
|
cause error
|
||||||
|
message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *publicOperationError) Error() string { return e.message }
|
||||||
|
func (e *publicOperationError) Unwrap() error { return e.cause }
|
||||||
|
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
|
||||||
|
|
||||||
func mapPublicError(err error) error {
|
func mapPublicError(err error) error {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var operation *usecase.OperationError
|
||||||
|
if errors.As(err, &operation) && operation != nil {
|
||||||
|
owner := publicErrorFor(operation.Category)
|
||||||
|
if owner != nil {
|
||||||
|
// Project the underlying specialized error separately, retaining both
|
||||||
|
// the outer operation and the original cause for Is/As callers.
|
||||||
|
mapped := mapPublicError(operation.Cause)
|
||||||
|
if errors.Is(operation.Cause, llm.ErrTransport) {
|
||||||
|
mapped = errors.Join(ErrTransport, mapped)
|
||||||
|
}
|
||||||
|
if errors.Is(operation.Cause, llm.ErrMalformedResponse) {
|
||||||
|
mapped = errors.Join(ErrMalformedResponse, mapped)
|
||||||
|
}
|
||||||
|
// The projection already retains ordinary causes; adding the original
|
||||||
|
// again would duplicate its entire subtree at each enclosing operation.
|
||||||
|
return &publicOperationError{owner: owner, cause: mapped, message: fmt.Sprintf("%v: %v", owner, mapped)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mapUnownedPublicError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapUnownedPublicError(err error) error {
|
||||||
var internalCapacityError *usecase.CapacityError
|
var internalCapacityError *usecase.CapacityError
|
||||||
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
|
|||||||
@@ -80,3 +80,12 @@ func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
|||||||
t.Fatalf("request encoding category = %v", err)
|
t.Fatalf("request encoding category = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorRetainsOwnerWithSpecializedCapacity(t *testing.T) {
|
||||||
|
internal := &usecase.CapacityError{BackendID: "limited"}
|
||||||
|
err := mapPublicError(&usecase.OperationError{Category: usecase.ErrLLMGenerate, Cause: internal})
|
||||||
|
var capacity *CapacityError
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) || !errors.As(err, &capacity) || capacity.BackendID != "limited" || !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("mapped owned capacity = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ func run() (summary, error) {
|
|||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
|
||||||
HTTPClient: &http.Client{Transport: transport},
|
HTTPClient: &http.Client{Transport: transport},
|
||||||
}, promptkit.WithSchemaFS(assets, "schemas"))
|
}, promptkit.WithBundledOpenRouter(false), promptkit.WithSchemaFS(assets, "schemas")) // Keep the example's configuration self-contained.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
|||||||
{Inputs: []configkit.Input{support, review}},
|
{Inputs: []configkit.Input{support, review}},
|
||||||
{Inputs: []configkit.Input{override}},
|
{Inputs: []configkit.Input{override}},
|
||||||
},
|
},
|
||||||
}})
|
}}, promptkit.WithBundledOpenRouter(false)) // Keep this example's inventory caller-owned.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
|||||||
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
|
{ID: "offline-example", Endpoint: &endpoint, Model: &model},
|
||||||
})}}}},
|
})}}}},
|
||||||
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml"))},
|
Main: &configkit.Input{Source: configkit.File(filepath.Join(exampleDir, "prompt.yaml"))},
|
||||||
}}, promptkit.WithLLMClient(deterministicClient{}))
|
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(deterministicClient{})) // Use only this example's offline configuration.
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
4
go.mod
4
go.mod
@@ -4,12 +4,12 @@ go 1.27.0
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
go.maximumdirect.net/configkit v1.0.0
|
go.maximumdirect.net/configkit v1.1.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.maximumdirect.net/releasekit v0.8.0 // indirect
|
go.maximumdirect.net/releasekit v0.9.0 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
golang.org/x/mod v0.40.0 // indirect
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
|
|||||||
8
go.sum
8
go.sum
@@ -4,10 +4,10 @@ github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEV
|
|||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
|
||||||
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||||
go.maximumdirect.net/configkit v1.0.0 h1:cguQnxkaRk3ClQOZNzzhTdc4bfzbraGKv8CkoCp+HLc=
|
go.maximumdirect.net/configkit v1.1.0 h1:UzXN2ZMnEkcL0O4YSLqD9qd9M1gb/NWCHQR/H8AaQzA=
|
||||||
go.maximumdirect.net/configkit v1.0.0/go.mod h1:B6fJ5xBLBF1nuywqfLeEI4sLm29UkIcDLu8i6I6gGXE=
|
go.maximumdirect.net/configkit v1.1.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
|
||||||
go.maximumdirect.net/releasekit v0.8.0 h1:hSaHPbUI7tAhnolfbu7UMj2fvmm9FGBY+jhkygdbxR0=
|
go.maximumdirect.net/releasekit v0.9.0 h1:sK+Aoj6gS6ATU/1reocjwBekp4YETYENwkcX5ti6mrE=
|
||||||
go.maximumdirect.net/releasekit v0.8.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
|
go.maximumdirect.net/releasekit v0.9.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ package artifact
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"fmt"
|
|
||||||
"mime"
|
"mime"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -153,9 +151,12 @@ func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef,
|
|||||||
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||||
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||||
}
|
}
|
||||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(want)})
|
||||||
if artifact.Hash != wantHash {
|
if err != nil {
|
||||||
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
b.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,9 +3,7 @@ package artifact
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -284,9 +282,12 @@ func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
|||||||
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||||
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||||
}
|
}
|
||||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(tc.body)})
|
||||||
if artifact.Hash != wantHash {
|
if err != nil {
|
||||||
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
t.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,17 +27,13 @@ type Registry struct {
|
|||||||
// NewEffectiveRegistry stores already-validated effective backend definitions
|
// NewEffectiveRegistry stores already-validated effective backend definitions
|
||||||
// under their exact configuration keys. Configuration validation and
|
// under their exact configuration keys. Configuration validation and
|
||||||
// normalization precede this ownership boundary.
|
// normalization precede this ownership boundary.
|
||||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
func NewEffectiveRegistry(definitions map[string]domain.Backend) *Registry {
|
||||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||||
for id, definition := range definitions {
|
for id, definition := range definitions {
|
||||||
definition.ID = id
|
definition.ID = id
|
||||||
cloned, err := domain.CloneBackend(definition)
|
registry.backends[id] = domain.CloneBackend(definition)
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("backend %q: %w", id, err)
|
|
||||||
}
|
|
||||||
registry.backends[id] = cloned
|
|
||||||
}
|
}
|
||||||
return registry, nil
|
return registry
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBackend returns a defensive copy of the backend registered with id.
|
// GetBackend returns a defensive copy of the backend registered with id.
|
||||||
@@ -49,11 +45,7 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
}
|
}
|
||||||
cloned, err := domain.CloneBackend(definition)
|
return domain.CloneBackend(definition), nil
|
||||||
if err != nil {
|
|
||||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
|
||||||
}
|
|
||||||
return cloned, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||||
|
|||||||
@@ -22,16 +22,13 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
|||||||
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
registry := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||||
" local.v1 ": {
|
" local.v1 ": {
|
||||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
ConcurrencyLimit: 2, QueueCapacity: 3,
|
||||||
SmokeTests: smokeTests,
|
SmokeTests: smokeTests,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
params["nested"].(map[string]any)["enabled"] = false
|
params["nested"].(map[string]any)["enabled"] = false
|
||||||
defaultProfile = "mutated-profile"
|
defaultProfile = "mutated-profile"
|
||||||
reachabilityEndpoint = "https://mutated.example/ready"
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
@@ -79,10 +76,7 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
||||||
registry, err := backend.NewEffectiveRegistry(nil)
|
registry := backend.NewEffectiveRegistry(nil)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
t.Fatalf("missing backend error = %v", err)
|
t.Fatalf("missing backend error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
package capacity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
|
||||||
)
|
|
||||||
|
|
||||||
type client struct {
|
|
||||||
manager *Manager
|
|
||||||
next llm.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient wraps next with configured active-generation limits. A nil manager
|
|
||||||
// leaves next unchanged.
|
|
||||||
func NewClient(manager *Manager, next llm.Client) llm.Client {
|
|
||||||
if manager == nil {
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
return &client{
|
|
||||||
manager: manager,
|
|
||||||
next: next,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *client) Generate(
|
|
||||||
ctx context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
if c.manager.getPool(req.Target.BackendID) == nil {
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,6 @@ package capacity
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -24,6 +23,18 @@ type clientFunc func(
|
|||||||
domain.GenerateRequest,
|
domain.GenerateRequest,
|
||||||
) (*domain.GenerateResponse, error)
|
) (*domain.GenerateResponse, error)
|
||||||
|
|
||||||
|
// newTestClient exercises one acquisition around a controlled call.
|
||||||
|
func newTestClient(manager *Manager, next llm.Client) llm.Client {
|
||||||
|
return clientFunc(func(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
release, err := manager.Acquire(ctx, req.Target.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
return next.Generate(ctx, req)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (f clientFunc) Generate(
|
func (f clientFunc) Generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -167,7 +178,7 @@ func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
|
|||||||
"second": secondRelease,
|
"second": secondRelease,
|
||||||
"third": thirdRelease,
|
"third": thirdRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
first := generateAsync(client, context.Background(), "limited", "first")
|
first := generateAsync(client, context.Background(), "limited", "first")
|
||||||
if got := receiveStarted(t, next.started); got != "first" {
|
if got := receiveStarted(t, next.started); got != "first" {
|
||||||
@@ -212,7 +223,7 @@ func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
|
|||||||
releases[string(rune('a'+i))] = gate
|
releases[string(rune('a'+i))] = gate
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
results := make([]<-chan generateResult, 0, len(releases))
|
results := make([]<-chan generateResult, 0, len(releases))
|
||||||
for id := range releases {
|
for id := range releases {
|
||||||
@@ -257,7 +268,7 @@ func TestClientRemovesCanceledWaiters(t *testing.T) {
|
|||||||
"three": make(chan struct{}),
|
"three": make(chan struct{}),
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
if got := receiveStarted(t, next.started); got != "holder" {
|
if got := receiveStarted(t, next.started); got != "holder" {
|
||||||
@@ -321,7 +332,7 @@ func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
waitForActiveCount(t, manager, "limited", 1)
|
waitForActiveCount(t, manager, "limited", 1)
|
||||||
@@ -394,7 +405,7 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
"alpha": alphaRelease,
|
"alpha": alphaRelease,
|
||||||
"beta": betaRelease,
|
"beta": betaRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
||||||
beta := generateAsync(client, context.Background(), "beta", "beta")
|
beta := generateAsync(client, context.Background(), "beta", "beta")
|
||||||
@@ -423,63 +434,6 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if got := NewClient(nil, next); got != next {
|
|
||||||
t.Fatal("nil manager did not return the wrapped client unchanged")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
|
|
||||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
|
||||||
"limited": {ConcurrencyLimit: 1},
|
|
||||||
})
|
|
||||||
request := domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{
|
|
||||||
SessionID: "session",
|
|
||||||
Messages: []domain.RenderedMessage{
|
|
||||||
{Role: "user", Content: "content"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Target: domain.ExecutionTarget{
|
|
||||||
BackendID: "limited",
|
|
||||||
Model: "model",
|
|
||||||
ExtraParams: map[string]any{"key": "value"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
response := &domain.GenerateResponse{
|
|
||||||
Content: "output",
|
|
||||||
Usage: domain.TokenUsage{TotalTokens: 7},
|
|
||||||
}
|
|
||||||
collaboratorErr := errors.New("collaborator failure")
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
response *domain.GenerateResponse
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
{name: "successful response", response: response},
|
|
||||||
{name: "nil response"},
|
|
||||||
{name: "collaborator error", response: response, err: collaboratorErr},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
var captured domain.GenerateRequest
|
|
||||||
next := clientFunc(func(
|
|
||||||
_ context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
captured = req
|
|
||||||
return tc.response, tc.err
|
|
||||||
})
|
|
||||||
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
|
|
||||||
if !reflect.DeepEqual(captured, request) {
|
|
||||||
t.Fatalf("request changed: %#v", captured)
|
|
||||||
}
|
|
||||||
if gotResponse != tc.response || gotErr != tc.err {
|
|
||||||
t.Fatalf("response=(%p, %v), want (%p, %v)",
|
|
||||||
gotResponse, gotErr, tc.response, tc.err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||||
@@ -496,7 +450,7 @@ func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: "recovered"}, nil
|
return &domain.GenerateResponse{Content: "recovered"}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
request := domain.GenerateRequest{
|
request := domain.GenerateRequest{
|
||||||
Target: domain.ExecutionTarget{BackendID: "limited"},
|
Target: domain.ExecutionTarget{BackendID: "limited"},
|
||||||
}
|
}
|
||||||
|
|||||||
16
internal/configuration/bundled.go
Normal file
16
internal/configuration/bundled.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed bundled/openrouter.yaml
|
||||||
|
var bundledFiles embed.FS
|
||||||
|
|
||||||
|
// BundledOpenRouterUnderlay returns the built-in OpenRouter configuration source.
|
||||||
|
func BundledOpenRouterUnderlay() configkit.Underlay {
|
||||||
|
filesystem := configkit.NewFilesystem("Promptkit bundled OpenRouter", bundledFiles)
|
||||||
|
return configkit.Underlay{Inputs: []configkit.Input{{Source: filesystem.File("bundled/openrouter.yaml")}}}
|
||||||
|
}
|
||||||
60
internal/configuration/bundled/openrouter.yaml
Normal file
60
internal/configuration/bundled/openrouter.yaml
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
backends:
|
||||||
|
- id: openrouter
|
||||||
|
endpoint: https://openrouter.ai/api/v1
|
||||||
|
api_key_env: OPENROUTER_API_KEY
|
||||||
|
smoke_tests:
|
||||||
|
default_profile: gpt-luna-latest
|
||||||
|
authentication:
|
||||||
|
kind: openrouter_key
|
||||||
|
profiles:
|
||||||
|
- id: gpt-astra-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-astra-latest
|
||||||
|
- id: gpt-sol-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-sol-latest
|
||||||
|
- id: gpt-terra-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-terra-latest
|
||||||
|
- id: gpt-luna-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~openai/gpt-luna-latest
|
||||||
|
- id: claude-fable-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-fable-latest
|
||||||
|
- id: claude-opus-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-opus-latest
|
||||||
|
- id: claude-sonnet-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-sonnet-latest
|
||||||
|
- id: claude-haiku-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~anthropic/claude-haiku-latest
|
||||||
|
- id: gemini-pro-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~google/gemini-pro-latest
|
||||||
|
- id: gemini-flash-latest
|
||||||
|
backend: openrouter
|
||||||
|
model: ~google/gemini-flash-latest
|
||||||
|
- id: auto
|
||||||
|
backend: openrouter
|
||||||
|
model: openrouter/auto
|
||||||
|
- id: free
|
||||||
|
backend: openrouter
|
||||||
|
model: openrouter/free
|
||||||
|
- id: auto-low
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: low}]}
|
||||||
|
- id: auto-medium
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: medium}]}
|
||||||
|
- id: auto-high
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: high}]}
|
||||||
|
- id: auto-xhigh
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: xhigh}]}
|
||||||
|
- id: auto-max
|
||||||
|
extends: {id: auto}
|
||||||
|
extra_params: {plugins: [{id: auto-router, cost_tier: max}]}
|
||||||
54
internal/configuration/bundled_test.go
Normal file
54
internal/configuration/bundled_test.go
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||||
|
state, err := Load(configkit.Input{Source: configkit.NoSource()}, []configkit.Underlay{BundledOpenRouterUnderlay()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backend, err := state.GetBackend("openrouter")
|
||||||
|
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||||
|
t.Fatalf("backend = %+v, %v", backend, err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
|
||||||
|
t.Fatalf("bundle inventories: backends=%v prompts=%v collections=%v", state.backendIDs, state.promptIDs, state.collections)
|
||||||
|
}
|
||||||
|
models := map[string]string{
|
||||||
|
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
|
||||||
|
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
|
||||||
|
"gemini-pro-latest": "~google/gemini-pro-latest", "gemini-flash-latest": "~google/gemini-flash-latest", "auto": "openrouter/auto", "free": "openrouter/free",
|
||||||
|
}
|
||||||
|
for _, tier := range []string{"low", "medium", "high", "xhigh", "max"} {
|
||||||
|
models["auto-"+tier] = "openrouter/auto"
|
||||||
|
}
|
||||||
|
if len(state.profileIDs) != len(models) {
|
||||||
|
t.Fatalf("profiles = %v", state.profileIDs)
|
||||||
|
}
|
||||||
|
for id, model := range models {
|
||||||
|
profile, err := state.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
||||||
|
}
|
||||||
|
want := defaults.ExecutionTargetDefault()
|
||||||
|
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
|
||||||
|
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
||||||
|
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(*profile.ResolvedTarget, want) {
|
||||||
|
t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
|
t.Fatalf("source evidence = %+v", state.result.Sources())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestTargetInterpretationPreservesComposedNestedValues(t *testing.T) {
|
func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
|
||||||
state, err := load(mapping("padded target", map[string]any{
|
state, err := load(mapping("padded target", map[string]any{
|
||||||
"backends": []any{map[string]any{
|
"backends": []any{map[string]any{
|
||||||
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 1,
|
"id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 1,
|
||||||
@@ -33,20 +33,32 @@ func TestTargetInterpretationPreservesComposedNestedValues(t *testing.T) {
|
|||||||
t.Fatalf("interpreted profile = %+v, %v", profile, err)
|
t.Fatalf("interpreted profile = %+v, %v", profile, err)
|
||||||
}
|
}
|
||||||
composed := state.result.Config.Backends[0]
|
composed := state.result.Config.Backends[0]
|
||||||
if *composed.Endpoint != " HTTPS://example.invalid/v1 " || composed.QueueCapacity != nil ||
|
if *composed.Endpoint != "https://example.invalid/v1" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 ||
|
||||||
*composed.SmokeTests.Reachability.Endpoint != " https://example.invalid/health " ||
|
*composed.SmokeTests.Reachability.Endpoint != "https://example.invalid/health" ||
|
||||||
*composed.SmokeTests.Authentication.Kind != " NONE " || *composed.SmokeTests.Authentication.Endpoint != " https://example.invalid/key " ||
|
*composed.SmokeTests.Authentication.Kind != "none" || *composed.SmokeTests.Authentication.Endpoint != "https://example.invalid/key" ||
|
||||||
*state.result.Config.Profiles[0].ServiceTier != " standard " || *state.result.Config.Profiles[0].APIKeyEnv != " TOKEN " {
|
*state.result.Config.Profiles[0].ServiceTier != "standard" || *state.result.Config.Profiles[0].APIKeyEnv != "TOKEN" {
|
||||||
t.Fatalf("composed values were mutated: %+v", state.result.Config)
|
t.Fatalf("resolved values were not normalized: %+v", state.result.Config)
|
||||||
}
|
}
|
||||||
path := configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint")
|
path := configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("endpoint")
|
||||||
field, found := state.result.Metadata().Lookup(path)
|
field, found := state.result.Metadata().Lookup(path)
|
||||||
if !found || len(field.Origin.Locations) == 0 || field.Origin.Operation == configkit.ChangeNormalization {
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
t.Fatalf("nested field origin changed: %+v", field)
|
t.Fatalf("nested field origin changed: %+v", field)
|
||||||
}
|
}
|
||||||
|
for _, path := range []configkit.Path{
|
||||||
|
configkit.Keys("backends").Index(0).Key("endpoint"),
|
||||||
|
configkit.Keys("backends").Index(0).Key("smoke_tests").Key("reachability").Key("endpoint"),
|
||||||
|
configkit.Keys("backends").Index(0).Key("smoke_tests").Key("authentication").Key("kind"),
|
||||||
|
configkit.Keys("profiles").Index(0).Key("service_tier"),
|
||||||
|
configkit.Keys("profiles").Index(0).Key("api_key_env"),
|
||||||
|
} {
|
||||||
|
field, found := state.result.Metadata().Lookup(path)
|
||||||
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
|
t.Errorf("normalization at %v: %+v", path, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptInterpretationPreservesComposedOutputAndRejectsDuplicateNames(t *testing.T) {
|
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
|
||||||
definition := map[string]any{
|
definition := map[string]any{
|
||||||
"id": "example", "inputs": []any{map[string]any{"name": " document "}},
|
"id": "example", "inputs": []any{map[string]any{"name": " document "}},
|
||||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||||
@@ -61,8 +73,8 @@ func TestPromptInterpretationPreservesComposedOutputAndRejectsDuplicateNames(t *
|
|||||||
t.Fatalf("interpreted prompt = %+v, %+v, %v", selected, inspection, err)
|
t.Fatalf("interpreted prompt = %+v, %+v, %v", selected, inspection, err)
|
||||||
}
|
}
|
||||||
composed := state.result.Config.Prompts[0]
|
composed := state.result.Config.Prompts[0]
|
||||||
if *composed.Output.SchemaPath != " schema.json " || composed.Inputs[0].Name != " document " {
|
if *composed.Output.SchemaPath != "schema.json" || composed.Inputs[0].Name != "document" {
|
||||||
t.Fatalf("composed prompt changed: %+v", composed)
|
t.Fatalf("prompt was not normalized: %+v", composed)
|
||||||
}
|
}
|
||||||
definition["inputs"] = []any{map[string]any{"name": " document "}, map[string]any{"name": "document"}}
|
definition["inputs"] = []any{map[string]any{"name": " document "}, map[string]any{"name": "document"}}
|
||||||
_, err = load(mapping("duplicate inputs", map[string]any{"prompts": []any{definition}}), nil)
|
_, err = load(mapping("duplicate inputs", map[string]any{"prompts": []any{definition}}), nil)
|
||||||
@@ -70,3 +82,71 @@ func TestPromptInterpretationPreservesComposedOutputAndRejectsDuplicateNames(t *
|
|||||||
t.Fatalf("duplicate normalized names were not rejected: %v", err)
|
t.Fatalf("duplicate normalized names were not rejected: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptResolutionRecordsMessageFields(t *testing.T) {
|
||||||
|
loadPrompt := func(role, cacheType, ttl string) *loaded {
|
||||||
|
t.Helper()
|
||||||
|
state, err := load(mapping("prompt", map[string]any{"prompts": []any{map[string]any{
|
||||||
|
"id": "example", "inputs": []any{map[string]any{"name": " input ", "content_type": nil, "description": " text "}},
|
||||||
|
"messages": []any{map[string]any{"role": role, "content": "", "cache_control": map[string]any{"type": cacheType, "ttl": ttl}}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic"},
|
||||||
|
}}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
padded := loadPrompt(" USER ", " ephemeral ", " 1h ")
|
||||||
|
canonical := loadPrompt("user", "ephemeral", "1h")
|
||||||
|
definition, inspection, err := padded.prompt("example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
other, otherInspection, err := canonical.prompt("example")
|
||||||
|
if err != nil || inspection.PromptHash != otherInspection.PromptHash || definition.Templates[0].Role != other.Templates[0].Role || definition.Templates[0].Content != "" {
|
||||||
|
t.Fatalf("equivalent prompt mismatch: %+v %+v %v", definition, other, err)
|
||||||
|
}
|
||||||
|
base := configkit.Keys("prompts").Index(0)
|
||||||
|
for _, path := range []configkit.Path{base.Key("inputs").Index(0).Key("name"), base.Key("inputs").Index(0).Key("description"), base.Key("messages").Index(0).Key("role"), base.Key("messages").Index(0).Key("cache_control").Key("type"), base.Key("messages").Index(0).Key("cache_control").Key("ttl")} {
|
||||||
|
field, found := padded.result.Metadata().Lookup(path)
|
||||||
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
|
t.Errorf("normalization at %v: %+v", path, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if padded.promptMetadata["example"][0].contentType != fieldNull {
|
||||||
|
t.Fatal("null input metadata was not retained")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolutionRecordsTargetDependencies(t *testing.T) {
|
||||||
|
backend := func(id string) map[string]any {
|
||||||
|
return map[string]any{"id": id, "endpoint": " HTTPS://example.invalid/v1 ", "concurrency_limit": 2, "queue_capacity": nil}
|
||||||
|
}
|
||||||
|
state, err := load(mapping("targets", map[string]any{
|
||||||
|
"backends": []any{backend("root")},
|
||||||
|
"collections": map[string]any{"scoped": map[string]any{"backends": []any{backend("local")}}},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, base := range []configkit.Path{configkit.Keys("backends").Index(0), configkit.Keys("collections", "scoped", "backends").Index(0)} {
|
||||||
|
endpoint, ok := state.result.Explain(base.Key("endpoint"))
|
||||||
|
if !ok || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") {
|
||||||
|
t.Fatalf("endpoint explanation: %+v", endpoint)
|
||||||
|
}
|
||||||
|
history := state.result.Metadata().History(base.Key("queue_capacity"))
|
||||||
|
found := false
|
||||||
|
for _, change := range history {
|
||||||
|
if change.Origin.Operation == configkit.ChangeDerivedDefault {
|
||||||
|
for _, dependency := range change.Dependencies {
|
||||||
|
if dependency.String() == base.Key("concurrency_limit").String() {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("missing queue dependency: %+v", history)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,36 +2,66 @@ package configuration
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// IdentityFilter selects all identities, one namespace, or unqualified IDs.
|
||||||
|
type IdentityFilter struct {
|
||||||
|
All bool
|
||||||
|
Namespace string
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectIdentities[T any](ctx context.Context, entries []T, filter IdentityFilter, id func(T) string) ([]T, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if filter.All {
|
||||||
|
result := append([]T(nil), entries...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if filter.Namespace != "" {
|
||||||
|
prefix := filter.Namespace + "/"
|
||||||
|
start := sort.Search(len(entries), func(i int) bool { return id(entries[i]) >= prefix })
|
||||||
|
end := start + sort.Search(len(entries)-start, func(i int) bool { return !strings.HasPrefix(id(entries[start+i]), prefix) })
|
||||||
|
result := append([]T(nil), entries[start:end]...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
var result []T
|
||||||
|
for _, entry := range entries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !strings.Contains(id(entry), "/") {
|
||||||
|
result = append(result, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
// CollectionIdentity is the captured metadata for one final collection entry.
|
// CollectionIdentity is the captured metadata for one final collection entry.
|
||||||
type CollectionIdentity struct {
|
type CollectionIdentity struct {
|
||||||
Namespace string
|
Namespace string
|
||||||
ReleaseVersion *string
|
ReleaseVersion *string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
func (state *loaded) ListProfiles(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.profileIDs, filter, func(id string) string { return id })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
profiles := append([]string(nil), state.profileIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return profiles, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
|
func (state *loaded) ListBackends(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.backendIDs, filter, func(id string) string { return id })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
backends := append([]string(nil), state.backendIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return backends, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
@@ -72,13 +102,6 @@ func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity,
|
|||||||
return collections, nil
|
return collections, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
func (state *loaded) ListPrompts(ctx context.Context, filter IdentityFilter) ([]domain.PromptIdentity, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.promptIDs, filter, func(entry domain.PromptIdentity) string { return entry.ID })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return prompts, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -32,7 +32,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -41,7 +41,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||||
}
|
}
|
||||||
|
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -65,36 +65,36 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles = append(profiles, "caller profile")
|
profiles = append(profiles, "caller profile")
|
||||||
profiles[0] = "changed profile"
|
profiles[0] = "changed profile"
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backends[0] = "changed backend"
|
backends[0] = "changed backend"
|
||||||
|
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
prompts[0].ID = "changed prompt"
|
prompts[0].ID = "changed prompt"
|
||||||
|
|
||||||
profiles, err = state.ListProfiles(context.Background())
|
profiles, err = state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||||
}
|
}
|
||||||
backends, err = state.ListBackends(context.Background())
|
backends, err = state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
||||||
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
||||||
}
|
}
|
||||||
prompts, err = state.ListPrompts(context.Background())
|
prompts, err = state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -104,20 +104,66 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadedListsSelectNamespaces(t *testing.T) {
|
||||||
|
ids := []string{"a", "a-b/one", "a/one", "a/two", "b", "z/one"}
|
||||||
|
state := &loaded{profileIDs: append([]string(nil), ids...), backendIDs: append([]string(nil), ids...)}
|
||||||
|
for _, id := range ids {
|
||||||
|
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
|
||||||
|
}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
filter IdentityFilter
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"all", IdentityFilter{All: true}, ids},
|
||||||
|
{"unqualified", IdentityFilter{}, []string{"a", "b"}},
|
||||||
|
{"namespace", IdentityFilter{Namespace: "a"}, []string{"a/one", "a/two"}},
|
||||||
|
{"prefix sharing", IdentityFilter{Namespace: "a-b"}, []string{"a-b/one"}},
|
||||||
|
{"missing", IdentityFilter{Namespace: "absent"}, nil},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
profiles, err := state.ListProfiles(context.Background(), tc.filter)
|
||||||
|
if err != nil || !reflect.DeepEqual(profiles, tc.want) {
|
||||||
|
t.Fatalf("profiles = %v, %v; want %v", profiles, err, tc.want)
|
||||||
|
}
|
||||||
|
backends, err := state.ListBackends(context.Background(), tc.filter)
|
||||||
|
if err != nil || !reflect.DeepEqual(backends, tc.want) {
|
||||||
|
t.Fatalf("backends = %v, %v; want %v", backends, err, tc.want)
|
||||||
|
}
|
||||||
|
prompts, err := state.ListPrompts(context.Background(), tc.filter)
|
||||||
|
if err != nil || len(prompts) != len(tc.want) {
|
||||||
|
t.Fatalf("prompts = %v, %v; want %v", prompts, err, tc.want)
|
||||||
|
}
|
||||||
|
for i := range prompts {
|
||||||
|
if prompts[i].ID != tc.want[i] {
|
||||||
|
t.Fatalf("prompt ID = %q; want %q", prompts[i].ID, tc.want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(profiles) > 0 {
|
||||||
|
profiles[0], backends[0], prompts[0].ID = "changed", "changed", "changed"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(state.profileIDs, ids) || !reflect.DeepEqual(state.backendIDs, ids) || state.promptIDs[0].ID != ids[0] {
|
||||||
|
t.Fatal("caller mutation changed captured identities")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadedListsEmptyInventory(t *testing.T) {
|
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||||
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(profiles) != 0 {
|
if err != nil || len(profiles) != 0 {
|
||||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||||
}
|
}
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(backends) != 0 {
|
if err != nil || len(backends) != 0 {
|
||||||
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
||||||
}
|
}
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(prompts) != 0 {
|
if err != nil || len(prompts) != 0 {
|
||||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||||
}
|
}
|
||||||
@@ -134,17 +180,19 @@ func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(ctx)
|
for _, filter := range []IdentityFilter{{All: true}, {}, {Namespace: "example"}} {
|
||||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
profiles, err := state.ListProfiles(ctx, filter)
|
||||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||||
}
|
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||||
backends, err := state.ListBackends(ctx)
|
}
|
||||||
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
backends, err := state.ListBackends(ctx, filter)
|
||||||
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
||||||
}
|
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
||||||
prompts, err := state.ListPrompts(ctx)
|
}
|
||||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
prompts, err := state.ListPrompts(ctx, filter)
|
||||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||||
|
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ package configuration
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -120,8 +119,6 @@ const (
|
|||||||
|
|
||||||
type loaded struct {
|
type loaded struct {
|
||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
interpreted document
|
|
||||||
backends map[string]map[string]fieldState
|
|
||||||
profiles map[string]map[string]fieldState
|
profiles map[string]map[string]fieldState
|
||||||
promptMetadata map[string][]promptInputPresence
|
promptMetadata map[string][]promptInputPresence
|
||||||
prompts map[string]*promptEntry
|
prompts map[string]*promptEntry
|
||||||
@@ -214,7 +211,6 @@ func indexedProfiles(config *document) map[string]profileConfig {
|
|||||||
|
|
||||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
|
|
||||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
||||||
|
|
||||||
// Load captures one immutable effective configuration for internal consumers.
|
// Load captures one immutable effective configuration for internal consumers.
|
||||||
@@ -239,15 +235,18 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
Sensitive: secretProviderPath,
|
Sensitive: secretProviderPath,
|
||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
captureFields(resources.backends, func(v backendConfig) string { return v.ID }, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
|
|
||||||
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
||||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
||||||
}
|
}
|
||||||
|
for _, resources := range resourceSets(config) {
|
||||||
|
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||||
|
resolveProfiles(resources.profiles, resources.bases.profiles, resolution)
|
||||||
|
resolvePrompts(resources.prompts, resources.bases.prompts, resolution)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
Validate: func(config *document, validation *configkit.Validation) {
|
Validate: func(config *document, validation *configkit.Validation) {
|
||||||
state.interpreted = interpretConfiguration(config, validation)
|
validateEffective(config, validation)
|
||||||
validateEffective(&state.interpreted, validation)
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -255,7 +254,6 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
state := &loaded{
|
state := &loaded{
|
||||||
backends: make(map[string]map[string]fieldState),
|
|
||||||
profiles: make(map[string]map[string]fieldState),
|
profiles: make(map[string]map[string]fieldState),
|
||||||
promptMetadata: make(map[string][]promptInputPresence),
|
promptMetadata: make(map[string][]promptInputPresence),
|
||||||
}
|
}
|
||||||
@@ -274,7 +272,7 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
|
|||||||
if err := state.buildExecutionProfiles(); err != nil {
|
if err := state.buildExecutionProfiles(); err != nil {
|
||||||
return nil, fmt.Errorf("load configuration: %w", err)
|
return nil, fmt.Errorf("load configuration: %w", err)
|
||||||
}
|
}
|
||||||
state.backendIDs = sortedKeys(indexedBackends(&state.interpreted))
|
state.backendIDs = sortedKeys(indexedBackends(&state.result.Config))
|
||||||
state.profileIDs = sortedKeys(state.indexedProfiles)
|
state.profileIDs = sortedKeys(state.indexedProfiles)
|
||||||
for _, id := range sortedKeys(state.prompts) {
|
for _, id := range sortedKeys(state.prompts) {
|
||||||
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
|
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
|
||||||
@@ -372,6 +370,29 @@ func captureFields[T any](entries []T, identity func(T) string, basePath configk
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// eligibleResourceList leaves invalid identities to the existing validation
|
||||||
|
// boundaries, without mutating a list that cannot be recorded by identity.
|
||||||
|
func eligibleResourceList[T any](entries []T, identity func(T) string) bool {
|
||||||
|
seen := make(map[string]struct{}, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
id := identity(entry)
|
||||||
|
if !domain.IsValidLocalResourceKey(id) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, exists := seen[id]; exists {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordStringNormalization(value **string, path configkit.Path, resolution *configkit.Resolution, normalized string) {
|
||||||
|
if *value != nil && **value != normalized {
|
||||||
|
configkit.Normalize(resolution, path, value, &normalized, "Normalize configured value")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func sortedKeys[T any](entries map[string]T) []string {
|
func sortedKeys[T any](entries map[string]T) []string {
|
||||||
keys := make([]string, 0, len(entries))
|
keys := make([]string, 0, len(entries))
|
||||||
for key := range entries {
|
for key := range entries {
|
||||||
@@ -381,113 +402,6 @@ func sortedKeys[T any](entries map[string]T) []string {
|
|||||||
return keys
|
return keys
|
||||||
}
|
}
|
||||||
|
|
||||||
func interpretConfiguration(config *document, validation *configkit.Validation) document {
|
|
||||||
interpreted := *config
|
|
||||||
interpreted.Backends = interpretBackends(config.Backends, configkit.Keys("backends"), validation)
|
|
||||||
interpreted.Profiles = interpretProfiles(config.Profiles, configkit.Keys("profiles"), validation)
|
|
||||||
interpreted.Prompts = interpretPrompts(config.Prompts)
|
|
||||||
if config.Collections != nil {
|
|
||||||
interpreted.Collections = make(map[string]collectionConfig, len(config.Collections))
|
|
||||||
for namespace, original := range config.Collections {
|
|
||||||
collection := original
|
|
||||||
base := configkit.Keys("collections", namespace)
|
|
||||||
collection.Backends = interpretBackends(original.Backends, base.Key("backends"), validation)
|
|
||||||
collection.Profiles = interpretProfiles(original.Profiles, base.Key("profiles"), validation)
|
|
||||||
collection.Prompts = interpretPrompts(original.Prompts)
|
|
||||||
interpreted.Collections[namespace] = collection
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return interpreted
|
|
||||||
}
|
|
||||||
|
|
||||||
func interpretBackends(source []backendConfig, basePath configkit.Path, validation *configkit.Validation) []backendConfig {
|
|
||||||
if source == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
backends := append([]backendConfig{}, source...)
|
|
||||||
for index := range backends {
|
|
||||||
backend := &backends[index]
|
|
||||||
base := basePath.Index(index)
|
|
||||||
if backend.Endpoint != nil {
|
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*backend.Endpoint)
|
|
||||||
if err != nil {
|
|
||||||
validation.Error(base.Key("endpoint"), err.Error())
|
|
||||||
} else {
|
|
||||||
backend.Endpoint = &endpoint
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if backend.APIKeyEnv != nil {
|
|
||||||
value := strings.TrimSpace(*backend.APIKeyEnv)
|
|
||||||
backend.APIKeyEnv = &value
|
|
||||||
}
|
|
||||||
if backend.SmokeTests != nil {
|
|
||||||
smokeTests := *backend.SmokeTests
|
|
||||||
backend.SmokeTests = &smokeTests
|
|
||||||
smokePath := base.Key("smoke_tests")
|
|
||||||
if smokeTests.Reachability != nil {
|
|
||||||
reachability := *smokeTests.Reachability
|
|
||||||
smokeTests.Reachability = &reachability
|
|
||||||
normalizeDiagnosticEndpoint(&reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), validation)
|
|
||||||
}
|
|
||||||
if smokeTests.Authentication != nil {
|
|
||||||
authentication := *smokeTests.Authentication
|
|
||||||
smokeTests.Authentication = &authentication
|
|
||||||
if authentication.Kind != nil {
|
|
||||||
kind := strings.ToLower(strings.TrimSpace(*authentication.Kind))
|
|
||||||
authentication.Kind = &kind
|
|
||||||
}
|
|
||||||
normalizeDiagnosticEndpoint(&authentication.Endpoint, smokePath.Key("authentication").Key("endpoint"), validation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit > 0 && backend.QueueCapacity == nil {
|
|
||||||
capacity := 1024
|
|
||||||
backend.QueueCapacity = &capacity
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return backends
|
|
||||||
}
|
|
||||||
|
|
||||||
func normalizeDiagnosticEndpoint(value **string, path configkit.Path, validation *configkit.Validation) {
|
|
||||||
if value == nil || *value == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
normalized := strings.TrimSpace(**value)
|
|
||||||
request, err := http.NewRequest(http.MethodGet, normalized, nil)
|
|
||||||
if normalized == "" || err != nil || request.URL == nil ||
|
|
||||||
(strings.ToLower(request.URL.Scheme) != "http" && strings.ToLower(request.URL.Scheme) != "https") ||
|
|
||||||
request.URL.Hostname() == "" {
|
|
||||||
validation.Error(path, "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
*value = &normalized
|
|
||||||
}
|
|
||||||
|
|
||||||
func interpretProfiles(source []profileConfig, basePath configkit.Path, validation *configkit.Validation) []profileConfig {
|
|
||||||
if source == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
profiles := append([]profileConfig{}, source...)
|
|
||||||
for index := range profiles {
|
|
||||||
profile := &profiles[index]
|
|
||||||
base := basePath.Index(index)
|
|
||||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint)
|
|
||||||
if err != nil {
|
|
||||||
validation.Error(base.Key("endpoint"), err.Error())
|
|
||||||
} else {
|
|
||||||
profile.Endpoint = &endpoint
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, value := range []**string{&profile.ServiceTier, &profile.ReasoningEffort, &profile.APIKeyEnv} {
|
|
||||||
if *value != nil {
|
|
||||||
normalized := strings.TrimSpace(**value)
|
|
||||||
*value = &normalized
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return profiles
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateEffective(config *document, validation *configkit.Validation) {
|
func validateEffective(config *document, validation *configkit.Validation) {
|
||||||
sets := resourceSets(config)
|
sets := resourceSets(config)
|
||||||
backendIDs := make(map[string]struct{})
|
backendIDs := make(map[string]struct{})
|
||||||
@@ -501,95 +415,110 @@ func validateEffective(config *document, validation *configkit.Validation) {
|
|||||||
}
|
}
|
||||||
for _, resources := range sets {
|
for _, resources := range sets {
|
||||||
for index, backend := range resources.backends {
|
for index, backend := range resources.backends {
|
||||||
base := resources.bases.backends.Index(index)
|
validateBackendRecord(backend, resources.bases.backends.Index(index), validation)
|
||||||
if !domain.IsValidLocalResourceKey(backend.ID) {
|
|
||||||
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
|
||||||
}
|
|
||||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
|
||||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
|
||||||
}
|
|
||||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
|
||||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
|
||||||
}
|
|
||||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
|
||||||
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
|
||||||
}
|
|
||||||
if backend.QueueCapacity != nil {
|
|
||||||
if *backend.QueueCapacity < 0 {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
|
||||||
}
|
|
||||||
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
|
||||||
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
|
||||||
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
|
||||||
smokePath := base.Key("smoke_tests")
|
|
||||||
if smokeTests.DefaultProfile != nil {
|
|
||||||
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
|
||||||
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
|
||||||
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
|
||||||
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if authentication := smokeTests.Authentication; authentication != nil {
|
|
||||||
kindPath := smokePath.Key("authentication").Key("kind")
|
|
||||||
if authentication.Kind == nil {
|
|
||||||
validation.Error(kindPath, "authentication kind is required")
|
|
||||||
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
|
||||||
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for index, profile := range resources.profiles {
|
for index, profile := range resources.profiles {
|
||||||
base := resources.bases.profiles.Index(index)
|
validateProfileRecord(profile, resources.bases.profiles.Index(index), backendIDs, validation)
|
||||||
if !domain.IsValidLocalResourceKey(profile.ID) {
|
|
||||||
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
|
||||||
}
|
|
||||||
if profile.Backend != nil && *profile.Backend != "" {
|
|
||||||
if !domain.IsValidResourceID(*profile.Backend) {
|
|
||||||
validation.Error(base.Key("backend"), "backend reference is malformed")
|
|
||||||
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
|
||||||
validation.Error(base.Key("backend"), "backend is not defined")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
|
||||||
validation.Error(base.Key("model"), "model must not be blank")
|
|
||||||
}
|
|
||||||
if profile.Temperature != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
|
||||||
validation.Error(base.Key("temperature"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.TopP != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
|
||||||
validation.Error(base.Key("top_p"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.MaxTokens != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
|
||||||
validation.Error(base.Key("max_tokens"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.TimeoutSeconds != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
|
||||||
validation.Error(base.Key("timeout_seconds"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
|
||||||
}
|
}
|
||||||
for index, prompt := range resources.prompts {
|
for index, prompt := range resources.prompts {
|
||||||
base := resources.bases.prompts.Index(index)
|
validatePromptRecord(prompt, resources.bases.prompts.Index(index), validation)
|
||||||
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
}
|
||||||
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
}
|
||||||
}
|
}
|
||||||
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
|
||||||
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
func validateBackendRecord(backend backendConfig, base configkit.Path, validation *configkit.Validation) {
|
||||||
|
validateBackendEndpoints(backend, base, validation)
|
||||||
|
if !domain.IsValidLocalResourceKey(backend.ID) {
|
||||||
|
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||||
|
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||||
|
}
|
||||||
|
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||||
|
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||||
|
}
|
||||||
|
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
||||||
|
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
||||||
|
}
|
||||||
|
if backend.QueueCapacity != nil {
|
||||||
|
if *backend.QueueCapacity < 0 {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
||||||
|
}
|
||||||
|
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
||||||
|
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||||
|
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||||
|
smokePath := base.Key("smoke_tests")
|
||||||
|
if smokeTests.DefaultProfile != nil {
|
||||||
|
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
||||||
|
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if authentication := smokeTests.Authentication; authentication != nil {
|
||||||
|
kindPath := smokePath.Key("authentication").Key("kind")
|
||||||
|
if authentication.Kind == nil {
|
||||||
|
validation.Error(kindPath, "authentication kind is required")
|
||||||
|
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||||
|
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateProfileRecord(profile profileConfig, base configkit.Path, backendIDs map[string]struct{}, validation *configkit.Validation) {
|
||||||
|
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||||
|
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
|
||||||
|
validation.Error(base.Key("endpoint"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !domain.IsValidLocalResourceKey(profile.ID) {
|
||||||
|
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if profile.Backend != nil && *profile.Backend != "" {
|
||||||
|
if !domain.IsValidResourceID(*profile.Backend) {
|
||||||
|
validation.Error(base.Key("backend"), "backend reference is malformed")
|
||||||
|
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
||||||
|
validation.Error(base.Key("backend"), "backend is not defined")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
||||||
|
validation.Error(base.Key("model"), "model must not be blank")
|
||||||
|
}
|
||||||
|
if profile.Temperature != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
||||||
|
validation.Error(base.Key("temperature"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.TopP != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
||||||
|
validation.Error(base.Key("top_p"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.MaxTokens != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
||||||
|
validation.Error(base.Key("max_tokens"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.TimeoutSeconds != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
||||||
|
validation.Error(base.Key("timeout_seconds"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePromptRecord(prompt promptConfig, base configkit.Path, validation *configkit.Validation) {
|
||||||
|
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
||||||
|
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
||||||
|
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -153,6 +153,50 @@ func TestIdentifiedResourcesRejectDuplicateStrictPeers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInvalidIdentitiesWithNormalizableFieldsAreRejected(t *testing.T) {
|
||||||
|
for _, kind := range []string{"backends", "profiles", "prompts"} {
|
||||||
|
for _, id := range []string{" ", "bad/id"} {
|
||||||
|
t.Run(kind+"/"+id, func(t *testing.T) {
|
||||||
|
record := map[string]any{"id": id}
|
||||||
|
switch kind {
|
||||||
|
case "backends":
|
||||||
|
record["endpoint"] = " HTTPS://example.invalid/v1 "
|
||||||
|
record["concurrency_limit"] = 1
|
||||||
|
case "profiles":
|
||||||
|
record["service_tier"] = " standard "
|
||||||
|
case "prompts":
|
||||||
|
record["description"] = " padded "
|
||||||
|
}
|
||||||
|
_, err := load(mapping("invalid", map[string]any{kind: []any{record}}), nil)
|
||||||
|
if err == nil || strings.Contains(strings.ToLower(err.Error()), "unrecorded") {
|
||||||
|
t.Fatalf("invalid identity error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDuplicateIdentitiesWithNormalizableFieldsAreRejected(t *testing.T) {
|
||||||
|
for _, kind := range []string{"backends", "profiles", "prompts"} {
|
||||||
|
t.Run(kind, func(t *testing.T) {
|
||||||
|
record := map[string]any{"id": "same"}
|
||||||
|
switch kind {
|
||||||
|
case "backends":
|
||||||
|
record["endpoint"] = " HTTPS://example.invalid/v1 "
|
||||||
|
record["concurrency_limit"] = 1
|
||||||
|
case "profiles":
|
||||||
|
record["service_tier"] = " standard "
|
||||||
|
case "prompts":
|
||||||
|
record["description"] = " padded "
|
||||||
|
}
|
||||||
|
_, err := load(mapping("duplicates", map[string]any{kind: []any{record, record}}), nil)
|
||||||
|
if err == nil || strings.Contains(strings.ToLower(err.Error()), "unrecorded") {
|
||||||
|
t.Fatalf("duplicate identity error = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
|
func TestFinalIdentitiesAreCheckedAfterApplicationResolution(t *testing.T) {
|
||||||
for _, kind := range []string{"backends", "profiles", "prompts"} {
|
for _, kind := range []string{"backends", "profiles", "prompts"} {
|
||||||
for _, namespace := range []string{"", "*"} {
|
for _, namespace := range []string{"", "*"} {
|
||||||
@@ -452,7 +496,7 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
|||||||
t.Fatalf("recursive overlay failed: %+v", provider)
|
t.Fatalf("recursive overlay failed: %+v", provider)
|
||||||
}
|
}
|
||||||
backend := loaded.result.Config.Backends[0]
|
backend := loaded.result.Config.Backends[0]
|
||||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 {
|
||||||
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
||||||
}
|
}
|
||||||
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
||||||
@@ -534,17 +578,22 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
|
|||||||
t.Fatalf("backend normalization/default was not applied: %+v", backend)
|
t.Fatalf("backend normalization/default was not applied: %+v", backend)
|
||||||
}
|
}
|
||||||
composed := loaded.result.Config.Backends[0]
|
composed := loaded.result.Config.Backends[0]
|
||||||
if composed.Endpoint == nil || *composed.Endpoint != " HTTP://localhost:8080/v1 " ||
|
if composed.Endpoint == nil || *composed.Endpoint != "http://localhost:8080/v1" ||
|
||||||
composed.APIKeyEnv == nil || *composed.APIKeyEnv != " TOKEN_NAME " || composed.QueueCapacity != nil {
|
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
|
||||||
t.Fatalf("composed backend was changed: %+v", composed)
|
t.Fatalf("resolved backend was not normalized: %+v", composed)
|
||||||
}
|
|
||||||
if loaded.backends["local"]["queue_capacity"] != fieldNull {
|
|
||||||
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
|
|
||||||
}
|
}
|
||||||
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
|
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
|
||||||
if !found || field.Origin.Operation == configkit.ChangeNormalization || len(field.Origin.Locations) == 0 {
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
t.Fatalf("endpoint origin was not preserved: %+v", field)
|
t.Fatalf("endpoint origin was not preserved: %+v", field)
|
||||||
}
|
}
|
||||||
|
field, found = loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("queue_capacity"))
|
||||||
|
if !found || field.Origin.Operation != configkit.ChangeDerivedDefault {
|
||||||
|
t.Fatalf("queue capacity default was not recorded: %+v", field)
|
||||||
|
}
|
||||||
|
field, found = loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("concurrency_limit"))
|
||||||
|
if !found || field.Origin.Operation == configkit.ChangeNormalization || field.Origin.Operation == configkit.ChangeDerivedDefault {
|
||||||
|
t.Fatalf("unchanged concurrency acquired a resolution record: %+v", field)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
|
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
|
||||||
@@ -711,7 +760,7 @@ func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backend, err := loaded.GetBackend("local")
|
backend, err := loaded.GetBackend("local")
|
||||||
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity != nil || loaded.backends["local"]["queue_capacity"] != fieldAbsent {
|
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 {
|
||||||
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,44 +28,6 @@ type promptEntry struct {
|
|||||||
messages []*template.Template
|
messages []*template.Template
|
||||||
}
|
}
|
||||||
|
|
||||||
func interpretPrompts(source []promptConfig) []promptConfig {
|
|
||||||
if source == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
prompts := append([]promptConfig{}, source...)
|
|
||||||
for index := range prompts {
|
|
||||||
definition := &prompts[index]
|
|
||||||
for _, value := range []**string{&definition.Description, &definition.SessionID} {
|
|
||||||
if *value != nil {
|
|
||||||
normalized := strings.TrimSpace(**value)
|
|
||||||
*value = &normalized
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if definition.Inputs != nil {
|
|
||||||
definition.Inputs = append([]promptInputConfig{}, definition.Inputs...)
|
|
||||||
for index := range definition.Inputs {
|
|
||||||
input := &definition.Inputs[index]
|
|
||||||
input.Name = strings.TrimSpace(input.Name)
|
|
||||||
for _, value := range []**string{&input.ContentType, &input.Description} {
|
|
||||||
if *value != nil {
|
|
||||||
normalized := strings.TrimSpace(**value)
|
|
||||||
*value = &normalized
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if definition.Output != nil {
|
|
||||||
output := *definition.Output
|
|
||||||
definition.Output = &output
|
|
||||||
if output.SchemaPath != nil {
|
|
||||||
normalized := strings.TrimSpace(*output.SchemaPath)
|
|
||||||
output.SchemaPath = &normalized
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return prompts
|
|
||||||
}
|
|
||||||
|
|
||||||
func (state *loaded) materializePrompts() error {
|
func (state *loaded) materializePrompts() error {
|
||||||
state.prompts = make(map[string]*promptEntry)
|
state.prompts = make(map[string]*promptEntry)
|
||||||
var metadata *configkit.Metadata
|
var metadata *configkit.Metadata
|
||||||
@@ -76,7 +38,7 @@ func (state *loaded) materializePrompts() error {
|
|||||||
}
|
}
|
||||||
return metadata.Lookup(path)
|
return metadata.Lookup(path)
|
||||||
}
|
}
|
||||||
for _, resources := range resourceSets(&state.interpreted) {
|
for _, resources := range resourceSets(&state.result.Config) {
|
||||||
for index, raw := range resources.prompts {
|
for index, raw := range resources.prompts {
|
||||||
promptBase := resources.bases.prompts.Index(index)
|
promptBase := resources.bases.prompts.Index(index)
|
||||||
if strings.TrimSpace(raw.ID) == "" {
|
if strings.TrimSpace(raw.ID) == "" {
|
||||||
@@ -110,107 +72,21 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
definition.SessionID = *raw.SessionID
|
definition.SessionID = *raw.SessionID
|
||||||
}
|
}
|
||||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
inputs, inputMetadata, err := state.capturePromptInputs(id, raw.Inputs, base)
|
||||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
if err != nil {
|
||||||
entry := &promptEntry{
|
return nil, err
|
||||||
messages: make([]*template.Template, 0, len(raw.Messages)),
|
|
||||||
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
|
|
||||||
}
|
}
|
||||||
for index, input := range raw.Inputs {
|
messages, compiledMessages, err := state.capturePromptMessages(raw.Messages, base, lookupOrigin)
|
||||||
path := base.Key("inputs").Index(index)
|
if err != nil {
|
||||||
if input.Name == "" {
|
return nil, err
|
||||||
return nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
|
||||||
}
|
|
||||||
if seenInputs[input.Name] {
|
|
||||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
|
||||||
}
|
|
||||||
seenInputs[input.Name] = true
|
|
||||||
contentType := ""
|
|
||||||
if input.ContentType != nil {
|
|
||||||
contentType = *input.ContentType
|
|
||||||
}
|
|
||||||
description := ""
|
|
||||||
if input.Description != nil {
|
|
||||||
description = *input.Description
|
|
||||||
}
|
|
||||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
|
||||||
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
|
||||||
})
|
|
||||||
presence := state.promptMetadata[id][index]
|
|
||||||
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
|
||||||
if presence.contentType == fieldValue {
|
|
||||||
metadata.ContentType = stringPointer(contentType)
|
|
||||||
}
|
|
||||||
if presence.description == fieldValue {
|
|
||||||
metadata.Description = stringPointer(description)
|
|
||||||
}
|
|
||||||
entry.requirementInputs = append(entry.requirementInputs, metadata)
|
|
||||||
}
|
}
|
||||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
contract, err := capturePromptOutput(raw.Output, base)
|
||||||
for index, message := range raw.Messages {
|
if err != nil {
|
||||||
path := base.Key("messages").Index(index)
|
return nil, err
|
||||||
if message.Role == nil {
|
|
||||||
return nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
|
||||||
}
|
|
||||||
role, err := domain.NormalizeMessageRole(*message.Role)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
|
||||||
}
|
|
||||||
if (message.Content != nil) == (message.ContentFile != nil) {
|
|
||||||
return nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
|
||||||
}
|
|
||||||
content := ""
|
|
||||||
if message.Content != nil {
|
|
||||||
content = *message.Content
|
|
||||||
} else {
|
|
||||||
var err error
|
|
||||||
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var control *domain.CacheControl
|
|
||||||
if message.CacheControl != nil {
|
|
||||||
if message.CacheControl.Type == nil {
|
|
||||||
return nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
|
||||||
}
|
|
||||||
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
|
||||||
if message.CacheControl.TTL != nil {
|
|
||||||
control.TTL = *message.CacheControl.TTL
|
|
||||||
}
|
|
||||||
var err error
|
|
||||||
control, err = domain.NormalizeCacheControl(control)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
|
||||||
}
|
|
||||||
entry.messages = append(entry.messages, compiled)
|
|
||||||
definition.Templates = append(definition.Templates, domain.PromptMessageTemplate{
|
|
||||||
Role: role, Content: content, CacheControl: control,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
if raw.Output == nil {
|
definition.Inputs = inputs
|
||||||
return nil, fmt.Errorf("%s: output is required", base.Key("output"))
|
definition.Templates = messages
|
||||||
}
|
entry := &promptEntry{requirementInputs: inputMetadata, messages: compiledMessages}
|
||||||
if raw.Output.Format == nil || raw.Output.ValidationMode == nil {
|
|
||||||
return nil, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
|
||||||
}
|
|
||||||
contract := domain.OutputContract{
|
|
||||||
Format: domain.OutputFormat(*raw.Output.Format),
|
|
||||||
ValidationMode: domain.ValidationMode(*raw.Output.ValidationMode),
|
|
||||||
RepairAttempts: raw.Output.RepairAttempts,
|
|
||||||
}
|
|
||||||
if raw.Output.SchemaPath != nil {
|
|
||||||
contract.SchemaPath = *raw.Output.SchemaPath
|
|
||||||
}
|
|
||||||
if err := domain.ValidateOutputContract(contract); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
|
||||||
}
|
|
||||||
definition.OutputFormat = contract.Format
|
|
||||||
definition.Validation = contract
|
definition.Validation = contract
|
||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
var err error
|
var err error
|
||||||
@@ -233,6 +109,113 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
return entry, nil
|
return entry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (state *loaded) capturePromptInputs(id string, rawInputs []promptInputConfig, base configkit.Path) ([]domain.PromptInput, []prompt.InputMetadata, error) {
|
||||||
|
seenInputs := make(map[string]bool, len(rawInputs))
|
||||||
|
inputs := make([]domain.PromptInput, 0, len(rawInputs))
|
||||||
|
metadata := make([]prompt.InputMetadata, 0, len(rawInputs))
|
||||||
|
for index, input := range rawInputs {
|
||||||
|
path := base.Key("inputs").Index(index)
|
||||||
|
if input.Name == "" {
|
||||||
|
return nil, nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
||||||
|
}
|
||||||
|
if seenInputs[input.Name] {
|
||||||
|
return nil, nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||||
|
}
|
||||||
|
seenInputs[input.Name] = true
|
||||||
|
contentType := ""
|
||||||
|
if input.ContentType != nil {
|
||||||
|
contentType = *input.ContentType
|
||||||
|
}
|
||||||
|
description := ""
|
||||||
|
if input.Description != nil {
|
||||||
|
description = *input.Description
|
||||||
|
}
|
||||||
|
inputs = append(inputs, domain.PromptInput{
|
||||||
|
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||||
|
})
|
||||||
|
presence := state.promptMetadata[id][index]
|
||||||
|
item := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||||
|
if presence.contentType == fieldValue {
|
||||||
|
item.ContentType = stringPointer(contentType)
|
||||||
|
}
|
||||||
|
if presence.description == fieldValue {
|
||||||
|
item.Description = stringPointer(description)
|
||||||
|
}
|
||||||
|
metadata = append(metadata, item)
|
||||||
|
}
|
||||||
|
return inputs, metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) capturePromptMessages(rawMessages []promptMessageConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) ([]domain.PromptMessageTemplate, []*template.Template, error) {
|
||||||
|
templates := make([]domain.PromptMessageTemplate, 0, len(rawMessages))
|
||||||
|
compiledMessages := make([]*template.Template, 0, len(rawMessages))
|
||||||
|
for index, message := range rawMessages {
|
||||||
|
path := base.Key("messages").Index(index)
|
||||||
|
if message.Role == nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
||||||
|
}
|
||||||
|
if _, err := domain.NormalizeMessageRole(*message.Role); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
||||||
|
}
|
||||||
|
if (message.Content != nil) == (message.ContentFile != nil) {
|
||||||
|
return nil, nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
||||||
|
}
|
||||||
|
content := ""
|
||||||
|
if message.Content != nil {
|
||||||
|
content = *message.Content
|
||||||
|
} else {
|
||||||
|
var err error
|
||||||
|
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var control *domain.CacheControl
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
if message.CacheControl.Type == nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
||||||
|
}
|
||||||
|
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
||||||
|
if message.CacheControl.TTL != nil {
|
||||||
|
control.TTL = *message.CacheControl.TTL
|
||||||
|
}
|
||||||
|
if _, err := domain.NormalizeCacheControl(control); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
||||||
|
}
|
||||||
|
compiledMessages = append(compiledMessages, compiled)
|
||||||
|
templates = append(templates, domain.PromptMessageTemplate{
|
||||||
|
Role: *message.Role, Content: content, CacheControl: control,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return templates, compiledMessages, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func capturePromptOutput(output *promptOutputConfig, base configkit.Path) (domain.OutputContract, error) {
|
||||||
|
if output == nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: output is required", base.Key("output"))
|
||||||
|
}
|
||||||
|
if output.Format == nil || output.ValidationMode == nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
||||||
|
}
|
||||||
|
contract := domain.OutputContract{
|
||||||
|
Format: domain.OutputFormat(*output.Format),
|
||||||
|
ValidationMode: domain.ValidationMode(*output.ValidationMode),
|
||||||
|
RepairAttempts: output.RepairAttempts,
|
||||||
|
}
|
||||||
|
if output.SchemaPath != nil {
|
||||||
|
contract.SchemaPath = *output.SchemaPath
|
||||||
|
}
|
||||||
|
if err := domain.ValidateOutputContract(contract); err != nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
||||||
|
}
|
||||||
|
return contract, nil
|
||||||
|
}
|
||||||
|
|
||||||
func stringPointer(value string) *string {
|
func stringPointer(value string) *string {
|
||||||
copy := value
|
copy := value
|
||||||
return ©
|
return ©
|
||||||
@@ -346,12 +329,6 @@ func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domai
|
|||||||
return ownPromptInspection(entry.inspection), nil
|
return ownPromptInspection(entry.inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
|
||||||
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
|
|
||||||
definition, _, err := state.GetPromptSnapshot(ctx, id)
|
|
||||||
return definition, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
||||||
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
||||||
if state == nil {
|
if state == nil {
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
|
|
||||||
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
||||||
files := fstest.MapFS{
|
files := fstest.MapFS{
|
||||||
"app/main.yaml": {Data: []byte("config:\n imports:\n - path: ../catalog/base.yaml\n required: true\ncollections:\n docs:\n prompts:\n - id: child\n extends: {id: base}\n other:\n prompts:\n - id: child\n messages:\n - role: user\n content_file: ./body.md\n output:\n format: text\n validation_mode: none\n")},
|
"app/main.yaml": {Data: []byte("config:\n imports:\n - path: ../catalog/base.yaml\n required: true\ncollections:\n docs:\n prompts:\n - id: child\n extends: {id: base}\n description: ' description '\n other:\n prompts:\n - id: child\n messages:\n - role: user\n content_file: ./body.md\n output:\n format: text\n validation_mode: none\n")},
|
||||||
"app/body.md": {Data: []byte("Replacement {{.name}}")},
|
"app/body.md": {Data: []byte("Replacement {{.name}}")},
|
||||||
"catalog/base.yaml": {Data: []byte("collections:\n docs:\n release_version: r1\n prompts:\n - id: base\n description: ' description '\n session_id: ' session '\n inputs:\n - name: ' document '\n content_type: ' text/plain '\n description: ' document source '\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
|
"catalog/base.yaml": {Data: []byte("collections:\n docs:\n release_version: r1\n prompts:\n - id: base\n description: ' description '\n session_id: ' session '\n inputs:\n - name: ' document '\n content_type: ' text/plain '\n description: ' document source '\n messages:\n - role: ' USER '\n cache_control:\n type: ' ephemeral '\n ttl: ' 1h '\n content_file: ./body.md\n output:\n format: text\n validation_mode: basic\n")},
|
||||||
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
|
"catalog/body.md": {Data: []byte("Captured {{.name}}")},
|
||||||
@@ -60,9 +60,9 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
t.Fatal("inherited prompt missing from effective list")
|
t.Fatal("inherited prompt missing from effective list")
|
||||||
}
|
}
|
||||||
composed := state.result.Config.Collections["docs"].Prompts[childIndex]
|
composed := state.result.Config.Collections["docs"].Prompts[childIndex]
|
||||||
if *composed.Description != " description " || *composed.SessionID != " session " || composed.Inputs[0].Name != " document " ||
|
if *composed.Description != "description" || *composed.SessionID != "session" || composed.Inputs[0].Name != "document" ||
|
||||||
*composed.Inputs[0].ContentType != " text/plain " || *composed.Inputs[0].Description != " document source " {
|
*composed.Inputs[0].ContentType != "text/plain" || *composed.Inputs[0].Description != "document source" {
|
||||||
t.Fatalf("composed metadata was changed: %+v", composed)
|
t.Fatalf("resolved metadata was not normalized: %+v", composed)
|
||||||
}
|
}
|
||||||
field, ok := state.result.Metadata().Lookup(configkit.Keys("collections", "docs", "prompts").Index(childIndex).Key("messages").Index(0).Key("content_file"))
|
field, ok := state.result.Metadata().Lookup(configkit.Keys("collections", "docs", "prompts").Index(childIndex).Key("messages").Index(0).Key("content_file"))
|
||||||
if !ok || len(field.Origin.Locations) != 1 {
|
if !ok || len(field.Origin.Locations) != 1 {
|
||||||
|
|||||||
68
internal/configuration/resolve_prompts.go
Normal file
68
internal/configuration/resolve_prompts.go
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func resolvePrompts(entries []promptConfig, base configkit.Path, resolution *configkit.Resolution) {
|
||||||
|
if !eligibleResourceList(entries, func(v promptConfig) string { return v.ID }) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i := range entries {
|
||||||
|
entry := &entries[i]
|
||||||
|
path := base.Index(i)
|
||||||
|
for _, field := range []struct {
|
||||||
|
value **string
|
||||||
|
name string
|
||||||
|
}{{&entry.Description, "description"}, {&entry.SessionID, "session_id"}} {
|
||||||
|
if *field.value != nil {
|
||||||
|
recordStringNormalization(field.value, path.Key(field.name), resolution, strings.TrimSpace(**field.value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for j := range entry.Inputs {
|
||||||
|
input := &entry.Inputs[j]
|
||||||
|
inputPath := path.Key("inputs").Index(j)
|
||||||
|
if normalized := strings.TrimSpace(input.Name); normalized != input.Name {
|
||||||
|
configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalized, "Normalize input name")
|
||||||
|
}
|
||||||
|
for _, field := range []struct {
|
||||||
|
value **string
|
||||||
|
name string
|
||||||
|
}{{&input.ContentType, "content_type"}, {&input.Description, "description"}} {
|
||||||
|
if *field.value != nil {
|
||||||
|
recordStringNormalization(field.value, inputPath.Key(field.name), resolution, strings.TrimSpace(**field.value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if entry.Output != nil && entry.Output.SchemaPath != nil {
|
||||||
|
recordStringNormalization(&entry.Output.SchemaPath, path.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*entry.Output.SchemaPath))
|
||||||
|
}
|
||||||
|
for j := range entry.Messages {
|
||||||
|
message := &entry.Messages[j]
|
||||||
|
messagePath := path.Key("messages").Index(j)
|
||||||
|
if message.Role != nil {
|
||||||
|
if value, err := domain.NormalizeMessageRole(*message.Role); err == nil {
|
||||||
|
recordStringNormalization(&message.Role, messagePath.Key("role"), resolution, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if message.CacheControl == nil || message.CacheControl.Type == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cache := message.CacheControl
|
||||||
|
control := &domain.CacheControl{Type: domain.CacheControlType(*cache.Type)}
|
||||||
|
if cache.TTL != nil {
|
||||||
|
control.TTL = *cache.TTL
|
||||||
|
}
|
||||||
|
if normalized, err := domain.NormalizeCacheControl(control); err == nil {
|
||||||
|
cachePath := messagePath.Key("cache_control")
|
||||||
|
recordStringNormalization(&cache.Type, cachePath.Key("type"), resolution, string(normalized.Type))
|
||||||
|
if cache.TTL != nil {
|
||||||
|
recordStringNormalization(&cache.TTL, cachePath.Key("ttl"), resolution, normalized.TTL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
102
internal/configuration/resolve_targets.go
Normal file
102
internal/configuration/resolve_targets.go
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func resolveBackends(entries []backendConfig, base configkit.Path, resolution *configkit.Resolution) {
|
||||||
|
if !eligibleResourceList(entries, func(v backendConfig) string { return v.ID }) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i := range entries {
|
||||||
|
entry := &entries[i]
|
||||||
|
path := base.Index(i)
|
||||||
|
if entry.Endpoint != nil {
|
||||||
|
if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil {
|
||||||
|
recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if entry.APIKeyEnv != nil {
|
||||||
|
recordStringNormalization(&entry.APIKeyEnv, path.Key("api_key_env"), resolution, strings.TrimSpace(*entry.APIKeyEnv))
|
||||||
|
}
|
||||||
|
if entry.ConcurrencyLimit != nil && *entry.ConcurrencyLimit > 0 && entry.QueueCapacity == nil {
|
||||||
|
capacity := 1024
|
||||||
|
entry.QueueCapacity = &capacity
|
||||||
|
resolution.RecordDerivedDefault(path.Key("queue_capacity"), "Derive queue capacity", path.Key("concurrency_limit"))
|
||||||
|
}
|
||||||
|
if entry.SmokeTests == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
smoke := entry.SmokeTests
|
||||||
|
smokePath := path.Key("smoke_tests")
|
||||||
|
if smoke.Reachability != nil {
|
||||||
|
resolveDiagnosticEndpoint(&smoke.Reachability.Endpoint, smokePath.Key("reachability").Key("endpoint"), resolution)
|
||||||
|
}
|
||||||
|
if smoke.Authentication != nil {
|
||||||
|
auth := smoke.Authentication
|
||||||
|
authPath := smokePath.Key("authentication")
|
||||||
|
if auth.Kind != nil {
|
||||||
|
recordStringNormalization(&auth.Kind, authPath.Key("kind"), resolution, strings.ToLower(strings.TrimSpace(*auth.Kind)))
|
||||||
|
}
|
||||||
|
resolveDiagnosticEndpoint(&auth.Endpoint, authPath.Key("endpoint"), resolution)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func diagnosticEndpointValid(value string) bool {
|
||||||
|
normalized := strings.TrimSpace(value)
|
||||||
|
request, err := http.NewRequest(http.MethodGet, normalized, nil)
|
||||||
|
return normalized != "" && err == nil && request.URL != nil && (strings.EqualFold(request.URL.Scheme, "http") || strings.EqualFold(request.URL.Scheme, "https")) && request.URL.Hostname() != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateBackendEndpoints(entry backendConfig, path configkit.Path, validation *configkit.Validation) {
|
||||||
|
if entry.Endpoint != nil {
|
||||||
|
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err != nil {
|
||||||
|
validation.Error(path.Key("endpoint"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if entry.SmokeTests == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
smoke := entry.SmokeTests
|
||||||
|
path = path.Key("smoke_tests")
|
||||||
|
if smoke.Reachability != nil && smoke.Reachability.Endpoint != nil && !diagnosticEndpointValid(*smoke.Reachability.Endpoint) {
|
||||||
|
validation.Error(path.Key("reachability").Key("endpoint"), "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
|
||||||
|
}
|
||||||
|
if smoke.Authentication != nil && smoke.Authentication.Endpoint != nil && !diagnosticEndpointValid(*smoke.Authentication.Endpoint) {
|
||||||
|
validation.Error(path.Key("authentication").Key("endpoint"), "diagnostic endpoint must be an absolute HTTP(S) URL with a host")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveDiagnosticEndpoint(value **string, path configkit.Path, resolution *configkit.Resolution) {
|
||||||
|
if *value != nil && diagnosticEndpointValid(**value) {
|
||||||
|
recordStringNormalization(value, path, resolution, strings.TrimSpace(**value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveProfiles(entries []profileConfig, base configkit.Path, resolution *configkit.Resolution) {
|
||||||
|
if !eligibleResourceList(entries, func(v profileConfig) string { return v.ID }) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i := range entries {
|
||||||
|
entry := &entries[i]
|
||||||
|
path := base.Index(i)
|
||||||
|
if entry.Endpoint != nil && *entry.Endpoint != "" {
|
||||||
|
if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil {
|
||||||
|
recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, field := range []struct {
|
||||||
|
value **string
|
||||||
|
name string
|
||||||
|
}{{&entry.ServiceTier, "service_tier"}, {&entry.ReasoningEffort, "reasoning_effort"}, {&entry.APIKeyEnv, "api_key_env"}} {
|
||||||
|
if *field.value != nil {
|
||||||
|
recordStringNormalization(field.value, path.Key(field.name), resolution, strings.TrimSpace(**field.value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,114 +12,120 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func (state *loaded) buildExecutionProfiles() error {
|
func (state *loaded) buildExecutionProfiles() error {
|
||||||
backendConfigs := indexedBackends(&state.interpreted)
|
backendConfigs := indexedBackends(&state.result.Config)
|
||||||
profileConfigs := indexedProfiles(&state.interpreted)
|
profileConfigs := indexedProfiles(&state.result.Config)
|
||||||
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
||||||
for _, id := range sortedKeys(backendConfigs) {
|
for _, id := range sortedKeys(backendConfigs) {
|
||||||
config := backendConfigs[id]
|
config := backendConfigs[id]
|
||||||
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
definitions[id] = captureBackend(id, config)
|
||||||
if config.SmokeTests != nil {
|
|
||||||
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
|
||||||
if config.SmokeTests.Reachability != nil {
|
|
||||||
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
|
||||||
}
|
|
||||||
if config.SmokeTests.Authentication != nil {
|
|
||||||
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
|
||||||
Kind: config.SmokeTests.Authentication.Kind,
|
|
||||||
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
definition.SmokeTests = smokeTests
|
|
||||||
}
|
|
||||||
if config.Endpoint != nil {
|
|
||||||
definition.Endpoint = *config.Endpoint
|
|
||||||
}
|
|
||||||
if config.APIKeyEnv != nil {
|
|
||||||
definition.APIKeyEnv = *config.APIKeyEnv
|
|
||||||
}
|
|
||||||
if config.ConcurrencyLimit != nil {
|
|
||||||
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
|
||||||
}
|
|
||||||
if config.QueueCapacity != nil {
|
|
||||||
definition.QueueCapacity = *config.QueueCapacity
|
|
||||||
definition.QueueCapacitySet = definition.ConcurrencyLimit > 0
|
|
||||||
}
|
|
||||||
definitions[id] = definition
|
|
||||||
}
|
}
|
||||||
registry, err := backend.NewEffectiveRegistry(definitions)
|
state.registry = backend.NewEffectiveRegistry(definitions)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
state.registry = registry
|
|
||||||
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
||||||
for _, id := range sortedKeys(profileConfigs) {
|
for _, id := range sortedKeys(profileConfigs) {
|
||||||
config := profileConfigs[id]
|
config := profileConfigs[id]
|
||||||
fields := state.profiles[id]
|
profile, err := captureExecutionProfile(id, config, state.profiles[id], state.registry)
|
||||||
target := defaults.ExecutionTargetDefault()
|
if err != nil {
|
||||||
if config.Backend != nil && *config.Backend != "" {
|
return err
|
||||||
selected, err := registry.GetBackend(*config.Backend)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("profile %q backend: %w", id, err)
|
|
||||||
}
|
|
||||||
target.BackendID = selected.ID
|
|
||||||
target.Endpoint = selected.Endpoint
|
|
||||||
target.APIKeyEnv = selected.APIKeyEnv
|
|
||||||
target.ExtraParams = selected.ExtraParams
|
|
||||||
}
|
}
|
||||||
presence := domain.ExecutionTargetPresence{}
|
|
||||||
if fields["endpoint"] != fieldAbsent {
|
|
||||||
target.Endpoint = valueOrZero(config.Endpoint)
|
|
||||||
}
|
|
||||||
if fields["model"] != fieldAbsent {
|
|
||||||
target.Model = valueOrZero(config.Model)
|
|
||||||
}
|
|
||||||
if fields["temperature"] != fieldAbsent {
|
|
||||||
target.Temperature = valueOrZero(config.Temperature)
|
|
||||||
presence.Temperature = fields["temperature"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["top_p"] != fieldAbsent {
|
|
||||||
target.TopP = valueOrZero(config.TopP)
|
|
||||||
presence.TopP = fields["top_p"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["max_tokens"] != fieldAbsent {
|
|
||||||
target.MaxTokens = valueOrZero(config.MaxTokens)
|
|
||||||
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["timeout_seconds"] == fieldValue {
|
|
||||||
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
|
||||||
presence.TimeoutSeconds = true
|
|
||||||
}
|
|
||||||
if fields["service_tier"] != fieldAbsent {
|
|
||||||
target.ServiceTier = valueOrZero(config.ServiceTier)
|
|
||||||
}
|
|
||||||
if fields["reasoning_effort"] != fieldAbsent {
|
|
||||||
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
|
||||||
}
|
|
||||||
if fields["api_key_env"] != fieldAbsent {
|
|
||||||
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
|
||||||
}
|
|
||||||
if fields["api_key_required"] != fieldAbsent {
|
|
||||||
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
|
||||||
}
|
|
||||||
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
|
||||||
target.APIKeyEnv = ""
|
|
||||||
}
|
|
||||||
switch fields["extra_params"] {
|
|
||||||
case fieldNull:
|
|
||||||
target.ExtraParams = nil
|
|
||||||
case fieldValue:
|
|
||||||
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("profile %q extra parameters: %w", id, err)
|
|
||||||
}
|
|
||||||
target.ExtraParams = merged
|
|
||||||
}
|
|
||||||
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
|
||||||
state.indexedProfiles[id] = profile
|
state.indexedProfiles[id] = profile
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func captureBackend(id string, config backendConfig) domain.Backend {
|
||||||
|
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
||||||
|
if config.SmokeTests != nil {
|
||||||
|
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
||||||
|
if config.SmokeTests.Reachability != nil {
|
||||||
|
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication != nil {
|
||||||
|
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
||||||
|
Kind: config.SmokeTests.Authentication.Kind,
|
||||||
|
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
definition.SmokeTests = smokeTests
|
||||||
|
}
|
||||||
|
if config.Endpoint != nil {
|
||||||
|
definition.Endpoint = *config.Endpoint
|
||||||
|
}
|
||||||
|
if config.APIKeyEnv != nil {
|
||||||
|
definition.APIKeyEnv = *config.APIKeyEnv
|
||||||
|
}
|
||||||
|
if config.ConcurrencyLimit != nil {
|
||||||
|
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
||||||
|
}
|
||||||
|
if config.QueueCapacity != nil {
|
||||||
|
definition.QueueCapacity = *config.QueueCapacity
|
||||||
|
}
|
||||||
|
return definition
|
||||||
|
}
|
||||||
|
|
||||||
|
func captureExecutionProfile(id string, config profileConfig, fields map[string]fieldState, registry *backend.Registry) (domain.ExecutionProfile, error) {
|
||||||
|
target := defaults.ExecutionTargetDefault()
|
||||||
|
if config.Backend != nil && *config.Backend != "" {
|
||||||
|
selected, err := registry.GetBackend(*config.Backend)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w", id, err)
|
||||||
|
}
|
||||||
|
target.BackendID = selected.ID
|
||||||
|
target.Endpoint = selected.Endpoint
|
||||||
|
target.APIKeyEnv = selected.APIKeyEnv
|
||||||
|
target.ExtraParams = selected.ExtraParams
|
||||||
|
}
|
||||||
|
presence := domain.ExecutionTargetPresence{}
|
||||||
|
if fields["endpoint"] != fieldAbsent {
|
||||||
|
target.Endpoint = valueOrZero(config.Endpoint)
|
||||||
|
}
|
||||||
|
if fields["model"] != fieldAbsent {
|
||||||
|
target.Model = valueOrZero(config.Model)
|
||||||
|
}
|
||||||
|
if fields["temperature"] != fieldAbsent {
|
||||||
|
target.Temperature = valueOrZero(config.Temperature)
|
||||||
|
presence.Temperature = fields["temperature"] == fieldValue
|
||||||
|
}
|
||||||
|
if fields["top_p"] != fieldAbsent {
|
||||||
|
target.TopP = valueOrZero(config.TopP)
|
||||||
|
presence.TopP = fields["top_p"] == fieldValue
|
||||||
|
}
|
||||||
|
if fields["max_tokens"] != fieldAbsent {
|
||||||
|
target.MaxTokens = valueOrZero(config.MaxTokens)
|
||||||
|
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
||||||
|
}
|
||||||
|
if fields["timeout_seconds"] == fieldValue {
|
||||||
|
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
||||||
|
presence.TimeoutSeconds = true
|
||||||
|
}
|
||||||
|
if fields["service_tier"] != fieldAbsent {
|
||||||
|
target.ServiceTier = valueOrZero(config.ServiceTier)
|
||||||
|
}
|
||||||
|
if fields["reasoning_effort"] != fieldAbsent {
|
||||||
|
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
||||||
|
}
|
||||||
|
if fields["api_key_env"] != fieldAbsent {
|
||||||
|
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
||||||
|
}
|
||||||
|
if fields["api_key_required"] != fieldAbsent {
|
||||||
|
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
||||||
|
}
|
||||||
|
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
||||||
|
target.APIKeyEnv = ""
|
||||||
|
}
|
||||||
|
switch fields["extra_params"] {
|
||||||
|
case fieldNull:
|
||||||
|
target.ExtraParams = nil
|
||||||
|
case fieldValue:
|
||||||
|
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionProfile{}, fmt.Errorf("profile %q extra parameters: %w", id, err)
|
||||||
|
}
|
||||||
|
target.ExtraParams = merged
|
||||||
|
}
|
||||||
|
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
||||||
|
return profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
func valueOrZero[T any](value *T) T {
|
func valueOrZero[T any](value *T) T {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
var zero T
|
var zero T
|
||||||
@@ -138,24 +144,33 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
|||||||
if merged == nil {
|
if merged == nil {
|
||||||
merged = make(map[string]any)
|
merged = make(map[string]any)
|
||||||
}
|
}
|
||||||
|
if err := overlayOwnedProviderMap(merged, upper); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return merged, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlayOwnedProviderMap(merged, upper map[string]any) error {
|
||||||
for key, value := range upper {
|
for key, value := range upper {
|
||||||
if child, ok := value.(map[string]any); ok {
|
if child, ok := value.(map[string]any); ok {
|
||||||
if existing, ok := merged[key].(map[string]any); ok {
|
if existing, ok := merged[key].(map[string]any); ok {
|
||||||
combined, err := overlayProviderMap(existing, child)
|
if existing == nil {
|
||||||
if err != nil {
|
existing = make(map[string]any)
|
||||||
return nil, err
|
merged[key] = existing
|
||||||
|
}
|
||||||
|
if err := overlayOwnedProviderMap(existing, child); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
merged[key] = combined
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
copied, err := jsonvalue.Copy(value)
|
copied, err := jsonvalue.Copy(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return err
|
||||||
}
|
}
|
||||||
merged[key] = copied
|
merged[key] = copied
|
||||||
}
|
}
|
||||||
return merged, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
||||||
@@ -180,11 +195,7 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
target := *profile.ResolvedTarget
|
target := *profile.ResolvedTarget
|
||||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
target.ExtraParams = params
|
|
||||||
profile.ResolvedTarget = &target
|
profile.ResolvedTarget = &target
|
||||||
return &profile, nil
|
return &profile, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,6 +42,28 @@ func TestLoadIndexesIdentifiedTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOverlayProviderMapOwnsInputsAndReplacesValues(t *testing.T) {
|
||||||
|
lower := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"old"}, "null": "old"}, "disjoint": []any{"original"}}
|
||||||
|
upper := map[string]any{"nested": map[string]any{"replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}}
|
||||||
|
got, err := overlayProviderMap(lower, upper)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}, "disjoint": []any{"original"}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("overlay = %#v", got)
|
||||||
|
}
|
||||||
|
lower["disjoint"].([]any)[0] = "changed"
|
||||||
|
upper["nested"].(map[string]any)["replace"].([]any)[0] = "changed"
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("overlay shares input storage: %#v", got)
|
||||||
|
}
|
||||||
|
got["nested"].(map[string]any)["added"].(map[string]any)["value"] = "changed"
|
||||||
|
if upper["nested"].(map[string]any)["added"].(map[string]any)["value"] != "upper" {
|
||||||
|
t.Fatal("result mutation reached upper")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
||||||
state, err := load(mapping("partial profile", map[string]any{
|
state, err := load(mapping("partial profile", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "partial"}},
|
"profiles": []any{map[string]any{"id": "partial"}},
|
||||||
@@ -53,7 +75,7 @@ func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
runner := usecase.NewRunner(state, state, nil, prompt.NewGoRenderer(), nil, nil, nil)
|
runner := usecase.NewRunner(state, state, nil, prompt.NewCompiledRenderer(state), nil, nil, nil)
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "example", ProfileID: "partial",
|
PromptID: "example", ProfileID: "partial",
|
||||||
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
|
||||||
@@ -97,7 +119,7 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
selected, err := state.registry.GetBackend("second")
|
selected, err := state.registry.GetBackend("second")
|
||||||
if err != nil || selected.QueueCapacity != 0 || !selected.QueueCapacitySet {
|
if err != nil || selected.QueueCapacity != 0 || selected.ConcurrencyLimit != 2 {
|
||||||
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
||||||
}
|
}
|
||||||
child, err := state.GetProfile(context.Background(), "child")
|
child, err := state.GetProfile(context.Background(), "child")
|
||||||
@@ -197,10 +219,7 @@ func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
||||||
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
||||||
|
|||||||
@@ -145,7 +145,6 @@ type PromptDefinition struct {
|
|||||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||||
Inputs []PromptInput `yaml:"inputs"`
|
Inputs []PromptInput `yaml:"inputs"`
|
||||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
Templates []PromptMessageTemplate `yaml:"templates"`
|
||||||
OutputFormat OutputFormat `yaml:"output_format"`
|
|
||||||
Validation OutputContract `yaml:"validation"`
|
Validation OutputContract `yaml:"validation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,7 +169,6 @@ type PromptInput struct {
|
|||||||
type PromptMessageTemplate struct {
|
type PromptMessageTemplate struct {
|
||||||
Role string `yaml:"role"`
|
Role string `yaml:"role"`
|
||||||
Content string `yaml:"content"`
|
Content string `yaml:"content"`
|
||||||
ContentFile string `yaml:"content_file"`
|
|
||||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -184,7 +182,6 @@ type Backend struct {
|
|||||||
SmokeTests *BackendSmokeTestConfig
|
SmokeTests *BackendSmokeTestConfig
|
||||||
ConcurrencyLimit int
|
ConcurrencyLimit int
|
||||||
QueueCapacity int
|
QueueCapacity int
|
||||||
QueueCapacitySet bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendCapacityPolicy describes normalized run and generation capacity for
|
// BackendCapacityPolicy describes normalized run and generation capacity for
|
||||||
|
|||||||
@@ -31,14 +31,6 @@ func qualifyResourceID(id, namespace string) (string, bool) {
|
|||||||
return namespace + "/" + id, true
|
return namespace + "/" + id, true
|
||||||
}
|
}
|
||||||
|
|
||||||
func resourceNamespaceFromID(id string) (string, bool) {
|
|
||||||
if !validResourceID(id) || !strings.Contains(id, "/") {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
namespace, _, _ := strings.Cut(id, "/")
|
|
||||||
return namespace, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
||||||
// valid namespace/local-ID pair.
|
// valid namespace/local-ID pair.
|
||||||
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
||||||
|
|||||||
@@ -86,25 +86,3 @@ func TestQualifyResourceID(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResourceNamespaceFromID(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
id string
|
|
||||||
namespace string
|
|
||||||
wantOK bool
|
|
||||||
}{
|
|
||||||
{name: "qualified", id: "team/item", namespace: "team", wantOK: true},
|
|
||||||
{name: "same-prefix namespace stays distinct", id: "team-extra/item", namespace: "team-extra", wantOK: true},
|
|
||||||
{name: "unqualified", id: "item", wantOK: false},
|
|
||||||
{name: "malformed", id: "team/sub/item", wantOK: false},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
got, ok := resourceNamespaceFromID(test.id)
|
|
||||||
if got != test.namespace || ok != test.wantOK {
|
|
||||||
t.Fatalf("resourceNamespaceFromID(%q) = (%q, %t), want (%q, %t)", test.id, got, ok, test.namespace, test.wantOK)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,5 +18,5 @@ func NormalizeSessionID(raw string) (string, error) {
|
|||||||
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
||||||
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
||||||
}
|
}
|
||||||
return normalized, nil
|
return strings.Clone(normalized), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,24 +112,21 @@ type SmokeTestAttempt struct {
|
|||||||
type SmokeTestReason string
|
type SmokeTestReason string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
SmokeTestReasonNone SmokeTestReason = ""
|
SmokeTestReasonNone SmokeTestReason = ""
|
||||||
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
||||||
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
||||||
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
||||||
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
||||||
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
||||||
|
SmokeTestReasonAuthenticationResponseLimitExceeded SmokeTestReason = "authentication_response_limit_exceeded"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CloneBackend returns a copy of a backend, including its mutable settings.
|
// CloneBackend returns a copy of a backend, including its mutable settings.
|
||||||
func CloneBackend(backend Backend) (Backend, error) {
|
func CloneBackend(backend Backend) Backend {
|
||||||
cloned := backend
|
cloned := backend
|
||||||
var err error
|
cloned.ExtraParams = jsonvalue.CloneTrusted(backend.ExtraParams).(map[string]any)
|
||||||
cloned.ExtraParams, err = jsonvalue.CopyMap(backend.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return Backend{}, err
|
|
||||||
}
|
|
||||||
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
||||||
return cloned, nil
|
return cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
||||||
@@ -153,105 +150,6 @@ func CloneBackendSmokeTestConfig(config *BackendSmokeTestConfig) *BackendSmokeTe
|
|||||||
return &cloned
|
return &cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneSmokeTestRequest returns a copy that owns its optional settings.
|
|
||||||
func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
|
|
||||||
cloned := request
|
|
||||||
cloned.TransientRetries = cloneInt(request.TransientRetries)
|
|
||||||
if request.Execution == nil {
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
execution := *request.Execution
|
|
||||||
execution.Temperature = cloneFloat64(request.Execution.Temperature)
|
|
||||||
execution.MaxTokens = cloneInt(request.Execution.MaxTokens)
|
|
||||||
execution.TopP = cloneFloat64(request.Execution.TopP)
|
|
||||||
execution.TimeoutSeconds = cloneInt(request.Execution.TimeoutSeconds)
|
|
||||||
execution.ReasoningEffort = cloneString(request.Execution.ReasoningEffort)
|
|
||||||
var err error
|
|
||||||
execution.ExtraParams, err = jsonvalue.CopyMap(request.Execution.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return SmokeTestRequest{}, err
|
|
||||||
}
|
|
||||||
cloned.Execution = &execution
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CloneSmokeTestReport returns a copy with independent optional facts and maps.
|
|
||||||
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
|
|
||||||
cloned := report
|
|
||||||
cloned.Applicable = cloneBool(report.Applicable)
|
|
||||||
cloned.Facts = cloneSmokeTestFacts(report.Facts)
|
|
||||||
cloned.Timing = cloneSmokeTestTiming(report.Timing)
|
|
||||||
var err error
|
|
||||||
cloned.Target.EffectiveSettings.ExtraParams, err = jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return SmokeTestReport{}, err
|
|
||||||
}
|
|
||||||
if report.Attempts != nil {
|
|
||||||
cloned.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
|
|
||||||
for index, attempt := range report.Attempts {
|
|
||||||
cloned.Attempts[index] = attempt
|
|
||||||
cloned.Attempts[index].HTTPStatus = cloneInt(attempt.HTTPStatus)
|
|
||||||
cloned.Attempts[index].Facts = cloneSmokeTestFacts(attempt.Facts)
|
|
||||||
cloned.Attempts[index].Timing = cloneSmokeTestTiming(attempt.Timing)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneSmokeTestFacts(facts SmokeTestFacts) SmokeTestFacts {
|
|
||||||
facts.CredentialsSupplied = cloneBool(facts.CredentialsSupplied)
|
|
||||||
facts.EndpointReachable = cloneBool(facts.EndpointReachable)
|
|
||||||
facts.CredentialsAccepted = cloneBool(facts.CredentialsAccepted)
|
|
||||||
facts.ResponseReceived = cloneBool(facts.ResponseReceived)
|
|
||||||
facts.GenerationCompleted = cloneBool(facts.GenerationCompleted)
|
|
||||||
facts.TextReturned = cloneBool(facts.TextReturned)
|
|
||||||
facts.ExpectedTextMatched = cloneBool(facts.ExpectedTextMatched)
|
|
||||||
facts.StructuredOutputRequested = cloneBool(facts.StructuredOutputRequested)
|
|
||||||
facts.ValidJSON = cloneBool(facts.ValidJSON)
|
|
||||||
facts.SchemaMatched = cloneBool(facts.SchemaMatched)
|
|
||||||
return facts
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneSmokeTestTiming(timing SmokeTestTiming) SmokeTestTiming {
|
|
||||||
timing.CapacityWait = cloneDuration(timing.CapacityWait)
|
|
||||||
timing.RequestDuration = cloneDuration(timing.RequestDuration)
|
|
||||||
timing.RetryDelay = cloneDuration(timing.RetryDelay)
|
|
||||||
timing.EvaluationDuration = cloneDuration(timing.EvaluationDuration)
|
|
||||||
return timing
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneBool(value *bool) *bool {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneDuration(value *time.Duration) *time.Duration {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneFloat64(value *float64) *float64 {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneInt(value *int) *int {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneString(value *string) *string {
|
func cloneString(value *string) *string {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
package domain
|
package domain
|
||||||
|
|
||||||
import (
|
import "testing"
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
||||||
profile := "profile"
|
profile := "profile"
|
||||||
@@ -22,10 +19,7 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cloned, err := CloneBackend(backend)
|
cloned := CloneBackend(backend)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.SmokeTests.DefaultProfile = "changed"
|
*cloned.SmokeTests.DefaultProfile = "changed"
|
||||||
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
||||||
*cloned.SmokeTests.Authentication.Kind = "changed"
|
*cloned.SmokeTests.Authentication.Kind = "changed"
|
||||||
@@ -41,72 +35,12 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCloneSmokeTestRequestOwnsOptionalSettings(t *testing.T) {
|
func TestCloneBackendPreservesProviderMapPresence(t *testing.T) {
|
||||||
temperature := 0.25
|
if cloned := CloneBackend(Backend{}); cloned.ExtraParams != nil {
|
||||||
retries := 2
|
t.Fatalf("nil provider map became %#v", cloned.ExtraParams)
|
||||||
reasoning := "low"
|
|
||||||
request := SmokeTestRequest{
|
|
||||||
APIKey: "secret",
|
|
||||||
TransientRetries: &retries,
|
|
||||||
Execution: &ExecutionTargetOverride{
|
|
||||||
Temperature: &temperature,
|
|
||||||
ReasoningEffort: &reasoning,
|
|
||||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
cloned := CloneBackend(Backend{ExtraParams: map[string]any{"empty": map[string]any{}, "null": nil}})
|
||||||
cloned, err := CloneSmokeTestRequest(request)
|
if cloned.ExtraParams == nil || cloned.ExtraParams["empty"].(map[string]any) == nil || cloned.ExtraParams["null"] != nil {
|
||||||
if err != nil {
|
t.Fatalf("provider map presence changed: %#v", cloned.ExtraParams)
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.TransientRetries = 3
|
|
||||||
*cloned.Execution.Temperature = 0.5
|
|
||||||
*cloned.Execution.ReasoningEffort = "high"
|
|
||||||
cloned.Execution.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
|
|
||||||
if retries != 2 || temperature != 0.25 || reasoning != "low" {
|
|
||||||
t.Fatal("cloned request shared optional scalar pointers")
|
|
||||||
}
|
|
||||||
if got := request.Execution.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
|
||||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
|
|
||||||
applicable := true
|
|
||||||
reachable := true
|
|
||||||
status := 200
|
|
||||||
wait := 3 * time.Millisecond
|
|
||||||
report := SmokeTestReport{
|
|
||||||
Applicable: &applicable,
|
|
||||||
Target: SmokeTestTarget{
|
|
||||||
EffectiveSettings: ExecutionTarget{
|
|
||||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
|
||||||
Attempts: []SmokeTestAttempt{{
|
|
||||||
HTTPStatus: &status,
|
|
||||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
|
||||||
Timing: SmokeTestTiming{CapacityWait: &wait},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
|
|
||||||
cloned, err := CloneSmokeTestReport(report)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.Applicable = false
|
|
||||||
*cloned.Facts.EndpointReachable = false
|
|
||||||
*cloned.Attempts[0].HTTPStatus = 503
|
|
||||||
*cloned.Attempts[0].Facts.EndpointReachable = false
|
|
||||||
*cloned.Attempts[0].Timing.CapacityWait = time.Second
|
|
||||||
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
|
|
||||||
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond {
|
|
||||||
t.Fatal("cloned report shared optional scalar pointers")
|
|
||||||
}
|
|
||||||
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
|
||||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,13 +41,17 @@ type ConfigurationSources struct {
|
|||||||
|
|
||||||
// Options collects construction choices from the public facade.
|
// Options collects construction choices from the public facade.
|
||||||
type Options struct {
|
type Options struct {
|
||||||
llmClient llm.Client
|
llmClient llm.Client
|
||||||
artifactReader artifact.Reader
|
artifactReader artifact.Reader
|
||||||
validator validate.Validator
|
validator validate.ValidationPreparer
|
||||||
validatorSource bool
|
validatorSource bool
|
||||||
artifactSource bool
|
artifactSource bool
|
||||||
|
disableBundledOpenRouter bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetBundledOpenRouter selects whether the built-in OpenRouter source is installed.
|
||||||
|
func (o *Options) SetBundledOpenRouter(enabled bool) { o.disableBundledOpenRouter = !enabled }
|
||||||
|
|
||||||
// SetLLMClient selects the client collaborator for one engine.
|
// SetLLMClient selects the client collaborator for one engine.
|
||||||
func (o *Options) SetLLMClient(client llm.Client) { o.llmClient = client }
|
func (o *Options) SetLLMClient(client llm.Client) { o.llmClient = client }
|
||||||
|
|
||||||
@@ -69,7 +73,7 @@ func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
|
|||||||
|
|
||||||
// SetSchemaFile selects a single schema file.
|
// SetSchemaFile selects a single schema file.
|
||||||
func (o *Options) SetSchemaFile(path string) error {
|
func (o *Options) SetSchemaFile(path string) error {
|
||||||
fsys, root, _, err := fileSource(path)
|
fsys, root, err := fileSource(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -94,9 +98,9 @@ type resourceNamespaces struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type identityInventory interface {
|
type identityInventory interface {
|
||||||
ListBackends(context.Context) ([]string, error)
|
ListBackends(context.Context, configuration.IdentityFilter) ([]string, error)
|
||||||
ListProfiles(context.Context) ([]string, error)
|
ListProfiles(context.Context, configuration.IdentityFilter) ([]string, error)
|
||||||
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
ListPrompts(context.Context, configuration.IdentityFilter) ([]domain.PromptIdentity, error)
|
||||||
ListNamespaces(context.Context, []string) ([]string, error)
|
ListNamespaces(context.Context, []string) ([]string, error)
|
||||||
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
||||||
}
|
}
|
||||||
@@ -124,6 +128,11 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
underlays := cfg.Sources.Underlays
|
underlays := cfg.Sources.Underlays
|
||||||
|
if !options.disableBundledOpenRouter {
|
||||||
|
underlays = make([]configkit.Underlay, 1, len(cfg.Sources.Underlays)+1)
|
||||||
|
underlays[0] = configuration.BundledOpenRouterUnderlay()
|
||||||
|
underlays = append(underlays, cfg.Sources.Underlays...)
|
||||||
|
}
|
||||||
main := configkit.Input{Source: configkit.NoSource()}
|
main := configkit.Input{Source: configkit.NoSource()}
|
||||||
if cfg.Sources.Main != nil {
|
if cfg.Sources.Main != nil {
|
||||||
main = *cfg.Sources.Main
|
main = *cfg.Sources.Main
|
||||||
@@ -148,10 +157,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
var client llm.Client
|
var client llm.Client
|
||||||
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
||||||
if options.llmClient == nil {
|
if options.llmClient == nil {
|
||||||
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
builtIn := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
||||||
if newErr != nil {
|
|
||||||
return nil, newErr
|
|
||||||
}
|
|
||||||
diagnosticHTTPClient = builtIn
|
diagnosticHTTPClient = builtIn
|
||||||
client = generation.NewBuiltIn(manager, builtIn)
|
client = generation.NewBuiltIn(manager, builtIn)
|
||||||
} else {
|
} else {
|
||||||
@@ -199,23 +205,23 @@ func validateSourceFS(fsys fs.FS, root string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, string, error) {
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
if strings.TrimSpace(name) == "" {
|
if strings.TrimSpace(name) == "" {
|
||||||
return nil, "", "", fmt.Errorf("source path is required")
|
return nil, "", fmt.Errorf("source path is required")
|
||||||
}
|
}
|
||||||
directory := filepath.Dir(name)
|
directory := filepath.Dir(name)
|
||||||
base := filepath.Base(name)
|
base := filepath.Base(name)
|
||||||
if base == "." || base == string(filepath.Separator) {
|
if base == "." || base == string(filepath.Separator) {
|
||||||
return nil, "", "", fmt.Errorf("source path must name a file")
|
return nil, "", fmt.Errorf("source path must name a file")
|
||||||
}
|
}
|
||||||
info, err := os.Stat(name)
|
info, err := os.Stat(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
return nil, "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
||||||
}
|
}
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return nil, "", "", fmt.Errorf("source path %q must be a file", name)
|
return nil, "", fmt.Errorf("source path %q must be a file", name)
|
||||||
}
|
}
|
||||||
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
return os.DirFS(directory), filepath.ToSlash(base), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
@@ -259,24 +265,11 @@ func (e *Engine) ListProfiles(ctx context.Context, filters ...ListingFilter) ([]
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
||||||
mode, namespace, err := resolveListingFilter(filters, e.namespaces.prompt)
|
filter, err := inventoryFilter(filters, e.namespaces.prompt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
prompts, err := e.listing.ListPrompts(ctx)
|
return e.listing.ListPrompts(ctx, filter)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var filtered []domain.PromptIdentity
|
|
||||||
for _, prompt := range prompts {
|
|
||||||
if matchesListingNamespace(prompt.ID, mode, namespace) {
|
|
||||||
filtered = append(filtered, prompt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return filtered, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
@@ -298,26 +291,21 @@ func filterIdentityList(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
filters []ListingFilter,
|
filters []ListingFilter,
|
||||||
scope string,
|
scope string,
|
||||||
list func(context.Context) ([]string, error),
|
list func(context.Context, configuration.IdentityFilter) ([]string, error),
|
||||||
) ([]string, error) {
|
) ([]string, error) {
|
||||||
|
filter, err := inventoryFilter(filters, scope)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return list(ctx, filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func inventoryFilter(filters []ListingFilter, scope string) (configuration.IdentityFilter, error) {
|
||||||
mode, namespace, err := resolveListingFilter(filters, scope)
|
mode, namespace, err := resolveListingFilter(filters, scope)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return configuration.IdentityFilter{}, err
|
||||||
}
|
}
|
||||||
ids, err := list(ctx)
|
return configuration.IdentityFilter{All: mode == ListingAll, Namespace: namespace}, nil
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var filtered []string
|
|
||||||
for _, id := range ids {
|
|
||||||
if matchesListingNamespace(id, mode, namespace) {
|
|
||||||
filtered = append(filtered, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return filtered, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
||||||
@@ -346,19 +334,6 @@ func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterM
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func matchesListingNamespace(id string, mode ListingFilterMode, namespace string) bool {
|
|
||||||
switch mode {
|
|
||||||
case ListingNamed:
|
|
||||||
idNamespace, _, qualified := strings.Cut(id, "/")
|
|
||||||
return qualified && idNamespace == namespace
|
|
||||||
case ListingUnqualified:
|
|
||||||
return !strings.Contains(id, "/")
|
|
||||||
case ListingAll:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||||
request, err := e.qualifyRunRequest(request)
|
request, err := e.qualifyRunRequest(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -67,15 +67,6 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
|
|||||||
return files, err
|
return files, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// RelativePath computes a clean relative path from root to path.
|
|
||||||
func RelativePath(root string, filePath string) string {
|
|
||||||
rel, err := filepath.Rel(root, filePath)
|
|
||||||
if err != nil {
|
|
||||||
return filepath.Clean(filePath)
|
|
||||||
}
|
|
||||||
return filepath.Clean(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
||||||
// nonblank leading and trailing whitespace.
|
// nonblank leading and trailing whitespace.
|
||||||
func CleanFSRoot(root string) string {
|
func CleanFSRoot(root string) string {
|
||||||
|
|||||||
@@ -81,16 +81,6 @@ func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRelativePathNested(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
|
||||||
got := RelativePath(root, path)
|
|
||||||
want := filepath.Join("nested", "profiles", "local.yaml")
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("expected relative path %q, got %q", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanFSRoot(t *testing.T) {
|
func TestCleanFSRoot(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -85,6 +85,12 @@ func (c *Coordinator) GenerateWithEvidence(
|
|||||||
return c.generate(ctx, req, true)
|
return c.generate(ctx, req, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreparationError identifies a built-in request preparation failure.
|
||||||
|
type PreparationError struct{ Cause error }
|
||||||
|
|
||||||
|
func (e *PreparationError) Error() string { return e.Cause.Error() }
|
||||||
|
func (e *PreparationError) Unwrap() error { return e.Cause }
|
||||||
|
|
||||||
func (c *Coordinator) generate(
|
func (c *Coordinator) generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -95,7 +101,7 @@ func (c *Coordinator) generate(
|
|||||||
}
|
}
|
||||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err, GenerationEvidence{}
|
return nil, &PreparationError{Cause: err}, GenerationEvidence{PreparationFailed: true}
|
||||||
}
|
}
|
||||||
|
|
||||||
evidence := GenerationEvidence{}
|
evidence := GenerationEvidence{}
|
||||||
@@ -120,66 +126,37 @@ func (c *Coordinator) generate(
|
|||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
ordinal := attempts + 1
|
ordinal := attempts + 1
|
||||||
outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
result := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
||||||
if dispatched {
|
if result.Dispatched {
|
||||||
attempts++
|
attempts++
|
||||||
}
|
}
|
||||||
if captureEvidence {
|
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
|
||||||
attempt := AttemptEvidence{
|
if result.Outcome.Err == nil {
|
||||||
StartedAt: startedAt.UTC(),
|
|
||||||
Dispatched: dispatched,
|
|
||||||
ResponseReceived: outcome.Evidence.ResponseReceived,
|
|
||||||
HTTP: cloneHTTPEvidence(outcome.Evidence),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
RequestDuration: generationDuration,
|
|
||||||
Failure: ClassifyFailure(outcome.Err),
|
|
||||||
ResponseDecoded: outcome.ResponseDecoded,
|
|
||||||
GenerationCompleted: outcome.GenerationCompleted,
|
|
||||||
}
|
|
||||||
if outcome.StructuredOutputRequested != nil {
|
|
||||||
requested := *outcome.StructuredOutputRequested
|
|
||||||
attempt.StructuredOutputRequested = &requested
|
|
||||||
}
|
|
||||||
if dispatched {
|
|
||||||
usage.Add(outcome.Usage, outcome.UsagePresence)
|
|
||||||
var attemptUsage observation.UsageAccumulator
|
|
||||||
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
|
|
||||||
attempt.Usage = attemptUsage.Observed()
|
|
||||||
}
|
|
||||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
|
||||||
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
|
||||||
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
|
|
||||||
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
|
||||||
}
|
|
||||||
if outcome.Err == nil {
|
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(outcome.Response, err)
|
return finish(result.Outcome.Response, err)
|
||||||
}
|
}
|
||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
return finish(outcome.Response, nil)
|
return finish(result.Outcome.Response, nil)
|
||||||
}
|
}
|
||||||
if !dispatched || !eligibleStatus(outcome) {
|
if !result.Dispatched || !eligibleStatus(result.Outcome) {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(outcome.Response, outcome.Err)
|
return finish(result.Outcome.Response, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
return finish(nil, outcome.Err)
|
return finish(nil, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
if retriesUsed == req.TransientRetries {
|
if retriesUsed == req.TransientRetries {
|
||||||
if retriesUsed == 0 {
|
if retriesUsed == 0 {
|
||||||
return finish(nil, outcome.Err)
|
return finish(nil, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err})
|
return finish(nil, &ExhaustedError{attempts: attempts, cause: result.Outcome.Err})
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
retryOrdinal := retriesUsed + 1
|
retryOrdinal := retriesUsed + 1
|
||||||
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
delay := retryDelay(c.backoff(retryOrdinal), result.Outcome.RetryNotBefore, c.now())
|
||||||
waitStart := c.now()
|
waitStart := c.now()
|
||||||
waitErr := c.wait(ctx, delay)
|
waitErr := c.wait(ctx, delay)
|
||||||
waitDuration := elapsed(waitStart, c.now())
|
waitDuration := elapsed(waitStart, c.now())
|
||||||
@@ -200,127 +177,133 @@ func (c *Coordinator) generate(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) generateInjected(
|
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest, captureEvidence bool) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
ctx context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
captureEvidence bool,
|
|
||||||
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
|
||||||
evidence := GenerationEvidence{}
|
evidence := GenerationEvidence{}
|
||||||
var usage observation.UsageAccumulator
|
var usage observation.UsageAccumulator
|
||||||
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Usage = usage.Observed()
|
|
||||||
}
|
|
||||||
return response, err, evidence
|
|
||||||
}
|
|
||||||
recorder := observation.FromContext(ctx)
|
|
||||||
generation := req.Generation
|
generation := req.Generation
|
||||||
if generation == 0 {
|
if generation == 0 {
|
||||||
generation = 1
|
generation = 1
|
||||||
}
|
}
|
||||||
start := c.now()
|
result := c.executeInjected(ctx, req)
|
||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
|
||||||
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
|
||||||
if err != nil {
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
|
||||||
StartedAt: start.UTC(),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
|
||||||
}
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
release()
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
|
||||||
StartedAt: start.UTC(),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
|
||||||
}
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
|
||||||
dispatchStart := c.now()
|
|
||||||
response, err := c.injected.Generate(ctx, req)
|
|
||||||
duration := elapsed(dispatchStart, c.now())
|
|
||||||
if err == nil && response == nil {
|
|
||||||
err = llm.ErrMalformedResponse
|
|
||||||
}
|
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
attempt := AttemptEvidence{
|
evidence.Usage = usage.Observed()
|
||||||
StartedAt: start.UTC(),
|
|
||||||
Dispatched: true,
|
|
||||||
ResponseReceived: response != nil,
|
|
||||||
RequestDuration: duration,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
GenerationCompleted: err == nil && response != nil,
|
|
||||||
}
|
|
||||||
requested := req.StructuredOutput != nil
|
|
||||||
attempt.StructuredOutputRequested = &requested
|
|
||||||
var reportedUsage domain.TokenUsage
|
|
||||||
var reportedPresence *domain.TokenUsagePresence
|
|
||||||
if err == nil && response != nil {
|
|
||||||
reportedUsage = response.Usage
|
|
||||||
reportedPresence = response.UsagePresence
|
|
||||||
}
|
|
||||||
usage.Add(reportedUsage, reportedPresence)
|
|
||||||
var attemptUsage observation.UsageAccumulator
|
|
||||||
attemptUsage.Add(reportedUsage, reportedPresence)
|
|
||||||
attempt.Usage = attemptUsage.Observed()
|
|
||||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
|
||||||
}
|
}
|
||||||
release()
|
if result.Outcome.Err != nil {
|
||||||
if recorder != nil {
|
return nil, result.Outcome.Err, evidence
|
||||||
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
|
||||||
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
|
|
||||||
if err == nil {
|
|
||||||
data.Usage = response.Usage
|
|
||||||
data.UsagePresence = response.UsagePresence
|
|
||||||
}
|
|
||||||
recorder.Attempt(data)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(response, err)
|
return result.Outcome.Response, err, evidence
|
||||||
}
|
}
|
||||||
return finish(nil, err)
|
return nil, err, evidence
|
||||||
}
|
}
|
||||||
return finish(response, nil)
|
return result.Outcome.Response, nil, evidence
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) {
|
func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRequest) attemptResult {
|
||||||
start := c.now()
|
result := attemptResult{StartedAt: c.now()}
|
||||||
startedAt = start
|
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||||
release, err := c.manager.Acquire(ctx, backendID)
|
result.CapacityWait = c.capacityWait(req.Target.BackendID, result.StartedAt)
|
||||||
capacityWait = c.capacityWait(backendID, start)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
start := c.now()
|
||||||
|
result.Dispatched = true
|
||||||
|
requested := req.StructuredOutput != nil
|
||||||
|
result.Outcome.StructuredOutputRequested = &requested
|
||||||
|
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
|
||||||
|
result.RequestDuration = elapsed(start, c.now())
|
||||||
|
if result.Outcome.Err == nil && result.Outcome.Response == nil {
|
||||||
|
result.Outcome.Err = llm.ErrMalformedResponse
|
||||||
|
}
|
||||||
|
if result.Outcome.Err == nil {
|
||||||
|
result.Outcome.Usage = result.Outcome.Response.Usage
|
||||||
|
result.Outcome.UsagePresence = result.Outcome.Response.UsagePresence
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type attemptResult struct {
|
||||||
|
Outcome llm.AttemptOutcome
|
||||||
|
Dispatched bool
|
||||||
|
CapacityWait time.Duration
|
||||||
|
RequestDuration time.Duration
|
||||||
|
StartedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordAttempt runs after the attempt has released its permit, including when
|
||||||
|
// the recorder calls application code. Injected clients expose no HTTP facts.
|
||||||
|
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
|
||||||
|
outcome := result.Outcome
|
||||||
|
failure := ClassifyFailure(outcome.Err)
|
||||||
|
if capture {
|
||||||
|
attempt := AttemptEvidence{
|
||||||
|
StartedAt: result.StartedAt.UTC(), Dispatched: result.Dispatched,
|
||||||
|
CapacityWait: result.CapacityWait, RequestDuration: result.RequestDuration,
|
||||||
|
Failure: failure,
|
||||||
|
}
|
||||||
|
if injected {
|
||||||
|
attempt.ResponseReceived = outcome.Response != nil
|
||||||
|
attempt.GenerationCompleted = outcome.Err == nil && outcome.Response != nil
|
||||||
|
if result.Dispatched {
|
||||||
|
attempt.CapacityWait = 0
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
attempt.ResponseReceived = outcome.Evidence.ResponseReceived
|
||||||
|
attempt.HTTP = cloneHTTPEvidence(outcome.Evidence)
|
||||||
|
attempt.ResponseDecoded = outcome.ResponseDecoded
|
||||||
|
attempt.GenerationCompleted = outcome.GenerationCompleted
|
||||||
|
}
|
||||||
|
if outcome.StructuredOutputRequested != nil {
|
||||||
|
requested := *outcome.StructuredOutputRequested
|
||||||
|
attempt.StructuredOutputRequested = &requested
|
||||||
|
}
|
||||||
|
if result.Dispatched {
|
||||||
|
usage.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
var single observation.UsageAccumulator
|
||||||
|
single.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
attempt.Usage = single.Observed()
|
||||||
|
}
|
||||||
|
evidence.Attempts = append(evidence.Attempts, attempt)
|
||||||
|
}
|
||||||
|
if recorder != nil {
|
||||||
|
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||||
|
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
|
||||||
|
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
|
||||||
|
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) attemptResult {
|
||||||
|
result := attemptResult{StartedAt: c.now()}
|
||||||
|
release, err := c.manager.Acquire(ctx, backendID)
|
||||||
|
result.CapacityWait = c.capacityWait(backendID, result.StartedAt)
|
||||||
|
if err != nil {
|
||||||
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
dispatchStart := c.now()
|
dispatchStart := c.now()
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
outcome = prepared.ExecuteWithEvidence(ctx)
|
result.Outcome = prepared.ExecuteWithEvidence(ctx)
|
||||||
dispatched = outcome.Evidence.DispatchInvoked
|
result.Dispatched = result.Outcome.Evidence.DispatchInvoked
|
||||||
} else {
|
} else {
|
||||||
outcome = prepared.Execute(ctx)
|
result.Outcome = prepared.Execute(ctx)
|
||||||
dispatched = true
|
result.Dispatched = true
|
||||||
}
|
}
|
||||||
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt
|
result.RequestDuration = elapsed(dispatchStart, c.now())
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
||||||
|
|||||||
@@ -40,10 +40,7 @@ func testCoordinator(t *testing.T, transport transportFunc) *Coordinator {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
coordinator := NewBuiltIn(manager, client)
|
coordinator := NewBuiltIn(manager, client)
|
||||||
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
||||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
// GenerationEvidence contains invocation-local request evidence without raw
|
// GenerationEvidence contains invocation-local request evidence without raw
|
||||||
// provider bodies, headers, credentials, or generated content.
|
// provider bodies, headers, credentials, or generated content.
|
||||||
type GenerationEvidence struct {
|
type GenerationEvidence struct {
|
||||||
|
// PreparationFailed identifies a built-in request rejected before attempts.
|
||||||
|
PreparationFailed bool
|
||||||
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
||||||
// Injected generation leaves it unknown.
|
// Injected generation leaves it unknown.
|
||||||
InitialEndpoint string
|
InitialEndpoint string
|
||||||
|
|||||||
@@ -144,6 +144,32 @@ func (c fixedClient) Generate(context.Context, domain.GenerateRequest) (*domain.
|
|||||||
return c.response, c.err
|
return c.response, c.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInjectedObserverRunsAfterPermitRelease(t *testing.T) {
|
||||||
|
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP request"); return nil, nil })
|
||||||
|
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{}})
|
||||||
|
called := false
|
||||||
|
recorder, err := observation.New(func(event domain.ExecutionEvent) {
|
||||||
|
if event.Kind != domain.ExecutionAttempt {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
called = true
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
release, err := coordinator.manager.Acquire(ctx, "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("observer could not acquire permit: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := coordinator.Generate(observation.WithRecorder(context.Background(), recorder), testRequest(0)); err != nil || !called {
|
||||||
|
t.Fatalf("generation error = %v, observer called = %v", err, called)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
||||||
manager, err := capacity.NewManager(nil)
|
manager, err := capacity.NewManager(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
97
internal/jsonvalue/trusted.go
Normal file
97
internal/jsonvalue/trusted.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import "reflect"
|
||||||
|
|
||||||
|
func cloneTrustedMap(src map[string]any) map[string]any {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = CloneTrusted(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneTrusted detaches a previously accepted value without revalidating it.
|
||||||
|
func CloneTrusted(value any) any {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch v := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return cloneTrustedMap(v)
|
||||||
|
case []any:
|
||||||
|
out := make([]any, len(v))
|
||||||
|
for i, item := range v {
|
||||||
|
out[i] = CloneTrusted(item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case []string:
|
||||||
|
if v == nil {
|
||||||
|
return []string(nil)
|
||||||
|
}
|
||||||
|
return append([]string{}, v...)
|
||||||
|
case []byte:
|
||||||
|
if v == nil {
|
||||||
|
return []byte(nil)
|
||||||
|
}
|
||||||
|
return append([]byte{}, v...)
|
||||||
|
default:
|
||||||
|
return cloneTrustedReflect(reflect.ValueOf(value)).Interface()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneTrustedReflect(value reflect.Value) reflect.Value {
|
||||||
|
if !value.IsValid() {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Interface:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
copied := cloneTrustedReflect(value.Elem())
|
||||||
|
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||||
|
return copied
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
out.Set(copied)
|
||||||
|
return out
|
||||||
|
case reflect.Pointer:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type().Elem())
|
||||||
|
out.Elem().Set(cloneTrustedReflect(value.Elem()))
|
||||||
|
return out
|
||||||
|
case reflect.Map:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||||
|
iter := value.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
out.SetMapIndex(cloneTrustedReflect(iter.Key()), cloneTrustedReflect(iter.Value()))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Slice:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Array:
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
27
internal/jsonvalue/trusted_test.go
Normal file
27
internal/jsonvalue/trusted_test.go
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type trustedNamed string
|
||||||
|
|
||||||
|
func TestCloneTrustedPreservesTypesAndDetaches(t *testing.T) {
|
||||||
|
named := trustedNamed("value")
|
||||||
|
original := map[string]any{"": map[string]any{"number": json.Number("9007199254740993")}, "typed": []string{"original"}, "pointer": &named, "nilmap": map[string]any(nil), "nilslice": []string(nil), "empty": []any{}}
|
||||||
|
copy := CloneTrusted(original).(map[string]any)
|
||||||
|
if !reflect.DeepEqual(copy, original) {
|
||||||
|
t.Fatalf("clone = %#v", copy)
|
||||||
|
}
|
||||||
|
copy[""].(map[string]any)["number"] = json.Number("1")
|
||||||
|
copy["typed"].([]string)[0] = "changed"
|
||||||
|
*copy["pointer"].(*trustedNamed) = "changed"
|
||||||
|
if original[""].(map[string]any)["number"] != json.Number("9007199254740993") || original["typed"].([]string)[0] != "original" || named != "value" {
|
||||||
|
t.Fatal("clone shares mutable storage")
|
||||||
|
}
|
||||||
|
if copy["nilmap"].(map[string]any) != nil || copy["nilslice"].([]string) != nil || copy["empty"].([]any) == nil {
|
||||||
|
t.Fatal("nil/empty shape changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,10 +25,7 @@ func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
|
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
|
||||||
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
|
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -88,7 +85,7 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
||||||
var authorization string
|
var authorization string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
authorization = request.Header.Get("Authorization")
|
authorization = request.Header.Get("Authorization")
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -98,9 +95,6 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, evidence, err := client.GetWithAuthenticationEvidence(
|
response, evidence, err := client.GetWithAuthenticationEvidence(
|
||||||
context.Background(), "https://example.invalid/key", 0, tc.target,
|
context.Background(), "https://example.invalid/key", 0, tc.target,
|
||||||
@@ -269,10 +263,7 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
|
|||||||
|
|
||||||
transport := http.DefaultTransport
|
transport := http.DefaultTransport
|
||||||
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
|
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
const calls = 8
|
const calls = 8
|
||||||
evidences := make([]HTTPExecutionEvidence, calls)
|
evidences := make([]HTTPExecutionEvidence, calls)
|
||||||
@@ -307,15 +298,12 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
|
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
|
||||||
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
<-request.Context().Done()
|
<-request.Context().Done()
|
||||||
return nil, request.Context().Err()
|
return nil, request.Context().Err()
|
||||||
}),
|
}),
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
|
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
|
||||||
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
|
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
|||||||
@@ -17,10 +17,10 @@ import (
|
|||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
|
||||||
ErrInvalidRequest = errors.New("invalid generate request")
|
ErrInvalidRequest = errors.New("invalid generate request")
|
||||||
ErrRequestFailed = errors.New("llm request failed")
|
ErrRequestFailed = errors.New("llm request failed")
|
||||||
ErrTransport = errors.New("llm transport failed")
|
ErrTransport = errors.New("llm transport failed")
|
||||||
@@ -43,28 +43,15 @@ func (e *requestFailedError) Unwrap() []error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type OpenAICompatibleConfig struct {
|
type OpenAICompatibleConfig struct {
|
||||||
BaseURL string
|
|
||||||
Model string
|
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
type OpenAICompatibleClient struct {
|
type OpenAICompatibleClient struct {
|
||||||
baseURL string
|
httpClient *http.Client
|
||||||
defaultModel string
|
|
||||||
httpClient *http.Client
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) *OpenAICompatibleClient {
|
||||||
baseURL := ""
|
|
||||||
if strings.TrimSpace(cfg.BaseURL) != "" {
|
|
||||||
var err error
|
|
||||||
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout := cfg.Timeout
|
timeout := cfg.Timeout
|
||||||
if timeout <= 0 {
|
if timeout <= 0 {
|
||||||
timeout = defaults.LLMRequestTimeoutDefault
|
timeout = defaults.LLMRequestTimeoutDefault
|
||||||
@@ -82,10 +69,8 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &OpenAICompatibleClient{
|
return &OpenAICompatibleClient{
|
||||||
baseURL: baseURL,
|
httpClient: client,
|
||||||
defaultModel: cfg.Model,
|
}
|
||||||
httpClient: client,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
@@ -132,11 +117,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
selectedEndpoint := req.Target.Endpoint
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(req.Target.Endpoint)
|
||||||
if strings.TrimSpace(selectedEndpoint) == "" {
|
|
||||||
selectedEndpoint = c.baseURL
|
|
||||||
}
|
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -145,12 +126,10 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
// Custom JSON encoders may mutate their receivers. Both payload encoding
|
||||||
if err != nil {
|
// and encoding-failure diagnostics must use values detached from metadata.
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
req.Target.ExtraParams = jsonvalue.CloneTrusted(req.Target.ExtraParams).(map[string]any)
|
||||||
}
|
wirePayload, err := openAIChatRequestPayload(req)
|
||||||
|
|
||||||
wirePayload, err := openAIChatRequestPayload(wireReq)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -179,7 +158,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return &PreparedGeneration{
|
return &PreparedGeneration{
|
||||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||||
structuredOutputRequested: wireReq.ResponseFormat != nil,
|
structuredOutputRequested: wirePayload["response_format"] != nil,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -370,40 +349,69 @@ func tokenCount(value *int) int {
|
|||||||
return *value
|
return *value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type responseErrorReader struct {
|
||||||
|
reader io.Reader
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *responseErrorReader) Read(p []byte) (int, error) {
|
||||||
|
n, err := r.reader.Read(p)
|
||||||
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
r.err = err
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||||
|
observed := &responseErrorReader{reader: body}
|
||||||
limited := &io.LimitedReader{
|
limited := &io.LimitedReader{
|
||||||
R: body,
|
R: observed,
|
||||||
N: maxOpenAIChatResponseBytes + 1,
|
N: maxOpenAIChatResponseBytes + 1,
|
||||||
}
|
}
|
||||||
decoder := json.NewDecoder(limited)
|
decoder := json.NewDecoder(limited)
|
||||||
|
|
||||||
var response openAIChatResponse
|
var response openAIChatResponse
|
||||||
if err := decoder.Decode(&response); err != nil {
|
if err := decoder.Decode(&response); err != nil {
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
}
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
|
||||||
var trailing any
|
var trailing any
|
||||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
}
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
|
|
||||||
return response, nil
|
return response, nil
|
||||||
}
|
}
|
||||||
@@ -416,99 +424,52 @@ func openAIChatResponseTooLargeError() error {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
func openAIChatRequestPayload(req domain.GenerateRequest) (map[string]any, error) {
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
model := strings.TrimSpace(req.Target.Model)
|
||||||
if model == "" {
|
if model == "" {
|
||||||
model = strings.TrimSpace(defaultModel)
|
return nil, errors.New("model is required")
|
||||||
}
|
|
||||||
if model == "" {
|
|
||||||
return openAIChatRequest{}, errors.New("model is required")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
wireReq := openAIChatRequest{
|
|
||||||
Model: model,
|
|
||||||
}
|
|
||||||
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return openAIChatRequest{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
wireReq.SessionID = sessionID
|
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
messages := make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||||
for _, msg := range req.Prompt.Messages {
|
for _, msg := range req.Prompt.Messages {
|
||||||
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
messages = append(messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
out := map[string]any{"model": model, "messages": messages}
|
||||||
|
if sessionID != "" {
|
||||||
|
out["session_id"] = sessionID
|
||||||
|
}
|
||||||
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
||||||
wireReq.Temperature = &req.Target.Temperature
|
out["temperature"] = req.Target.Temperature
|
||||||
}
|
}
|
||||||
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
||||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
out["max_tokens"] = req.Target.MaxTokens
|
||||||
}
|
}
|
||||||
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
||||||
wireReq.TopP = &req.Target.TopP
|
out["top_p"] = req.Target.TopP
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
||||||
wireReq.ServiceTier = req.Target.ServiceTier
|
out["service_tier"] = req.Target.ServiceTier
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
||||||
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
out["reasoning_effort"] = req.Target.ReasoningEffort
|
||||||
}
|
|
||||||
if len(req.Target.ExtraParams) > 0 {
|
|
||||||
wireReq.ExtraParams = req.Target.ExtraParams
|
|
||||||
}
|
}
|
||||||
if req.StructuredOutput != nil {
|
if req.StructuredOutput != nil {
|
||||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return openAIChatRequest{}, err
|
return nil, err
|
||||||
|
}
|
||||||
|
if responseFormat != nil {
|
||||||
|
out["response_format"] = responseFormat
|
||||||
}
|
}
|
||||||
wireReq.ResponseFormat = responseFormat
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return wireReq, nil
|
for key, value := range req.Target.ExtraParams {
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
|
||||||
Model string `json:"model"`
|
|
||||||
SessionID string `json:"session_id,omitempty"`
|
|
||||||
Messages []openAIChatRequestMessage `json:"messages"`
|
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
||||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
|
||||||
ExtraParams map[string]any `json:"-"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|
||||||
out := map[string]any{
|
|
||||||
"model": req.Model,
|
|
||||||
"messages": req.Messages,
|
|
||||||
}
|
|
||||||
if req.SessionID != "" {
|
|
||||||
out["session_id"] = req.SessionID
|
|
||||||
}
|
|
||||||
if req.Temperature != nil {
|
|
||||||
out["temperature"] = *req.Temperature
|
|
||||||
}
|
|
||||||
if req.MaxTokens != nil {
|
|
||||||
out["max_tokens"] = *req.MaxTokens
|
|
||||||
}
|
|
||||||
if req.TopP != nil {
|
|
||||||
out["top_p"] = *req.TopP
|
|
||||||
}
|
|
||||||
if req.ServiceTier != "" {
|
|
||||||
out["service_tier"] = req.ServiceTier
|
|
||||||
}
|
|
||||||
if req.ReasoningEffort != "" {
|
|
||||||
out["reasoning_effort"] = req.ReasoningEffort
|
|
||||||
}
|
|
||||||
if req.ResponseFormat != nil {
|
|
||||||
out["response_format"] = req.ResponseFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
for key, value := range req.ExtraParams {
|
|
||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, errors.New("extra_params key must not be empty")
|
return nil, errors.New("extra_params key must not be empty")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -123,26 +123,19 @@ func (p *recordingProvider) lastRequest(t *testing.T) recordedProviderRequest {
|
|||||||
return p.requests[len(p.requests)-1]
|
return p.requests[len(p.requests)-1]
|
||||||
}
|
}
|
||||||
|
|
||||||
func newProviderClient(t *testing.T, provider *recordingProvider, config OpenAICompatibleConfig) *OpenAICompatibleClient {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if config.BaseURL == "" {
|
|
||||||
config.BaseURL = provider.endpoint("/v1")
|
|
||||||
}
|
|
||||||
client, err := NewOpenAICompatibleClient(config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
|
|
||||||
func ordinaryGenerateRequest() domain.GenerateRequest {
|
func ordinaryGenerateRequest() domain.GenerateRequest {
|
||||||
return domain.GenerateRequest{
|
return domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "model"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *recordingProvider) generateRequest() domain.GenerateRequest {
|
||||||
|
req := ordinaryGenerateRequest()
|
||||||
|
req.Target.Endpoint = p.endpoint("/v1")
|
||||||
|
return req
|
||||||
|
}
|
||||||
|
|
||||||
type deadlineCapturingTransport struct {
|
type deadlineCapturingTransport struct {
|
||||||
deadline time.Time
|
deadline time.Time
|
||||||
hasDeadline bool
|
hasDeadline bool
|
||||||
@@ -253,35 +246,13 @@ func assertDeadlineNear(t *testing.T, deadline, before, after time.Time, duratio
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewOpenAICompatibleClientRejectsInvalidBaseURL(t *testing.T) {
|
|
||||||
for _, endpoint := range []string{
|
|
||||||
"://invalid",
|
|
||||||
"/v1",
|
|
||||||
"https:///v1",
|
|
||||||
"ftp://provider.example/v1",
|
|
||||||
"https://user@provider.example/v1",
|
|
||||||
"https://provider.example/v1?mode=chat",
|
|
||||||
"https://provider.example/v1#chat",
|
|
||||||
} {
|
|
||||||
t.Run(endpoint, func(t *testing.T) {
|
|
||||||
_, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: endpoint})
|
|
||||||
if !errors.Is(err, ErrInvalidConfig) {
|
|
||||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
|
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
|
||||||
transport := http.DefaultTransport
|
transport := http.DefaultTransport
|
||||||
supplied := &http.Client{Transport: transport}
|
supplied := &http.Client{Transport: transport}
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != 0 {
|
if supplied.Timeout != 0 {
|
||||||
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
|
||||||
@@ -305,13 +276,10 @@ func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *t
|
|||||||
Transport: transport,
|
Transport: transport,
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: 2 * time.Second,
|
Timeout: 2 * time.Second,
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != suppliedTimeout {
|
if supplied.Timeout != suppliedTimeout {
|
||||||
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
|
||||||
@@ -335,13 +303,10 @@ func TestNewOpenAICompatibleClientTreatsSuppliedNegativeTimeoutAsUnset(t *testin
|
|||||||
}
|
}
|
||||||
configuredTimeout := 23 * time.Second
|
configuredTimeout := 23 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: configuredTimeout,
|
Timeout: configuredTimeout,
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != -time.Second {
|
if supplied.Timeout != -time.Second {
|
||||||
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
|
||||||
@@ -370,7 +335,6 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
|||||||
{name: "optional response fields", run: checkOptionalResponseFieldOmission},
|
{name: "optional response fields", run: checkOptionalResponseFieldOmission},
|
||||||
{name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping},
|
{name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping},
|
||||||
{name: "optional request field presence", run: checkRequestFieldPresence},
|
{name: "optional request field presence", run: checkRequestFieldPresence},
|
||||||
{name: "configured model fallback", run: checkConfiguredModelFallback},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -380,14 +344,14 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
|||||||
|
|
||||||
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
|
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
const content = " exact content\nwith whitespace "
|
const content = " exact content\nwith whitespace "
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||||
Role: domain.RoleDeveloper,
|
Role: domain.RoleDeveloper,
|
||||||
Content: content,
|
Content: content,
|
||||||
}}},
|
}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Generate() error = %v", err)
|
t.Fatalf("Generate() error = %v", err)
|
||||||
@@ -413,7 +377,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
|
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
|
||||||
}`)
|
}`)
|
||||||
|
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: 2 * time.Second,
|
Timeout: 2 * time.Second,
|
||||||
})
|
})
|
||||||
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key")
|
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key")
|
||||||
@@ -424,6 +388,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
{Role: "user", Content: "Say hello"},
|
{Role: "user", Content: "Say hello"},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: provider.endpoint("/v1"),
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
Temperature: 0.4,
|
Temperature: 0.4,
|
||||||
MaxTokens: 123,
|
MaxTokens: 123,
|
||||||
@@ -594,9 +559,11 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
tc.configureEnv(t)
|
tc.configureEnv(t)
|
||||||
}
|
}
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Target = tc.target
|
request.Target = tc.target
|
||||||
|
request.Target.Endpoint = provider.endpoint("/v1")
|
||||||
|
request.Target.Model = "model"
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), request)
|
_, err := client.Generate(context.Background(), request)
|
||||||
if tc.wantError != nil {
|
if tc.wantError != nil {
|
||||||
@@ -625,7 +592,7 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
|
|
||||||
func checkCacheControlledMessageMapping(t *testing.T) {
|
func checkCacheControlledMessageMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
@@ -639,7 +606,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{Role: "user", Content: "Dynamic request."},
|
{Role: "user", Content: "Dynamic request."},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -691,7 +658,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
|
|||||||
|
|
||||||
func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
@@ -703,7 +670,7 @@ func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -745,8 +712,8 @@ func checkSessionIDMapping(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Prompt.SessionID = tc.sessionID
|
request.Prompt.SessionID = tc.sessionID
|
||||||
|
|
||||||
if _, err := client.Generate(context.Background(), request); err != nil {
|
if _, err := client.Generate(context.Background(), request); err != nil {
|
||||||
@@ -766,19 +733,14 @@ func checkSessionIDMapping(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "model",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{
|
Prompt: domain.RenderedPrompt{
|
||||||
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
},
|
},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected invalid request error")
|
t.Fatal("expected invalid request error")
|
||||||
@@ -828,9 +790,9 @@ func checkContentPresence(t *testing.T) {
|
|||||||
"cache_write_tokens": 5
|
"cache_write_tokens": 5
|
||||||
}
|
}
|
||||||
}`)
|
}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
response, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
}
|
}
|
||||||
@@ -865,10 +827,11 @@ func checkCacheUsageMapping(t *testing.T) {
|
|||||||
"cache_write_tokens": 60
|
"cache_write_tokens": 60
|
||||||
}
|
}
|
||||||
}`)
|
}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -901,8 +864,8 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
|
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
resp, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
resp, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -913,17 +876,17 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
|
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
if resp, err := client.Generate(context.Background(), ordinaryGenerateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
|
if resp, err := client.Generate(context.Background(), provider.generateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
|
||||||
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
|
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
_, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -938,11 +901,12 @@ func checkOptionalResponseFieldOmission(t *testing.T) {
|
|||||||
|
|
||||||
func checkReasoningAndExtraParameterMapping(t *testing.T) {
|
func checkReasoningAndExtraParameterMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: provider.endpoint("/v1"),
|
||||||
Model: "model",
|
Model: "model",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: "high",
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
@@ -1020,8 +984,8 @@ func checkRequestFieldPresence(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.TargetPresence = tc.presence
|
request.TargetPresence = tc.presence
|
||||||
|
|
||||||
if _, err := client.Generate(context.Background(), request); err != nil {
|
if _, err := client.Generate(context.Background(), request); err != nil {
|
||||||
@@ -1073,21 +1037,17 @@ func checkOmittedGenerationTimeout(t *testing.T) {
|
|||||||
transport := &deadlineCapturingTransport{}
|
transport := &deadlineCapturingTransport{}
|
||||||
clientTimeout := 5 * time.Second
|
clientTimeout := 5 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Timeout: clientTimeout,
|
Timeout: clientTimeout,
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model", TimeoutSeconds: 0},
|
||||||
})
|
})
|
||||||
after := time.Now()
|
after := time.Now()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -1146,11 +1106,11 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model", ExtraParams: tc.extraParams},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected invalid request error")
|
t.Fatal("expected invalid request error")
|
||||||
@@ -1192,8 +1152,8 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Target.ExtraParams = tc.extraParams
|
request.Target.ExtraParams = tc.extraParams
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
@@ -1218,30 +1178,12 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkConfiguredModelFallback(t *testing.T) {
|
|
||||||
provider := newRecordingProvider(t)
|
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
var model string
|
|
||||||
if !provider.lastRequest(t).decodeField(t, "model", &model) || model != "default-model" {
|
|
||||||
t.Fatalf("model = %q, want default-model", model)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
run func(*testing.T)
|
run func(*testing.T)
|
||||||
}{
|
}{
|
||||||
{name: "request endpoint overrides configured endpoint", run: checkEndpointOverride},
|
{name: "request endpoint", run: checkSelectedEndpoint},
|
||||||
{name: "request endpoint works without configured endpoint", run: checkEmptyConfiguredEndpoint},
|
|
||||||
{name: "completion URL preserves valid base paths", run: checkCompletionURLComposition},
|
{name: "completion URL preserves valid base paths", run: checkCompletionURLComposition},
|
||||||
{name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection},
|
{name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection},
|
||||||
{name: "an endpoint is required", run: checkRequiredEndpoint},
|
{name: "an endpoint is required", run: checkRequiredEndpoint},
|
||||||
@@ -1252,35 +1194,6 @@ func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkEndpointOverride(t *testing.T) {
|
|
||||||
configuredProvider := newRecordingProvider(t)
|
|
||||||
configuredProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"default"}}]}`)
|
|
||||||
selectedProvider := newRecordingProvider(t)
|
|
||||||
selectedProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"override"}}]}`)
|
|
||||||
|
|
||||||
client := newProviderClient(t, configuredProvider, OpenAICompatibleConfig{Model: "m"})
|
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{Endpoint: selectedProvider.endpoint("/v1")},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if resp.Content != "override" {
|
|
||||||
t.Fatalf("expected override response, got %q", resp.Content)
|
|
||||||
}
|
|
||||||
if got := configuredProvider.requestCount(); got != 0 {
|
|
||||||
t.Fatalf("configured provider calls = %d, want 0", got)
|
|
||||||
}
|
|
||||||
if got := selectedProvider.requestCount(); got != 1 {
|
|
||||||
t.Fatalf("selected provider calls = %d, want 1", got)
|
|
||||||
}
|
|
||||||
if got := selectedProvider.lastRequest(t).url; got != "/v1/chat/completions" {
|
|
||||||
t.Fatalf("selected URL = %q, want /v1/chat/completions", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkCommonResponseFailures(t *testing.T) {
|
func checkCommonResponseFailures(t *testing.T) {
|
||||||
const sensitiveBody = `provider-secret-fragment request_payload_details`
|
const sensitiveBody = `provider-secret-fragment request_payload_details`
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -1310,9 +1223,9 @@ func checkCommonResponseFailures(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(tc.statusCode, tc.body)
|
provider.respond(tc.statusCode, tc.body)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "m"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
response, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if response != nil {
|
if response != nil {
|
||||||
t.Fatalf("expected no partial response, got %+v", response)
|
t.Fatalf("expected no partial response, got %+v", response)
|
||||||
}
|
}
|
||||||
@@ -1357,9 +1270,7 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
body := sizedSuccessResponseBody(tc.size)
|
body := sizedSuccessResponseBody(tc.size)
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1370,12 +1281,10 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if tc.wantErr {
|
if tc.wantErr {
|
||||||
if response != nil {
|
if response != nil {
|
||||||
@@ -1425,9 +1334,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
strings.NewReader(prefix),
|
strings.NewReader(prefix),
|
||||||
continuation,
|
continuation,
|
||||||
)}
|
)}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1438,9 +1345,6 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type outcome struct {
|
type outcome struct {
|
||||||
response *domain.GenerateResponse
|
response *domain.GenerateResponse
|
||||||
@@ -1450,6 +1354,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
go func() {
|
go func() {
|
||||||
response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{
|
response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
done <- outcome{response: response, err: generateErr}
|
done <- outcome{response: response, err: generateErr}
|
||||||
}()
|
}()
|
||||||
@@ -1497,9 +1402,7 @@ func checkSingleResponseDocument(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
body := &countingReadCloser{reader: strings.NewReader(tc.body)}
|
body := &countingReadCloser{reader: strings.NewReader(tc.body)}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1510,12 +1413,10 @@ func checkSingleResponseDocument(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if tc.wantErr {
|
if tc.wantErr {
|
||||||
if response != nil {
|
if response != nil {
|
||||||
@@ -1541,22 +1442,19 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
|
|||||||
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
||||||
generationTimeout := 2 * time.Second
|
generationTimeout := 2 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: 10 * time.Second,
|
Timeout: 10 * time.Second,
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://example.com/v1",
|
||||||
|
Model: "m",
|
||||||
TimeoutSeconds: int(generationTimeout / time.Second),
|
TimeoutSeconds: int(generationTimeout / time.Second),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -1578,25 +1476,22 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
|
|||||||
|
|
||||||
func checkCallerDeadlinePrecedence(t *testing.T) {
|
func checkCallerDeadlinePrecedence(t *testing.T) {
|
||||||
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: 10 * time.Second,
|
Timeout: 10 * time.Second,
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
callerDeadline := time.Now().Add(time.Second)
|
callerDeadline := time.Now().Add(time.Second)
|
||||||
ctx, cancel := context.WithDeadline(context.Background(), callerDeadline)
|
ctx, cancel := context.WithDeadline(context.Background(), callerDeadline)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://example.com/v1",
|
||||||
|
Model: "m",
|
||||||
TimeoutSeconds: 2,
|
TimeoutSeconds: 2,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -1618,22 +1513,18 @@ func checkCallerDeadlinePrecedence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkCallerCancellationIdentity(t *testing.T) {
|
func checkCallerCancellationIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: contextErrorTransport{},
|
Transport: contextErrorTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected cancellation error")
|
t.Fatal("expected cancellation error")
|
||||||
@@ -1647,21 +1538,17 @@ func checkCallerCancellationIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: contextErrorTransport{},
|
Transport: contextErrorTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0))
|
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0))
|
||||||
defer cancel()
|
defer cancel()
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
t.Fatalf("expected request failure and caller deadline identities, got %v", err)
|
t.Fatalf("expected request failure and caller deadline identities, got %v", err)
|
||||||
@@ -1669,20 +1556,16 @@ func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkWholeRequestTimeoutIdentity(t *testing.T) {
|
func checkWholeRequestTimeoutIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: time.Millisecond,
|
Timeout: time.Millisecond,
|
||||||
Transport: waitingContextTransport{},
|
Transport: waitingContextTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
t.Fatalf("expected request failure and client timeout identities, got %v", err)
|
t.Fatalf("expected request failure and client timeout identities, got %v", err)
|
||||||
@@ -1696,20 +1579,15 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
|
|||||||
apiKey = "sensitive-api-key"
|
apiKey = "sensitive-api-key"
|
||||||
content = "sensitive prompt content"
|
content = "sensitive prompt content"
|
||||||
)
|
)
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: endpoint,
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return nil, transportCause
|
return nil, transportCause
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}},
|
||||||
Target: domain.ExecutionTarget{APIKey: apiKey},
|
Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m", APIKey: apiKey},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) {
|
||||||
t.Fatalf("expected request failure and transport cause identities, got %v", err)
|
t.Fatalf("expected request failure and transport cause identities, got %v", err)
|
||||||
@@ -1726,13 +1604,7 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type testCase struct {
|
type testCase struct {
|
||||||
name string
|
name string
|
||||||
@@ -1765,11 +1637,9 @@ func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkEmptyConfiguredEndpoint(t *testing.T) {
|
func checkSelectedEndpoint(t *testing.T) {
|
||||||
var selectedURL string
|
var selectedURL string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
selectedURL = req.URL.String()
|
selectedURL = req.URL.String()
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -1782,13 +1652,10 @@ func checkEmptyConfiguredEndpoint(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected empty configured base URL to be allowed, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1"},
|
Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate with request endpoint: %v", err)
|
t.Fatalf("generate with request endpoint: %v", err)
|
||||||
@@ -1817,9 +1684,7 @@ func checkCompletionURLComposition(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
var selectedURL string
|
var selectedURL string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: tc.baseURL,
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
selectedURL = req.URL.String()
|
selectedURL = req.URL.String()
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -1830,12 +1695,10 @@ func checkCompletionURLComposition(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: tc.baseURL, Model: "m"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
@@ -1860,21 +1723,16 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
|
|||||||
for _, endpoint := range invalidEndpoints {
|
for _, endpoint := range invalidEndpoints {
|
||||||
t.Run(endpoint, func(t *testing.T) {
|
t.Run(endpoint, func(t *testing.T) {
|
||||||
transportCalls := 0
|
transportCalls := 0
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://configured.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
transportCalls++
|
transportCalls++
|
||||||
return nil, errors.New("transport must not be called")
|
return nil, errors.New("transport must not be called")
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Endpoint: endpoint},
|
Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
@@ -1890,17 +1748,11 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkRequiredEndpoint(t *testing.T) {
|
func checkRequiredEndpoint(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{},
|
Target: domain.ExecutionTarget{Model: "m"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected endpoint-required error")
|
t.Fatal("expected endpoint-required error")
|
||||||
@@ -1909,3 +1761,13 @@ func checkRequiredEndpoint(t *testing.T) {
|
|||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientRequiresModel(t *testing.T) {
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
_, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1"},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("missing model error = %v, want invalid request", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
108
internal/llm/openrouter_authentication.go
Normal file
108
internal/llm/openrouter_authentication.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxAuthenticationResponseBytes = 1 << 20
|
||||||
|
|
||||||
|
// AuthenticationEvidence describes the observable result of a key probe.
|
||||||
|
type AuthenticationEvidence struct {
|
||||||
|
HTTP HTTPExecutionEvidence
|
||||||
|
Accepted *bool
|
||||||
|
Overflow bool
|
||||||
|
Unrecognized bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRouterKeyEndpoint derives the default key endpoint from a base endpoint.
|
||||||
|
func OpenRouterKeyEndpoint(base string) (string, error) {
|
||||||
|
return url.JoinPath(base, "key")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProbeOpenRouterKey executes and interprets an OpenRouter key request.
|
||||||
|
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
|
||||||
|
parsed, err := url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return AuthenticationEvidence{}, err
|
||||||
|
}
|
||||||
|
parsed.User = nil
|
||||||
|
response, evidence, requestErr := c.GetWithAuthenticationEvidence(ctx, parsed.String(), timeoutSeconds, target)
|
||||||
|
result := AuthenticationEvidence{HTTP: evidence}
|
||||||
|
if response != nil && response.Body != nil {
|
||||||
|
defer response.Body.Close()
|
||||||
|
}
|
||||||
|
if !evidence.ResponseReceived && response == nil {
|
||||||
|
return result, requestErr
|
||||||
|
}
|
||||||
|
status := evidence.HTTPStatus
|
||||||
|
if status == nil && response != nil {
|
||||||
|
status = &response.StatusCode
|
||||||
|
}
|
||||||
|
if status == nil {
|
||||||
|
result.Unrecognized = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
switch *status {
|
||||||
|
case http.StatusUnauthorized:
|
||||||
|
accepted := false
|
||||||
|
result.Accepted = &accepted
|
||||||
|
case http.StatusOK:
|
||||||
|
if response == nil || response.Body == nil {
|
||||||
|
result.Unrecognized = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
contents, err := io.ReadAll(io.LimitReader(response.Body, maxAuthenticationResponseBytes+1))
|
||||||
|
if err != nil && ctx.Err() != nil {
|
||||||
|
return result, ctx.Err()
|
||||||
|
}
|
||||||
|
if len(contents) > maxAuthenticationResponseBytes {
|
||||||
|
result.Overflow = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
if recognizedOpenRouterKeyResponse(contents) {
|
||||||
|
accepted := true
|
||||||
|
result.Accepted = &accepted
|
||||||
|
} else {
|
||||||
|
result.Unrecognized = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recognizedOpenRouterKeyResponse(contents []byte) bool {
|
||||||
|
var envelope map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var data map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(envelope["data"], &data); err != nil || data == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
label := bytes.TrimSpace(data["label"])
|
||||||
|
var labelValue string
|
||||||
|
if len(label) == 0 || label[0] != '"' || json.Unmarshal(label, &labelValue) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
freeTier := bytes.TrimSpace(data["is_free_tier"])
|
||||||
|
if !bytes.Equal(freeTier, []byte("true")) && !bytes.Equal(freeTier, []byte("false")) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data["usage"]))
|
||||||
|
decoder.UseNumber()
|
||||||
|
var number any
|
||||||
|
if decoder.Decode(&number) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, ok := number.(json.Number)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
134
internal/llm/openrouter_authentication_test.go
Normal file
134
internal/llm/openrouter_authentication_test.go
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKey(t *testing.T) {
|
||||||
|
valid := `{"data":{"label":"x","is_free_tier":false,"usage":0}}`
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
body string
|
||||||
|
key string
|
||||||
|
accepted *bool
|
||||||
|
unrecognized, overflow bool
|
||||||
|
}{
|
||||||
|
{name: "recognized", status: 200, body: valid, key: "secret", accepted: boolValue(true)},
|
||||||
|
{name: "exact limit", status: 200, body: valid + strings.Repeat(" ", maxAuthenticationResponseBytes-len(valid)), accepted: boolValue(true)},
|
||||||
|
{name: "unknown", status: 200, body: `{"data":{"label":"x","is_free_tier":false,"usage":"0"}}`, unrecognized: true},
|
||||||
|
{name: "trailing value", status: 200, body: valid + ` {}`, unrecognized: true},
|
||||||
|
{name: "overflow", status: 200, body: strings.Repeat("x", maxAuthenticationResponseBytes+1), overflow: true},
|
||||||
|
{name: "unauthorized anonymous", status: 401, accepted: boolValue(false)},
|
||||||
|
{name: "other status", status: 503},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var authorization string
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
if req.URL.User != nil {
|
||||||
|
t.Fatal("request retained URL credentials")
|
||||||
|
}
|
||||||
|
authorization = req.Header.Get("Authorization")
|
||||||
|
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.HTTP.Endpoint != "https://example.invalid/key" || got.HTTP.HTTPStatus == nil || *got.HTTP.HTTPStatus != tc.status || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow {
|
||||||
|
t.Fatalf("evidence = %+v", got)
|
||||||
|
}
|
||||||
|
if (got.Accepted == nil) != (tc.accepted == nil) || got.Accepted != nil && *got.Accepted != *tc.accepted {
|
||||||
|
t.Fatalf("acceptance = %v", got.Accepted)
|
||||||
|
}
|
||||||
|
if (authorization != "") != (tc.key != "") || got.HTTP.CredentialsSupplied != (tc.key != "") {
|
||||||
|
t.Fatalf("credential evidence = %+v", got.HTTP)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolValue(value bool) *bool { return &value }
|
||||||
|
|
||||||
|
type authenticationFailingBody struct{ err error }
|
||||||
|
|
||||||
|
func (b authenticationFailingBody) Read([]byte) (int, error) { return 0, b.err }
|
||||||
|
func (b authenticationFailingBody) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
|
||||||
|
want := errors.New("interrupted")
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
|
||||||
|
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type observedAuthenticationBody struct {
|
||||||
|
reader io.Reader
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *observedAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n, err := b.reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *observedAuthenticationBody) Close() error {
|
||||||
|
b.closed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
wantRead int
|
||||||
|
}{
|
||||||
|
{"overflow", http.StatusOK, maxAuthenticationResponseBytes + 1},
|
||||||
|
{"unauthorized", http.StatusUnauthorized, 0},
|
||||||
|
{"other status", http.StatusServiceUnavailable, 0},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
body := &observedAuthenticationBody{reader: strings.NewReader(strings.Repeat("x", 2*maxAuthenticationResponseBytes))}
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
|
||||||
|
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenRouterKeyEndpoint(t *testing.T) {
|
||||||
|
got, err := OpenRouterKeyEndpoint("https://example.invalid/api/v1/")
|
||||||
|
if err != nil || got != "https://example.invalid/api/v1/key" {
|
||||||
|
t.Fatalf("endpoint = %q, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
<-req.Context().Done()
|
||||||
|
return nil, req.Context().Err()
|
||||||
|
})}})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
|
||||||
|
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,13 +12,7 @@ import (
|
|||||||
var prepareGenerationBenchmarkSink *PreparedGeneration
|
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||||
|
|
||||||
func BenchmarkPrepareGeneration(b *testing.B) {
|
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "https://api.example.test/v1",
|
|
||||||
Model: "benchmark-model",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("construct benchmark client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prompt := domain.RenderedPrompt{
|
prompt := domain.RenderedPrompt{
|
||||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||||
@@ -31,14 +25,14 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "no_extras",
|
name: "no_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "small_extras",
|
name: "small_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"priority": "low",
|
"priority": "low",
|
||||||
"attempt": 3,
|
"attempt": 3,
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
@@ -49,7 +43,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "1MiB_string_extra",
|
name: "1MiB_string_extra",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"payload": strings.Repeat("x", 1<<20),
|
"payload": strings.Repeat("x", 1<<20),
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
@@ -58,7 +52,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "nested_json_extras",
|
name: "nested_json_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"options": map[string]any{
|
"options": map[string]any{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"limits": []any{2, 4, 8},
|
"limits": []any{2, 4, 8},
|
||||||
@@ -71,7 +65,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "structured_output",
|
name: "structured_output",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
|
||||||
StructuredOutput: &domain.StructuredOutputSpec{
|
StructuredOutput: &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
|||||||
@@ -73,9 +73,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
response := responseFor(request, test.status, "")
|
response := responseFor(request, test.status, "")
|
||||||
response.Body = io.NopCloser(strings.NewReader(test.body))
|
response.Body = io.NopCloser(strings.NewReader(test.body))
|
||||||
@@ -83,9 +81,6 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
return response, nil
|
return response, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
request := ordinaryGenerateRequest()
|
request := ordinaryGenerateRequest()
|
||||||
if test.structured {
|
if test.structured {
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
@@ -136,9 +131,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
|
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
|
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -149,9 +142,6 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
|
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -174,16 +164,11 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
|
|||||||
|
|
||||||
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
|
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
|
||||||
transportErr := errors.New("provider unavailable")
|
transportErr := errors.New("provider unavailable")
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return nil, transportErr
|
return nil, transportErr
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
request := ordinaryGenerateRequest()
|
request := ordinaryGenerateRequest()
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
|||||||
@@ -26,8 +26,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
|
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
|
||||||
t.Setenv(envName, tc.initialKey)
|
t.Setenv(envName, tc.initialKey)
|
||||||
var bodies, auth, urls []string
|
var bodies, auth, urls []string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "default-model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
body, err := io.ReadAll(req.Body)
|
body, err := io.ReadAll(req.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -39,12 +38,9 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req := domain.GenerateRequest{
|
req := domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
|
||||||
Target: domain.ExecutionTarget{APIKeyEnv: envName},
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "default-model", APIKeyEnv: envName},
|
||||||
}
|
}
|
||||||
prepared, err := client.PrepareGeneration(req)
|
prepared, err := client.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -68,8 +64,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
|
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
|
||||||
var contexts []context.Context
|
var contexts []context.Context
|
||||||
var closed int
|
var closed int
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
if err := req.Context().Err(); err != nil {
|
if err := req.Context().Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -81,12 +76,9 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
|
|||||||
}}, nil
|
}}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{TimeoutSeconds: 1},
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model", TimeoutSeconds: 1},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -115,17 +107,14 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -137,17 +126,14 @@ func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -160,17 +146,14 @@ func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
|||||||
|
|
||||||
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
|
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
|
||||||
closed := false
|
closed := false
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -127,9 +127,7 @@ func TestOpenAICompatibleClientNonSuccessBodyOwnership(t *testing.T) {
|
|||||||
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: statusCode,
|
StatusCode: statusCode,
|
||||||
@@ -138,8 +136,5 @@ func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
return client
|
return client
|
||||||
}
|
}
|
||||||
|
|||||||
48
internal/llm/response_read_error_test.go
Normal file
48
internal/llm/response_read_error_test.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type responseBytesAndError struct {
|
||||||
|
data []byte
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *responseBytesAndError) Read(p []byte) (int, error) {
|
||||||
|
if len(r.data) == 0 {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
n := copy(p, r.data)
|
||||||
|
r.data = r.data[n:]
|
||||||
|
if len(r.data) == 0 {
|
||||||
|
return n, r.err
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSuccessfulResponseReadFailuresRetainCause(t *testing.T) {
|
||||||
|
cause := errors.New("private reader diagnostic")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, prefix string
|
||||||
|
together bool
|
||||||
|
}{
|
||||||
|
{"partial", `{"choices":[`, false},
|
||||||
|
{"complete then failure", successfulProviderResponse, false},
|
||||||
|
{"bytes and error together", successfulProviderResponse, true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var reader io.Reader = io.MultiReader(strings.NewReader(test.prefix), failingReader{err: cause})
|
||||||
|
if test.together {
|
||||||
|
reader = &responseBytesAndError{data: []byte(test.prefix), err: cause}
|
||||||
|
}
|
||||||
|
_, err := decodeOpenAIChatResponse(reader)
|
||||||
|
if !errors.Is(err, ErrTransport) || !errors.Is(err, cause) || errors.Is(err, ErrMalformedResponse) || strings.Contains(err.Error(), cause.Error()) {
|
||||||
|
t.Fatalf("read failure = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,4 +13,5 @@ var ErrProfileNotFound = errors.New("execution profile not found")
|
|||||||
// Repository selects execution profiles from a captured index.
|
// Repository selects execution profiles from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
||||||
|
GetProfileIdentity(ctx context.Context, id string) (string, error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,10 +24,6 @@ type goRenderer struct {
|
|||||||
templates TemplateLookup
|
templates TemplateLookup
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGoRenderer() Renderer {
|
|
||||||
return &goRenderer{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewCompiledRenderer renders definitions using templates captured with the
|
// NewCompiledRenderer renders definitions using templates captured with the
|
||||||
// configuration snapshot. Per-call clones bind request-specific input helpers.
|
// configuration snapshot. Per-call clones bind request-specific input helpers.
|
||||||
func NewCompiledRenderer(templates TemplateLookup) Renderer {
|
func NewCompiledRenderer(templates TemplateLookup) Renderer {
|
||||||
@@ -56,13 +52,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
funcs := template.FuncMap{
|
funcs := template.FuncMap{
|
||||||
"input": resolver.resolve,
|
"input": resolver.resolve,
|
||||||
}
|
}
|
||||||
var compiled CompiledTemplates
|
if r.templates == nil {
|
||||||
if r.templates != nil {
|
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
||||||
var found bool
|
}
|
||||||
compiled, found = r.templates.GetCompiledTemplates(definition.ID)
|
compiled, found := r.templates.GetCompiledTemplates(definition.ID)
|
||||||
if !found || len(compiled.Messages) != len(definition.Templates) {
|
if !found || len(compiled.Messages) != len(definition.Templates) {
|
||||||
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -70,15 +65,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}
|
}
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var err error
|
var err error
|
||||||
if r.templates != nil {
|
if definition.SessionID != "" {
|
||||||
if definition.SessionID != "" {
|
if compiled.Session == nil {
|
||||||
if compiled.Session == nil {
|
return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
|
||||||
return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
|
|
||||||
}
|
|
||||||
sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
|
|
||||||
}
|
}
|
||||||
} else {
|
sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
|
||||||
sessionID, err = renderSessionID(ctx, definition.SessionID, funcs, vars)
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -102,18 +93,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
var tmpl *template.Template
|
if compiled.Messages[i] == nil {
|
||||||
var parseErr error
|
return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure)
|
||||||
if r.templates != nil {
|
}
|
||||||
if compiled.Messages[i] == nil {
|
tmpl, parseErr := compiled.Messages[i].Clone()
|
||||||
return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure)
|
if parseErr == nil {
|
||||||
}
|
tmpl.Funcs(funcs)
|
||||||
tmpl, parseErr = compiled.Messages[i].Clone()
|
|
||||||
if parseErr == nil {
|
|
||||||
tmpl.Funcs(funcs)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
tmpl, parseErr = parseTemplate(fmt.Sprintf("msg_%d", i), tmplMsg.Content, funcs)
|
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -201,20 +186,6 @@ func (r *artifactTextResolver) resolve(name string) (string, error) {
|
|||||||
return text, nil
|
return text, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func renderSessionID(ctx context.Context, raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
tmpl, parseErr := parseTemplate("session_id", raw, funcs)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if parseErr != nil {
|
|
||||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, parseErr)
|
|
||||||
}
|
|
||||||
return executeSessionTemplate(ctx, tmpl, vars)
|
|
||||||
}
|
|
||||||
|
|
||||||
func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) {
|
func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestGoRenderer_Render(t *testing.T) {
|
func TestGoRenderer_Render(t *testing.T) {
|
||||||
renderer := NewGoRenderer()
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
inputs := map[string]*domain.Artifact{
|
inputs := map[string]*domain.Artifact{
|
||||||
@@ -33,7 +32,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -49,11 +48,11 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "user", Content: "From file: {{input \"transcript\"}}", ContentFile: "/tmp/user.tmpl"},
|
{Role: "user", Content: "From file: {{input \"transcript\"}}"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -71,7 +70,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -99,7 +98,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -129,7 +128,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -151,7 +150,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -169,7 +168,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, map[string]string{
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": "agent-session-123",
|
"session_id": "agent-session-123",
|
||||||
})
|
})
|
||||||
@@ -190,7 +189,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -208,7 +207,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -223,7 +222,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
})
|
})
|
||||||
@@ -241,7 +240,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": "session" + string([]byte{0xff}),
|
"session_id": "session" + string([]byte{0xff}),
|
||||||
})
|
})
|
||||||
@@ -258,7 +257,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -278,7 +277,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -298,7 +297,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -315,7 +314,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, map[string]*domain.Artifact{}, vars)
|
||||||
if !errors.Is(err, ErrMissingRequiredInput) {
|
if !errors.Is(err, ErrMissingRequiredInput) {
|
||||||
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
|
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -329,9 +328,9 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := CompileTemplate("message", def.Templates[0].Content)
|
||||||
if !errors.Is(err, ErrInvalidTemplate) {
|
if err == nil {
|
||||||
t.Fatalf("expected ErrInvalidTemplate, got %v", err)
|
t.Fatal("expected template compilation to fail")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -343,7 +342,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -359,7 +358,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
{Role: "", Content: "Hello"},
|
{Role: "", Content: "Hello"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
if !errors.Is(err, ErrInvalidMessageRole) {
|
||||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -371,7 +370,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
|
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
res, err := renderer.Render(ctx, def, nil, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -385,7 +384,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
|
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
|
||||||
}
|
}
|
||||||
_, err := renderer.Render(ctx, def, nil, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
|
||||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
if !errors.Is(err, ErrInvalidMessageRole) {
|
||||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -396,9 +395,9 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGoRendererCancellation(t *testing.T) {
|
func TestGoRendererCancellation(t *testing.T) {
|
||||||
t.Run("before session parsing", func(t *testing.T) {
|
t.Run("before rendering", func(t *testing.T) {
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
SessionID: "{{ malformed",
|
SessionID: "{{.session}}",
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "user", Content: "not rendered"},
|
{Role: "user", Content: "not rendered"},
|
||||||
},
|
},
|
||||||
@@ -406,18 +405,11 @@ func TestGoRendererCancellation(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
result, err := renderer.Render(ctx, definition, nil, nil)
|
||||||
if result != nil || !errors.Is(err, context.Canceled) {
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
}
|
}
|
||||||
if errors.Is(err, ErrInvalidTemplate) {
|
|
||||||
t.Fatalf("pre-canceled render parsed the malformed session: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil)
|
|
||||||
if result != nil || !errors.Is(err, ErrInvalidTemplate) {
|
|
||||||
t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("during artifact text conversion", func(t *testing.T) {
|
t.Run("during artifact text conversion", func(t *testing.T) {
|
||||||
@@ -447,14 +439,15 @@ func TestGoRendererCancellation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
counter := &checkCountingContext{Context: context.Background()}
|
counter := &checkCountingContext{Context: context.Background()}
|
||||||
if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil {
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
if _, err := renderer.Render(counter, definition, nil, nil); err != nil {
|
||||||
t.Fatalf("count render checkpoints: %v", err)
|
t.Fatalf("count render checkpoints: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The final three checks occur after template execution, after the
|
// The final three checks occur after template execution, after the
|
||||||
// message is assembled, and immediately before publication.
|
// message is assembled, and immediately before publication.
|
||||||
ctx := newCancelOnCheckContext(counter.checks - 2)
|
ctx := newCancelOnCheckContext(counter.checks - 2)
|
||||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
result, err := renderer.Render(ctx, definition, nil, nil)
|
||||||
if result != nil || !errors.Is(err, context.Canceled) {
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
}
|
}
|
||||||
@@ -472,7 +465,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
first, err := renderer.Render(context.Background(), definition, inputs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("first render: %v", err)
|
t.Fatalf("first render: %v", err)
|
||||||
}
|
}
|
||||||
@@ -488,7 +482,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
||||||
t.Fatal("completed render aliases the artifact body")
|
t.Fatal("completed render aliases the artifact body")
|
||||||
}
|
}
|
||||||
second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
second, err := renderer.Render(context.Background(), definition, inputs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("second render: %v", err)
|
t.Fatalf("second render: %v", err)
|
||||||
}
|
}
|
||||||
@@ -498,7 +492,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
nilInputs := map[string]*domain.Artifact{"document": nil}
|
nilInputs := map[string]*domain.Artifact{"document": nil}
|
||||||
result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil)
|
result, err := renderer.Render(context.Background(), definition, nilInputs, nil)
|
||||||
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
||||||
}
|
}
|
||||||
@@ -559,11 +553,56 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCompiledRendererRequiresCapturedTemplates(t *testing.T) {
|
||||||
|
message, err := CompileTemplate("message", "hello")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
lookup TemplateLookup
|
||||||
|
sessionID string
|
||||||
|
}{
|
||||||
|
{name: "missing lookup"},
|
||||||
|
{name: "missing prompt", lookup: compiledRendererTestLookup{id: "other"}},
|
||||||
|
{name: "missing message", lookup: compiledRendererTestLookup{id: "prompt"}},
|
||||||
|
{name: "nil message", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{nil}}}},
|
||||||
|
{name: "missing session", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{message}}}, sessionID: "session"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{ID: "prompt", SessionID: tc.sessionID, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hello"}}}
|
||||||
|
if _, err := NewCompiledRenderer(tc.lookup).Render(context.Background(), definition, nil, nil); !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("render error = %v, want render failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type compiledRendererTestLookup struct {
|
type compiledRendererTestLookup struct {
|
||||||
id string
|
id string
|
||||||
templates CompiledTemplates
|
templates CompiledTemplates
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func capturedTestRenderer(t testing.TB, definition *domain.PromptDefinition) Renderer {
|
||||||
|
t.Helper()
|
||||||
|
var compiled CompiledTemplates
|
||||||
|
var err error
|
||||||
|
if definition.SessionID != "" {
|
||||||
|
compiled.Session, err = CompileTemplate("session", definition.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, message := range definition.Templates {
|
||||||
|
parsed, err := CompileTemplate("message", message.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
compiled.Messages = append(compiled.Messages, parsed)
|
||||||
|
}
|
||||||
|
return NewCompiledRenderer(compiledRendererTestLookup{id: definition.ID, templates: compiled})
|
||||||
|
}
|
||||||
|
|
||||||
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
||||||
if id != lookup.id {
|
if id != lookup.id {
|
||||||
return CompiledTemplates{}, false
|
return CompiledTemplates{}, false
|
||||||
@@ -571,8 +610,7 @@ func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (Compil
|
|||||||
return lookup.templates, true
|
return lookup.templates, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
|
// BenchmarkGoRendererArtifactReferences measures rendering captured templates.
|
||||||
// template-parsing fallback rather than the captured compiled path.
|
|
||||||
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||||
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||||
inputs := map[string]*domain.Artifact{
|
inputs := map[string]*domain.Artifact{
|
||||||
@@ -604,7 +642,7 @@ func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
b.Run(tc.name, func(b *testing.B) {
|
b.Run(tc.name, func(b *testing.B) {
|
||||||
renderer := NewGoRenderer()
|
renderer := capturedTestRenderer(b, tc.definition)
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.SetBytes(int64(len(body)))
|
b.SetBytes(int64(len(body)))
|
||||||
for range b.N {
|
for range b.N {
|
||||||
|
|||||||
@@ -319,24 +319,27 @@ func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluat
|
|||||||
result := conditionEvaluation{
|
result := conditionEvaluation{
|
||||||
unsupported: left.unsupported || right.unsupported,
|
unsupported: left.unsupported || right.unsupported,
|
||||||
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
||||||
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
|
missingVariables: mergeOwnedNames(left.missingVariables, right.missingVariables),
|
||||||
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
|
contentInputs: mergeOwnedNames(left.contentInputs, right.contentInputs),
|
||||||
}
|
|
||||||
for name := range left.missingVariables {
|
|
||||||
result.missingVariables[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range right.missingVariables {
|
|
||||||
result.missingVariables[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range left.contentInputs {
|
|
||||||
result.contentInputs[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range right.contentInputs {
|
|
||||||
result.contentInputs[name] = struct{}{}
|
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Condition results own their sets; callers retain operand values and flags,
|
||||||
|
// not independent snapshots of these maps after a merge.
|
||||||
|
func mergeOwnedNames(left, right map[string]struct{}) map[string]struct{} {
|
||||||
|
if len(left) < len(right) {
|
||||||
|
left, right = right, left
|
||||||
|
}
|
||||||
|
if left == nil {
|
||||||
|
return right
|
||||||
|
}
|
||||||
|
for name := range right {
|
||||||
|
left[name] = struct{}{}
|
||||||
|
}
|
||||||
|
return left
|
||||||
|
}
|
||||||
|
|
||||||
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
||||||
if parent == requirementInactive {
|
if parent == requirementInactive {
|
||||||
return requirementInactive
|
return requirementInactive
|
||||||
@@ -448,6 +451,34 @@ func (walker *requirementWalker) walkTemplate(compiled *template.Template) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) visitBranch(branch *parse.BranchNode, reach requirementReachability, dot dotIdentity, changesDot bool) error {
|
||||||
|
condition := conditionEvaluation{}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if err := walker.visit(branch.Pipe, requirementInactive, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
condition = walker.evaluateCondition(branch.Pipe, reach, dot)
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
bodyDot := dot
|
||||||
|
if changesDot {
|
||||||
|
bodyDot = walker.withDotIdentity(branch.Pipe, dot, condition)
|
||||||
|
}
|
||||||
|
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||||
|
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||||
|
if err := walker.visit(branch.List, thenReach, bodyDot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := walker.visit(branch.ElseList, elseReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if reach != requirementInactive && !condition.known && walker.dependencies != before {
|
||||||
|
walker.recordConditionLimitation(condition)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
||||||
if err := walker.ctx.Err(); err != nil {
|
if err := walker.ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -468,55 +499,9 @@ func (walker *requirementWalker) visit(node parse.Node, reach requirementReachab
|
|||||||
case *parse.ActionNode:
|
case *parse.ActionNode:
|
||||||
return walker.visit(node.Pipe, reach, dot)
|
return walker.visit(node.Pipe, reach, dot)
|
||||||
case *parse.IfNode:
|
case *parse.IfNode:
|
||||||
if reach == requirementInactive {
|
return walker.visitBranch(&node.BranchNode, reach, dot, false)
|
||||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
|
||||||
}
|
|
||||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
|
||||||
before := walker.dependencies
|
|
||||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
|
||||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
|
||||||
if err := walker.visit(node.List, thenReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !condition.known && walker.dependencies != before {
|
|
||||||
walker.recordConditionLimitation(condition)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case *parse.WithNode:
|
case *parse.WithNode:
|
||||||
if reach == requirementInactive {
|
return walker.visitBranch(&node.BranchNode, reach, dot, true)
|
||||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
|
|
||||||
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
|
||||||
}
|
|
||||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
|
||||||
before := walker.dependencies
|
|
||||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
|
||||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
|
||||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
|
|
||||||
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !condition.known && walker.dependencies != before {
|
|
||||||
walker.recordConditionLimitation(condition)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case *parse.RangeNode:
|
case *parse.RangeNode:
|
||||||
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -314,7 +314,7 @@ func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
|
|||||||
"active": {Name: "active", Body: []byte("active")},
|
"active": {Name: "active", Body: []byte("active")},
|
||||||
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
||||||
}
|
}
|
||||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -406,6 +406,35 @@ func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *tes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConditionEvaluationMergesOwnedEvidence(t *testing.T) {
|
||||||
|
left := conditionEvaluation{missingVariables: map[string]struct{}{"a": {}, "b": {}}, contentInputs: map[string]struct{}{"document": {}}, unsupported: true}
|
||||||
|
right := conditionEvaluation{missingVariables: map[string]struct{}{"b": {}, "c": {}}, inputNameUnknown: true}
|
||||||
|
merged := mergeConditionEvaluations(left, right)
|
||||||
|
if !merged.unsupported || !merged.inputNameUnknown || len(merged.missingVariables) != 3 || len(merged.contentInputs) != 1 {
|
||||||
|
t.Fatalf("merged evidence = %+v", merged)
|
||||||
|
}
|
||||||
|
transferred := mergeConditionEvaluations(conditionEvaluation{}, conditionEvaluation{missingVariables: map[string]struct{}{"only": {}}})
|
||||||
|
if len(transferred.missingVariables) != 1 || transferred.contentInputs != nil {
|
||||||
|
t.Fatalf("transferred evidence = %+v", transferred)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsRetainsLeftHeavyMissingConditions(t *testing.T) {
|
||||||
|
compiled := compileRequirementTemplate(t, "nested", `{{if eq (eq (eq .a .b) .c) .d}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{compiled}}}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Variables) != 4 || len(result.Inputs) != 2 {
|
||||||
|
t.Fatalf("requirements = %+v", result)
|
||||||
|
}
|
||||||
|
for i, name := range []string{"a", "b", "c", "d"} {
|
||||||
|
if result.Variables[i].Name != name {
|
||||||
|
t.Fatalf("variables = %+v", result.Variables)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
||||||
description := "declared for a reason"
|
description := "declared for a reason"
|
||||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||||
@@ -497,7 +526,7 @@ func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
|
|||||||
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
||||||
}
|
}
|
||||||
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
||||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/prompt"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
|
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
|
||||||
@@ -12,5 +13,7 @@ var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
|||||||
|
|
||||||
// Repository selects prompt definitions from a captured index.
|
// Repository selects prompt definitions from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error)
|
GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
|
||||||
|
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
|
||||||
|
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
||||||
}
|
}
|
||||||
|
|||||||
38
internal/usecase/operation_error.go
Normal file
38
internal/usecase/operation_error.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OperationError identifies the operation that owns a failure while retaining
|
||||||
|
// the original cause for inspection.
|
||||||
|
type OperationError struct {
|
||||||
|
Category error
|
||||||
|
Cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *OperationError) Error() string { return fmt.Sprintf("%v: %v", e.Category, e.Cause) }
|
||||||
|
func (e *OperationError) Unwrap() error { return e.Cause }
|
||||||
|
func (e *OperationError) Is(target error) bool { return target == e.Category }
|
||||||
|
|
||||||
|
func promptLookupError(cause error) error {
|
||||||
|
category := ErrPromptLoad
|
||||||
|
if errors.Is(cause, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
category = promptdef.ErrPromptDefinitionNotFound
|
||||||
|
cause = fmt.Errorf("%w: %w", ErrPromptLoad, cause)
|
||||||
|
}
|
||||||
|
return &OperationError{Category: category, Cause: cause}
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileLookupError(cause error) error {
|
||||||
|
category := ErrProfileLoad
|
||||||
|
if errors.Is(cause, profile.ErrProfileNotFound) {
|
||||||
|
category = profile.ErrProfileNotFound
|
||||||
|
cause = fmt.Errorf("%w: %w", ErrProfileLoad, cause)
|
||||||
|
}
|
||||||
|
return &OperationError{Category: category, Cause: cause}
|
||||||
|
}
|
||||||
34
internal/usecase/operation_error_test.go
Normal file
34
internal/usecase/operation_error_test.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOperationErrorRetainsOwnerAndCause(t *testing.T) {
|
||||||
|
cause := fmt.Errorf("nested: %w", promptdef.ErrPromptDefinitionNotFound)
|
||||||
|
outer := &OperationError{Category: ErrLLMGenerate, Cause: cause}
|
||||||
|
if !errors.Is(outer, ErrLLMGenerate) || !errors.Is(outer, promptdef.ErrPromptDefinitionNotFound) || errors.Unwrap(outer) != cause {
|
||||||
|
t.Fatalf("operation error = %v", outer)
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
err error
|
||||||
|
category error
|
||||||
|
}{
|
||||||
|
{promptLookupError(cause), promptdef.ErrPromptDefinitionNotFound},
|
||||||
|
{profileLookupError(profile.ErrProfileNotFound), profile.ErrProfileNotFound},
|
||||||
|
{promptLookupError(contextFailure), ErrPromptLoad},
|
||||||
|
{profileLookupError(contextFailure), ErrProfileLoad},
|
||||||
|
} {
|
||||||
|
var operation *OperationError
|
||||||
|
if !errors.As(test.err, &operation) || operation.Category != test.category {
|
||||||
|
t.Errorf("operation = %+v", operation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var contextFailure = errors.New("lookup failed")
|
||||||
@@ -172,14 +172,13 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
|
|||||||
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
||||||
collaborators.validator.prepareCalls != 0 || collaborators.validator.directValidateCalls != 0 ||
|
collaborators.validator.prepareCalls != 0 ||
|
||||||
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d validation=%d admissions=%d generation=%d",
|
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d admissions=%d generation=%d",
|
||||||
collaborators.artifacts.calls,
|
collaborators.artifacts.calls,
|
||||||
collaborators.renderer.calls,
|
collaborators.renderer.calls,
|
||||||
collaborators.validator.prepareCalls,
|
collaborators.validator.prepareCalls,
|
||||||
collaborators.validator.directValidateCalls,
|
|
||||||
len(collaborators.admitter.backendIDs),
|
len(collaborators.admitter.backendIDs),
|
||||||
collaborators.llm.calls,
|
collaborators.llm.calls,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
@@ -22,7 +21,7 @@ const (
|
|||||||
|
|
||||||
// PreparedExecution owns one frozen, single-use runner execution.
|
// PreparedExecution owns one frozen, single-use runner execution.
|
||||||
type PreparedExecution struct {
|
type PreparedExecution struct {
|
||||||
owner *Runner
|
owner *runnerIdentity
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
state preparedExecutionState
|
state preparedExecutionState
|
||||||
details *domain.PreparedRun
|
details *domain.PreparedRun
|
||||||
@@ -50,24 +49,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
executionSnapshot := operation.run
|
|
||||||
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
|
|
||||||
|
|
||||||
details, err := clonePreparedRun(executionSnapshot)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &PreparedExecution{
|
return &PreparedExecution{
|
||||||
owner: r,
|
owner: r.identity,
|
||||||
state: preparedExecutionReady,
|
state: preparedExecutionReady,
|
||||||
details: details,
|
details: operation.run,
|
||||||
payload: &preparedExecutionPayload{
|
payload: &preparedExecutionPayload{
|
||||||
prepared: executionSnapshot,
|
prepared: operation.run,
|
||||||
validation: operation.validation,
|
validation: operation.validation,
|
||||||
directKey: state.effectiveModel.APIKey,
|
directKey: state.effectiveModel.APIKey,
|
||||||
transientRetries: state.transientRetries,
|
transientRetries: state.transientRetries,
|
||||||
@@ -76,7 +63,8 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Details returns a fresh credential-redacted copy of the prepared run.
|
// Details returns immutable, credential-redacted metadata borrowed from the
|
||||||
|
// handle. Callers must copy it before exposing it to mutable external code.
|
||||||
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -85,11 +73,7 @@ func (p *PreparedExecution) Details() *domain.PreparedRun {
|
|||||||
detailsSnapshot := p.details
|
detailsSnapshot := p.details
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
details, err := clonePreparedRun(detailsSnapshot)
|
return detailsSnapshot
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return details
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Discard invalidates an unclaimed execution and drops its private payload.
|
// Discard invalidates an unclaimed execution and drops its private payload.
|
||||||
@@ -170,7 +154,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
||||||
if p == nil || owner == nil || p.owner != owner {
|
if p == nil || owner == nil || owner.identity == nil || p.owner != owner.identity {
|
||||||
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,9 +173,6 @@ func (p *preparedExecutionPayload) clear() {
|
|||||||
if p == nil {
|
if p == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if p.prepared != nil {
|
|
||||||
p.prepared.EffectiveModelParams.APIKey = ""
|
|
||||||
}
|
|
||||||
p.prepared = nil
|
p.prepared = nil
|
||||||
p.validation = nil
|
p.validation = nil
|
||||||
p.directKey = ""
|
p.directKey = ""
|
||||||
@@ -222,43 +203,3 @@ func (p noOpPreparedValidation) Validate(
|
|||||||
func (noOpPreparedValidation) SchemaDocument() any {
|
func (noOpPreparedValidation) SchemaDocument() any {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
|
||||||
if source == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
copied := *source
|
|
||||||
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
copied.EffectiveModelParams.ExtraParams = extraParams
|
|
||||||
|
|
||||||
if source.InputHashes != nil {
|
|
||||||
copied.InputHashes = make(map[string]string, len(source.InputHashes))
|
|
||||||
for name, hash := range source.InputHashes {
|
|
||||||
copied.InputHashes[name] = hash
|
|
||||||
}
|
|
||||||
}
|
|
||||||
copied.Messages = domain.CloneRenderedMessages(source.Messages)
|
|
||||||
|
|
||||||
if source.StructuredOutput != nil {
|
|
||||||
structuredOutput := *source.StructuredOutput
|
|
||||||
copied.StructuredOutput = &structuredOutput
|
|
||||||
if source.StructuredOutput.JSONSchema != nil {
|
|
||||||
jsonSchema := *source.StructuredOutput.JSONSchema
|
|
||||||
copied.StructuredOutput.JSONSchema = &jsonSchema
|
|
||||||
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
|
|
||||||
if copyErr != nil {
|
|
||||||
return nil, copyErr
|
|
||||||
}
|
|
||||||
copied.StructuredOutput.JSONSchema.Schema = schema
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &copied, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneJSONValue(source any) (any, error) {
|
|
||||||
return jsonvalue.Copy(source)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
@@ -47,25 +48,15 @@ func (p *recordingPreparedValidation) SchemaDocument() any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type recordingValidationPreparer struct {
|
type recordingValidationPreparer struct {
|
||||||
plan *recordingPreparedValidation
|
plan *recordingPreparedValidation
|
||||||
prepareErr error
|
prepareErr error
|
||||||
prepareCalls int
|
prepareCalls int
|
||||||
directValidateCalls int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type validationOnly struct{}
|
type nilPlanPreparer struct{}
|
||||||
|
|
||||||
func (validationOnly) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
|
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||||
return domain.ValidationResult{}, nil
|
return nil, nil
|
||||||
}
|
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) Validate(
|
|
||||||
context.Context,
|
|
||||||
*domain.Artifact,
|
|
||||||
domain.OutputContract,
|
|
||||||
) (domain.ValidationResult, error) {
|
|
||||||
v.directValidateCalls++
|
|
||||||
return domain.ValidationResult{}, errors.New("live validation must not be used")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) PrepareValidation(
|
func (v *recordingValidationPreparer) PrepareValidation(
|
||||||
@@ -129,12 +120,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
|||||||
}
|
}
|
||||||
defer prepared.Discard()
|
defer prepared.Discard()
|
||||||
|
|
||||||
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
|
if validator.prepareCalls != 1 {
|
||||||
t.Fatalf(
|
t.Fatalf("validation preparations = %d, want 1", validator.prepareCalls)
|
||||||
"validation calls=(prepare=%d direct=%d), want (1, 0)",
|
|
||||||
validator.prepareCalls,
|
|
||||||
validator.directValidateCalls,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
if reader.calls != 1 || renderer.calls != 1 {
|
if reader.calls != 1 || renderer.calls != 1 {
|
||||||
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
||||||
@@ -156,90 +143,37 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
|||||||
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
||||||
}
|
}
|
||||||
|
|
||||||
first.Messages[0].Content = "caller mutation"
|
|
||||||
first.InputHashes["input"] = "caller mutation"
|
|
||||||
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
|
|
||||||
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
|
|
||||||
renderer.rendered.Messages[0].Content = "source mutation"
|
renderer.rendered.Messages[0].Content = "source mutation"
|
||||||
|
|
||||||
second := prepared.Details()
|
second := prepared.Details()
|
||||||
if second.Messages[0].Content != "original message" ||
|
if second != first || second.Messages[0].Content != "original message" ||
|
||||||
second.InputHashes["input"] == "caller mutation" ||
|
|
||||||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||||
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
|
t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) {
|
func TestRunnerPreparationDetachesDeepPreparedSchema(t *testing.T) {
|
||||||
operations := []struct {
|
var nested any = "original"
|
||||||
name string
|
for range 150 {
|
||||||
run func(*Runner, domain.RunRequest) error
|
nested = map[string]any{"child": nested}
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "Prepare",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.Prepare(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Run",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.Run(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PrepareExecution",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.PrepareExecution(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
schema := map[string]any{"annotation": nested}
|
||||||
for _, operation := range operations {
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
t.Run(operation.name, func(t *testing.T) {
|
def.Validation.SchemaPath = "schema.json"
|
||||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{schemaDocument: schema}}
|
||||||
def.Validation.SchemaPath = "schema.json"
|
runner := NewRunner(&fakePromptRepo{def: def},
|
||||||
llmClient := &fakeLLM{forbid: true}
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
validator := &recordingValidationPreparer{
|
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, validator, nil)
|
||||||
plan: &recordingPreparedValidation{schemaDocument: excessivelyDeepPreparedJSONValue()},
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
}
|
if err != nil {
|
||||||
runner := NewRunner(
|
t.Fatalf("prepare execution: %v", err)
|
||||||
&fakePromptRepo{def: def},
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
||||||
|
|
||||||
defaultArtifactReader(),
|
|
||||||
defaultRenderer(),
|
|
||||||
llmClient,
|
|
||||||
validator,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
err := operation.run(runner, domain.RunRequest{
|
|
||||||
PromptID: "p",
|
|
||||||
ProfileID: "exec",
|
|
||||||
Inputs: singleInputRef(),
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrValidation) {
|
|
||||||
t.Fatalf("expected ErrValidation, got %v", err)
|
|
||||||
}
|
|
||||||
if llmClient.calls != 0 {
|
|
||||||
t.Fatalf("invalid prepared schema reached generation: %d calls", llmClient.calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
schema["annotation"].(map[string]any)["child"] = "source mutation"
|
||||||
|
first := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||||
func excessivelyDeepPreparedJSONValue() any {
|
if _, ok := first["annotation"].(map[string]any)["child"].(map[string]any); !ok {
|
||||||
const clearlyUnsafeContainerDepth = 1_000
|
t.Fatal("prepared schema aliases the validator document")
|
||||||
var value any = true
|
|
||||||
for level := 0; level < clearlyUnsafeContainerDepth; level++ {
|
|
||||||
value = map[string]any{"child": value}
|
|
||||||
}
|
}
|
||||||
return value
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
||||||
@@ -421,8 +355,8 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
|||||||
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
||||||
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
||||||
}
|
}
|
||||||
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
if repairer.calls != 1 {
|
||||||
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
t.Fatalf("repair calls = %d, want 1", repairer.calls)
|
||||||
}
|
}
|
||||||
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
|
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
|
||||||
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
|
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
|
||||||
@@ -498,7 +432,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
repairer: &fakeRepairer{err: repairInvalidRequest},
|
repairer: &fakeRepairer{err: repairInvalidRequest},
|
||||||
wantError: ErrInvalidRequest,
|
wantError: ErrLLMGenerate,
|
||||||
wantSource: repairInvalidRequest,
|
wantSource: repairInvalidRequest,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -600,6 +534,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
owner := newRunner()
|
owner := newRunner()
|
||||||
|
if other := newRunner(); owner.identity == nil || owner.identity == other.identity {
|
||||||
|
t.Fatal("constructed runners must have independent identity tokens")
|
||||||
|
}
|
||||||
prepared, err := owner.PrepareExecution(context.Background(), request)
|
prepared, err := owner.PrepareExecution(context.Background(), request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare execution: %v", err)
|
t.Fatalf("prepare execution: %v", err)
|
||||||
@@ -616,6 +553,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
if prepared.Details() == nil {
|
if prepared.Details() == nil {
|
||||||
t.Fatal("details unavailable after execution")
|
t.Fatal("details unavailable after execution")
|
||||||
}
|
}
|
||||||
|
if prepared.owner != owner.identity || prepared.payload != nil {
|
||||||
|
t.Fatal("claimed handle retained payload or lost owner identity")
|
||||||
|
}
|
||||||
|
|
||||||
discarded, err := owner.PrepareExecution(context.Background(), request)
|
discarded, err := owner.PrepareExecution(context.Background(), request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -623,6 +563,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
discarded.Discard()
|
discarded.Discard()
|
||||||
discarded.Discard()
|
discarded.Discard()
|
||||||
|
if discarded.owner != owner.identity || discarded.payload != nil {
|
||||||
|
t.Fatal("discarded handle retained payload or lost owner identity")
|
||||||
|
}
|
||||||
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
||||||
}
|
}
|
||||||
@@ -687,7 +630,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
wantLabel := "profile source"
|
wantLabel := "profile source"
|
||||||
if test.requestOverride {
|
if test.requestOverride {
|
||||||
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras}
|
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: jsonvalue.CloneTrusted(requestExtras).(map[string]any)}
|
||||||
wantLabel = "request source"
|
wantLabel = "request source"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -711,11 +654,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||||
t.Fatalf("prepared details retained a source alias: %#v", details)
|
t.Fatalf("prepared details retained a source alias: %#v", details)
|
||||||
}
|
}
|
||||||
details.Messages[0].Content = "mutated details"
|
|
||||||
if again := prepared.Details(); again == nil || len(again.Messages) != 1 || again.Messages[0].Content != "captured rendering" {
|
|
||||||
t.Fatalf("details mutation escaped into the retained snapshot: %+v", again)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
||||||
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
||||||
@@ -737,22 +675,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
|
|
||||||
profile := defaultExecutionProfile()
|
|
||||||
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
|
|
||||||
runner := NewRunner(
|
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
|
||||||
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
|
|
||||||
)
|
|
||||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
||||||
PromptID: "p", ProfileID: "exec",
|
|
||||||
})
|
|
||||||
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
|
|
||||||
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||||
@@ -782,7 +704,7 @@ func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
func TestRunnerPrepareExecutionRejectsNilValidationPlan(t *testing.T) {
|
||||||
reader := defaultArtifactReader()
|
reader := defaultArtifactReader()
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
@@ -791,7 +713,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
|||||||
reader,
|
reader,
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
validationOnly{},
|
nilPlanPreparer{},
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type resolvedProfileSelection struct {
|
type resolvedProfileSelection struct {
|
||||||
@@ -15,10 +14,6 @@ type resolvedProfileSelection struct {
|
|||||||
profile *domain.ExecutionProfile
|
profile *domain.ExecutionProfile
|
||||||
}
|
}
|
||||||
|
|
||||||
type profileIdentityRepository interface {
|
|
||||||
GetProfileIdentity(context.Context, string) (string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
||||||
if strings.TrimSpace(profileID) == "" {
|
if strings.TrimSpace(profileID) == "" {
|
||||||
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||||
@@ -39,7 +34,7 @@ func (r *Runner) resolveProfileSelection(
|
|||||||
}
|
}
|
||||||
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, profileLookupError(err)
|
||||||
}
|
}
|
||||||
if selectedProfile == nil {
|
if selectedProfile == nil {
|
||||||
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
||||||
@@ -61,18 +56,11 @@ func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if indexed, ok := r.profiles.(profileIdentityRepository); ok {
|
identity, err := r.profiles.GetProfileIdentity(ctx, lookupID)
|
||||||
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
return identity, nil
|
|
||||||
}
|
|
||||||
selection, err := r.resolveProfileSelection(ctx, lookupID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", profileLookupError(err)
|
||||||
}
|
}
|
||||||
return selection.id, nil
|
return identity, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||||
@@ -108,7 +96,7 @@ func (r *Runner) InspectProfile(
|
|||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
|
return nil, &OperationError{Category: ErrProfileLoad, Cause: ctx.Err()}
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,12 +106,6 @@ func (r *Runner) InspectProfile(
|
|||||||
}
|
}
|
||||||
target, _ := resolveExecutionTarget(selection.profile, nil)
|
target, _ := resolveExecutionTarget(selection.profile, nil)
|
||||||
target.APIKey = ""
|
target.APIKey = ""
|
||||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: profile extra parameters: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
target.ExtraParams = params
|
|
||||||
|
|
||||||
return &domain.ProfileInspection{
|
return &domain.ProfileInspection{
|
||||||
ProfileID: selection.id,
|
ProfileID: selection.id,
|
||||||
EffectiveModelParams: target,
|
EffectiveModelParams: target,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/profile"
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,7 +24,21 @@ func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (
|
|||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
return nil, r.err
|
return nil, r.err
|
||||||
}
|
}
|
||||||
return r.profile, nil
|
if r.profile == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
profile := *r.profile
|
||||||
|
if profile.ResolvedTarget != nil {
|
||||||
|
target := *profile.ResolvedTarget
|
||||||
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
|
profile.ResolvedTarget = &target
|
||||||
|
}
|
||||||
|
return &profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionProfileRepository) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
_, err := r.GetProfile(ctx, id)
|
||||||
|
return id, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {
|
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {
|
||||||
|
|||||||
@@ -13,10 +13,6 @@ type resolvedPromptDefinition struct {
|
|||||||
hash string
|
hash string
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptInspectionRepository interface {
|
|
||||||
GetPromptInspection(context.Context, string) (*domain.PromptInspection, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
|
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
|
||||||
if strings.TrimSpace(promptID) == "" {
|
if strings.TrimSpace(promptID) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
@@ -25,29 +21,14 @@ func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (
|
|||||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
|
|
||||||
var definition *domain.PromptDefinition
|
definition, hash, err := r.promptDefs.GetPromptSnapshot(ctx, promptID)
|
||||||
var hash string
|
|
||||||
var err error
|
|
||||||
if indexed, ok := r.promptDefs.(interface {
|
|
||||||
GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error)
|
|
||||||
}); ok {
|
|
||||||
definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID)
|
|
||||||
} else {
|
|
||||||
definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID)
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, promptLookupError(err)
|
||||||
}
|
}
|
||||||
if definition == nil {
|
if definition == nil {
|
||||||
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hash == "" {
|
|
||||||
hash, err = hashPromptDefinition(definition)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,29 +39,14 @@ func (r *Runner) resolvePromptInspection(ctx context.Context, promptID string) (
|
|||||||
if r == nil || r.promptDefs == nil {
|
if r == nil || r.promptDefs == nil {
|
||||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
inspection, err := r.promptDefs.GetPromptInspection(ctx, promptID)
|
||||||
inspection, err := indexed.GetPromptInspection(ctx, promptID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
|
||||||
}
|
|
||||||
if inspection == nil {
|
|
||||||
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
|
||||||
}
|
|
||||||
return inspection, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
selection, err := r.resolvePromptDefinition(ctx, promptID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, promptLookupError(err)
|
||||||
}
|
}
|
||||||
definition := selection.definition
|
if inspection == nil {
|
||||||
return &domain.PromptInspection{
|
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
||||||
PromptID: definition.ID,
|
}
|
||||||
PromptHash: selection.hash,
|
return inspection, nil
|
||||||
DefaultProfileID: definition.DefaultProfile,
|
|
||||||
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
|
||||||
OutputContract: definition.Validation,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt without execution work.
|
// InspectPrompt resolves one explicit prompt without execution work.
|
||||||
@@ -90,7 +56,7 @@ func (r *Runner) InspectPrompt(ctx context.Context, promptID string) (*domain.Pr
|
|||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: ctx.Err()}
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/prompt"
|
||||||
"go.maximumdirect.net/promptkit/internal/promptdef"
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,20 +24,33 @@ type promptMetadataRepository struct {
|
|||||||
lookupID string
|
lookupID string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
func (r *inspectionPromptRepository) GetPromptSnapshot(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
) (*domain.PromptDefinition, error) {
|
) (*domain.PromptDefinition, string, error) {
|
||||||
r.calls++
|
r.calls++
|
||||||
r.id = id
|
r.id = id
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
return nil, r.err
|
return nil, "", r.err
|
||||||
}
|
}
|
||||||
return r.definition, nil
|
return r.definition, "captured-hash", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string) (*domain.PromptDefinition, error) {
|
func (r *inspectionPromptRepository) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
return nil, errors.New("full prompt definition should not be requested")
|
definition, hash, err := r.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*inspectionPromptRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *promptMetadataRepository) GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error) {
|
||||||
|
return nil, "", errors.New("full prompt definition should not be requested")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*promptMetadataRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptInspection(
|
func (r *promptMetadataRepository) GetPromptInspection(
|
||||||
@@ -71,15 +85,11 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
t.Fatalf("inspect prompt: %v", err)
|
||||||
}
|
}
|
||||||
wantHash, err := hashPromptDefinition(definition)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("hash prompt definition: %v", err)
|
|
||||||
}
|
|
||||||
if repository.calls != 1 || repository.id != " prompt-id " {
|
if repository.calls != 1 || repository.id != " prompt-id " {
|
||||||
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != definition.ID ||
|
if inspection.PromptID != definition.ID ||
|
||||||
inspection.PromptHash != wantHash ||
|
inspection.PromptHash != "captured-hash" ||
|
||||||
inspection.DefaultProfileID != definition.DefaultProfile ||
|
inspection.DefaultProfileID != definition.DefaultProfile ||
|
||||||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
||||||
inspection.OutputContract != definition.Validation {
|
inspection.OutputContract != definition.Validation {
|
||||||
@@ -96,7 +106,7 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
func TestRunnerInspectPromptUsesOwnedCapturedMetadata(t *testing.T) {
|
||||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
||||||
PromptID: "captured.prompt", PromptHash: "cached-prompt-hash",
|
PromptID: "captured.prompt", PromptHash: "cached-prompt-hash",
|
||||||
DefaultProfileID: "partial-profile",
|
DefaultProfileID: "partial-profile",
|
||||||
@@ -119,7 +129,7 @@ func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
func TestRunnerInspectPromptUsesCapturedHashAndMetadata(t *testing.T) {
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: &domain.PromptDefinition{
|
definition: &domain.PromptDefinition{
|
||||||
ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}},
|
ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
@@ -128,7 +138,7 @@ func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *te
|
|||||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
||||||
if err != nil || inspection.PromptID != "captured.prompt" ||
|
if err != nil || inspection.PromptID != "captured.prompt" ||
|
||||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
||||||
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
t.Fatalf("captured inspection=(%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
48
internal/usecase/repair_prefix.go
Normal file
48
internal/usecase/repair_prefix.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// boundedDiagnosticJSON returns a JSON string prefix and whether it represents
|
||||||
|
// the complete normalized input. It retains at most the useful prefix.
|
||||||
|
func boundedDiagnosticJSON(value string, maxBytes int) ([]byte, bool) {
|
||||||
|
if maxBytes < 2 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
limit := maxBytes - 2
|
||||||
|
var prefix strings.Builder
|
||||||
|
used := 0
|
||||||
|
more := false
|
||||||
|
for i := 0; i < len(value); {
|
||||||
|
r, size := utf8.DecodeRuneInString(value[i:])
|
||||||
|
piece := value[i : i+size]
|
||||||
|
if r == utf8.RuneError && size == 1 {
|
||||||
|
piece = "\uFFFD"
|
||||||
|
i++
|
||||||
|
for i < len(value) {
|
||||||
|
r, size = utf8.DecodeRuneInString(value[i:])
|
||||||
|
if r != utf8.RuneError || size != 1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
i += size
|
||||||
|
}
|
||||||
|
if used+len(piece) > limit {
|
||||||
|
more = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
prefix.WriteString(piece)
|
||||||
|
used += len(piece)
|
||||||
|
}
|
||||||
|
normalized := prefix.String()
|
||||||
|
encoded, _ := json.Marshal(normalized)
|
||||||
|
if len(encoded) > maxBytes {
|
||||||
|
return truncateDiagnosticJSONValue(normalized, maxBytes), false
|
||||||
|
}
|
||||||
|
return encoded, !more
|
||||||
|
}
|
||||||
33
internal/usecase/repair_prefix_test.go
Normal file
33
internal/usecase/repair_prefix_test.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBoundedDiagnosticJSON(t *testing.T) {
|
||||||
|
for _, input := range []string{"", "plain", "<&>\n\"", "雪☃", "a\xff\xfe\xffb", "\xff\xfe" + strings.Repeat("z", 100)} {
|
||||||
|
for budget := 0; budget <= 30; budget++ {
|
||||||
|
encoded, complete := boundedDiagnosticJSON(input, budget)
|
||||||
|
if budget < 2 {
|
||||||
|
if encoded != nil || complete {
|
||||||
|
t.Fatalf("budget %d: %q %v", budget, encoded, complete)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(encoded) > budget || !utf8.Valid(encoded) {
|
||||||
|
t.Fatalf("budget %d: %q", budget, encoded)
|
||||||
|
}
|
||||||
|
var decoded string
|
||||||
|
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
normalized := strings.ToValidUTF8(input, "\uFFFD")
|
||||||
|
if !strings.HasPrefix(normalized, decoded) || complete != (decoded == normalized) {
|
||||||
|
t.Fatalf("input %q budget %d: %q complete=%v normalized=%q", input, budget, decoded, complete, normalized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -117,27 +117,38 @@ func repairGuidance(mode domain.ValidationMode) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func formatRepairDiagnostics(errors []string) string {
|
func formatRepairDiagnostics(errors []string) string {
|
||||||
diagnostics := make([]string, len(errors))
|
complete := []byte{'['}
|
||||||
for index, diagnostic := range errors {
|
fits := true
|
||||||
diagnostics[index] = strings.ToValidUTF8(diagnostic, "\uFFFD")
|
for _, diagnostic := range errors {
|
||||||
|
separator := 0
|
||||||
|
if len(complete) > 1 {
|
||||||
|
separator = 1
|
||||||
|
}
|
||||||
|
entry, whole := boundedDiagnosticJSON(diagnostic, maxRepairDiagnosticBytes-len(complete)-separator-1)
|
||||||
|
if !whole {
|
||||||
|
fits = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if separator != 0 {
|
||||||
|
complete = append(complete, ',')
|
||||||
|
}
|
||||||
|
complete = append(complete, entry...)
|
||||||
}
|
}
|
||||||
|
if fits {
|
||||||
complete, _ := json.Marshal(diagnostics)
|
return string(append(complete, ']'))
|
||||||
if len(complete) <= maxRepairDiagnosticBytes {
|
|
||||||
return string(complete)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
||||||
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
|
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
|
||||||
encoded = append(encoded, '[')
|
encoded = append(encoded, '[')
|
||||||
for _, diagnostic := range diagnostics {
|
for _, diagnostic := range errors {
|
||||||
entry, _ := json.Marshal(diagnostic)
|
|
||||||
separator := 0
|
separator := 0
|
||||||
if len(encoded) > 1 {
|
if len(encoded) > 1 {
|
||||||
separator = 1
|
separator = 1
|
||||||
}
|
}
|
||||||
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
|
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
|
||||||
if len(entry) <= available {
|
entry, whole := boundedDiagnosticJSON(diagnostic, available)
|
||||||
|
if whole {
|
||||||
if separator != 0 {
|
if separator != 0 {
|
||||||
encoded = append(encoded, ',')
|
encoded = append(encoded, ',')
|
||||||
}
|
}
|
||||||
@@ -151,7 +162,7 @@ func formatRepairDiagnostics(errors []string) string {
|
|||||||
if separator != 0 {
|
if separator != 0 {
|
||||||
encoded = append(encoded, ',')
|
encoded = append(encoded, ',')
|
||||||
}
|
}
|
||||||
encoded = append(encoded, truncateDiagnosticJSONValue(diagnostic, available)...)
|
encoded = append(encoded, entry...)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if len(encoded) > 1 {
|
if len(encoded) > 1 {
|
||||||
|
|||||||
@@ -298,6 +298,23 @@ func TestFormatRepairDiagnosticsBoundsAndPreservesData(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFormatRepairDiagnosticsEmptyAndEscapedOversize(t *testing.T) {
|
||||||
|
for _, values := range [][]string{nil, {}} {
|
||||||
|
if got := formatRepairDiagnostics(values); got != "[]" {
|
||||||
|
t.Fatalf("empty diagnostics = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value := strings.Repeat("<&>\x00\xff\xfe", maxRepairDiagnosticBytes)
|
||||||
|
encoded := formatRepairDiagnostics([]string{value})
|
||||||
|
var got []string
|
||||||
|
if err := json.Unmarshal([]byte(encoded), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(encoded) > maxRepairDiagnosticBytes || len(got) != 2 || got[1] != omittedRepairDiagnostics || !strings.HasPrefix(strings.ToValidUTF8(value, "\uFFFD"), got[0]) {
|
||||||
|
t.Fatalf("oversize diagnostics length=%d, values=%d", len(encoded), len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func manyRepairDiagnostics() []string {
|
func manyRepairDiagnostics() []string {
|
||||||
diagnostics := make([]string, 1_000)
|
diagnostics := make([]string, 1_000)
|
||||||
for index := range diagnostics {
|
for index := range diagnostics {
|
||||||
|
|||||||
@@ -18,10 +18,6 @@ type RequirementInspectionRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
type requirementSnapshotRepository interface {
|
|
||||||
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
|
||||||
}
|
|
||||||
|
|
||||||
// InspectRequirements discovers prompt inputs and caller variables without
|
// InspectRequirements discovers prompt inputs and caller variables without
|
||||||
// loading artifacts or performing execution preparation.
|
// loading artifacts or performing execution preparation.
|
||||||
func (r *Runner) InspectRequirements(
|
func (r *Runner) InspectRequirements(
|
||||||
@@ -36,7 +32,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
selection, err := r.resolvePromptInspection(ctx, req.PromptID)
|
selection, err := r.resolvePromptInspection(ctx, req.PromptID)
|
||||||
@@ -44,7 +40,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, &OperationError{Category: ErrProfileLoad, Cause: err}
|
||||||
}
|
}
|
||||||
selectedProfileID := req.ProfileID
|
selectedProfileID := req.ProfileID
|
||||||
if strings.TrimSpace(selectedProfileID) == "" {
|
if strings.TrimSpace(selectedProfileID) == "" {
|
||||||
@@ -58,21 +54,17 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
repository, ok := r.promptDefs.(requirementSnapshotRepository)
|
snapshot, ok := r.promptDefs.GetRequirementSnapshot(selection.PromptID)
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
|
||||||
}
|
|
||||||
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID)
|
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: selection.PromptID}
|
definition := &domain.PromptDefinition{ID: selection.PromptID}
|
||||||
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
inspection.ProfileID = profileIdentity
|
inspection.ProfileID = profileIdentity
|
||||||
return &inspection, nil
|
return &inspection, nil
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user