Compare commits
120 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8ff9aa3efd | |||
| 78b78ea00b | |||
| 63e9f0b731 | |||
| 20b0f78af2 | |||
| 101a2840d4 | |||
| f083ad5309 | |||
| 19a0604874 | |||
| ea31e71280 | |||
| 27966c1686 | |||
| ea63a7b0ff | |||
| b5b9b1f209 | |||
| 129959b24c | |||
| dfe9d6566f | |||
| bf3d2e19bd | |||
| 4cbd4e2da0 | |||
| b0a16ccce8 | |||
| 8add7c49e6 | |||
| 8563379216 | |||
| 951739eda0 | |||
| a0f696181f | |||
| 3951dcf1f3 | |||
| 363920deb2 | |||
| 06315062b3 | |||
| c669f14610 | |||
| 52287ab2d1 | |||
| a43e0f17ee | |||
| ffaaf552c8 | |||
| d7007eb072 | |||
| 4f3212bfb6 | |||
| 4faf43cc29 | |||
| c9663ec63c | |||
| d8c98ac575 | |||
| 5ee602cc4b | |||
| 96c619f7d4 | |||
| b3b5ec3820 | |||
| 5dd1418953 | |||
| f5c900cb0a | |||
| c5f3de13f3 | |||
| 88725e79ab | |||
| 6c015a17fa | |||
| 9463c9463a | |||
| 1fbd175e53 | |||
| 4b4ce43796 | |||
| d78474a3bf | |||
| acaf2ccd8b | |||
| 97e5739bfb | |||
| 90f796aef7 | |||
| 0dec3dcec0 | |||
| 448e64dc40 | |||
| de46935a0e | |||
| c69703b2b6 | |||
| f660b6e0fc | |||
| 5f56c65427 | |||
| cfa3a85308 | |||
| 1ff6d778af | |||
| 3f93763c9c | |||
| 4aeb58f913 | |||
| ac1e150168 | |||
| ff57dbf8b3 | |||
| beb18cc9d6 | |||
| 7203e5d6a1 | |||
| 356563a6e5 | |||
| 6673284312 | |||
| 6f42e1e007 | |||
| cee9bbc7b2 | |||
| 7db5572b62 | |||
| a91bc1608b | |||
| e7a3665443 | |||
| 59e8010623 | |||
| ada5377f32 | |||
| df208e132b | |||
| 8c0fb15727 | |||
| 2e2fba4993 | |||
| 68668cee56 | |||
| f745b14b71 | |||
| dbba1485a7 | |||
| 7c435032b6 | |||
| e834e7d806 | |||
| a4901cbb0e | |||
| 5dcfc2e953 | |||
| a02561d40f | |||
| 6112ee8fb4 | |||
| 0aceff8c08 | |||
| cddb2b389c | |||
| e056cb1607 | |||
| 1e122f0473 | |||
| 2f0f5fa607 | |||
| 7d500ee62f | |||
| 7be677c80b | |||
| 1c075bd510 | |||
| b8dc8a49a9 | |||
| cd373ee46b | |||
| e881bbdcf4 | |||
| 106c808442 | |||
| 4ec35f5ca6 | |||
| 6a1a32ae49 | |||
| e85ea2db35 | |||
| 3ef2a9e18e | |||
| a85a6788b4 | |||
| 21fea46fb3 | |||
| 62bf82a56b | |||
| bd3dfa992c | |||
| 425f53a62e | |||
| fcc94eb9e9 | |||
| 95730a3c11 | |||
| c542b81666 | |||
| e8e112186e | |||
| aa08d46ef6 | |||
| 668d09997a | |||
| 69203dcb12 | |||
| 3b760310a6 | |||
| b39460a507 | |||
| 5177db0611 | |||
| 31f1e6a89d | |||
| f7085d76f6 | |||
| cf0bfd4d76 | |||
| 6b1323d1fe | |||
| fb87fabc2e | |||
| 5adc1c1e64 | |||
| 0d27252999 |
@@ -45,6 +45,14 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
The [v0.20.0 release notes](docs/releases/v0.20.0.md) cover resolved resource
|
||||||
|
inspection, generic JSON/YAML reports, typed prompt sources, and backend
|
||||||
|
inheritance. Upgrading consumers should review the changed inspection result
|
||||||
|
types and profile null-fallback behavior.
|
||||||
|
|
||||||
|
The [v0.19.0 release notes](docs/releases/v0.19.0.md) cover audit remediation
|
||||||
|
and compatibility changes to artifact ownership, programmatic sources, and schemas.
|
||||||
|
|
||||||
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
The [v0.18.0 release notes](docs/releases/v0.18.0.md) cover the default-on
|
||||||
OpenRouter backend and profiles, recorded configuration normalization, and
|
OpenRouter backend and profiles, recorded configuration normalization, and
|
||||||
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
ConfigKit v1.1. The notes explain inventory changes and the bundle opt-out;
|
||||||
|
|||||||
43
artifact_ownership_contract_test.go
Normal file
43
artifact_ownership_contract_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type independentArtifactReader struct{}
|
||||||
|
|
||||||
|
func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
return &promptkit.Artifact{Body: []byte(ref.Body), Hash: ref.Body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("ownership", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithArtifactReader(independentArtifactReader{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var group sync.WaitGroup
|
||||||
|
for i := range 24 {
|
||||||
|
group.Go(func() {
|
||||||
|
value := fmt.Sprintf("body-%d", i)
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"document": promptkit.Inline(value)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("prepare: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != value || prepared.InputHashes["document"] != value {
|
||||||
|
t.Errorf("prepared = %+v", prepared)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
}
|
||||||
25
backends.go
25
backends.go
@@ -19,10 +19,13 @@ const BackendLocal = "local"
|
|||||||
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
|
||||||
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
// empty map supplies an empty mapping. A nil SmokeTests pointer is omitted; a
|
||||||
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
// non-nil pointer supplies a smoke_tests mapping. Use raw ConfigKit Mapping
|
||||||
// inputs for explicit null. BackendConfig has no stable JSON representation.
|
// inputs for explicit null. Extends names a sibling backend in the same list.
|
||||||
|
// BackendConfig has no stable JSON representation.
|
||||||
type BackendConfig struct {
|
type BackendConfig struct {
|
||||||
// ID is the local backend identity within its resource list.
|
// ID is the local backend identity within its resource list.
|
||||||
ID string
|
ID string
|
||||||
|
// Extends names a sibling backend to inherit through ConfigKit.
|
||||||
|
Extends *string
|
||||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||||
Endpoint *string
|
Endpoint *string
|
||||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||||
@@ -44,13 +47,13 @@ type BackendSmokeTestConfig struct {
|
|||||||
// DefaultProfile is the exact profile ID selected as the backend's
|
// DefaultProfile is the exact profile ID selected as the backend's
|
||||||
// diagnostic inference default. A supplied value must not be blank; the ID
|
// diagnostic inference default. A supplied value must not be blank; the ID
|
||||||
// is resolved only when that default is selected.
|
// is resolved only when that default is selected.
|
||||||
DefaultProfile *string
|
DefaultProfile *string `json:"default_profile,omitempty" yaml:"default_profile,omitempty"`
|
||||||
// Reachability optionally selects a full HTTP destination for reachability
|
// Reachability optionally selects a full HTTP destination for reachability
|
||||||
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
||||||
Reachability *ReachabilityProbeConfig
|
Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"`
|
||||||
// Authentication optionally declares a named authentication diagnostic.
|
// Authentication optionally declares a named authentication diagnostic.
|
||||||
// A supplied object requires Kind.
|
// A supplied object requires Kind.
|
||||||
Authentication *AuthenticationProbeConfig
|
Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReachabilityProbeConfig configures an optional reachability destination.
|
// ReachabilityProbeConfig configures an optional reachability destination.
|
||||||
@@ -58,7 +61,7 @@ type ReachabilityProbeConfig struct {
|
|||||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
// Nil selects the effective backend endpoint; an explicit blank value is
|
// Nil selects the effective backend endpoint; an explicit blank value is
|
||||||
// invalid.
|
// invalid.
|
||||||
Endpoint *string
|
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// AuthenticationProbeConfig configures an optional named authentication
|
// AuthenticationProbeConfig configures an optional named authentication
|
||||||
@@ -66,11 +69,11 @@ type ReachabilityProbeConfig struct {
|
|||||||
type AuthenticationProbeConfig struct {
|
type AuthenticationProbeConfig struct {
|
||||||
// Kind selects the authentication adapter. Supported values are "none" and
|
// Kind selects the authentication adapter. Supported values are "none" and
|
||||||
// "openrouter_key"; surrounding whitespace and case are normalized.
|
// "openrouter_key"; surrounding whitespace and case are normalized.
|
||||||
Kind *string
|
Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"`
|
||||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||||
// Nil selects the adapter's backend-relative destination; an explicit blank
|
// Nil selects the adapter's backend-relative destination; an explicit blank
|
||||||
// value is invalid.
|
// value is invalid.
|
||||||
Endpoint *string
|
Endpoint *string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendSource creates a labeled ConfigKit mapping containing backend records
|
// BackendSource creates a labeled ConfigKit mapping containing backend records
|
||||||
@@ -78,11 +81,15 @@ type AuthenticationProbeConfig struct {
|
|||||||
// It encodes fields without validating them; NewEngine validates the final
|
// It encodes fields without validating them; NewEngine validates the final
|
||||||
// effective configuration after all selected source layers are combined.
|
// effective configuration after all selected source layers are combined.
|
||||||
// Mapping captures a copy of supported input values at creation; unsupported
|
// Mapping captures a copy of supported input values at creation; unsupported
|
||||||
// values fail when the source is loaded.
|
// values fail when the source is loaded. ExtraParams must use ConfigKit Mapping
|
||||||
|
// input forms; callers must explicitly convert numbers or containers as needed.
|
||||||
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||||
definitions := make([]any, 0, len(backends))
|
definitions := make([]any, 0, len(backends))
|
||||||
for _, config := range backends {
|
for _, config := range backends {
|
||||||
fields := map[string]any{"id": config.ID}
|
fields := map[string]any{"id": config.ID}
|
||||||
|
if config.Extends != nil {
|
||||||
|
fields["extends"] = map[string]any{"id": *config.Extends}
|
||||||
|
}
|
||||||
if config.Endpoint != nil {
|
if config.Endpoint != nil {
|
||||||
fields["endpoint"] = *config.Endpoint
|
fields["endpoint"] = *config.Endpoint
|
||||||
}
|
}
|
||||||
@@ -96,7 +103,7 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
|||||||
fields["queue_capacity"] = *config.QueueCapacity
|
fields["queue_capacity"] = *config.QueueCapacity
|
||||||
}
|
}
|
||||||
if config.ExtraParams != nil {
|
if config.ExtraParams != nil {
|
||||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
fields["extra_params"] = config.ExtraParams
|
||||||
}
|
}
|
||||||
if config.SmokeTests != nil {
|
if config.SmokeTests != nil {
|
||||||
smokeTests := make(map[string]any)
|
smokeTests := make(map[string]any)
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ package promptkit_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"reflect"
|
||||||
"strings"
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
@@ -19,7 +19,7 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
underlayProfile, err := underlayEngine.InspectProfile(context.Background(), "gpt-sol-latest")
|
||||||
if err != nil || underlayProfile.EffectiveModelParams.Model != "underlay-model" || underlayProfile.EffectiveModelParams.BackendID != "" {
|
if err != nil || underlayProfile.Model == nil || *underlayProfile.Model != "underlay-model" || underlayProfile.Backend != nil {
|
||||||
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
t.Fatalf("underlay replacement = %+v, %v", underlayProfile, err)
|
||||||
}
|
}
|
||||||
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{
|
main := configkit.Input{Source: configkit.Mapping("caller main", map[string]any{
|
||||||
@@ -38,31 +38,30 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
inspect := func(id string) promptkit.ExecutionTarget {
|
inspect := func(id string) *promptkit.ProfileInspection {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
profile, err := engine.InspectProfile(context.Background(), id)
|
profile, err := engine.InspectProfile(context.Background(), id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return profile.EffectiveModelParams
|
return profile
|
||||||
}
|
}
|
||||||
if target := inspect("gpt-sol-latest"); target.Model != "main-model" || target.BackendID != "" {
|
if target := inspect("gpt-sol-latest"); target.Model == nil || *target.Model != "main-model" || target.Backend != nil {
|
||||||
t.Fatalf("replacement = %+v", target)
|
t.Fatalf("replacement = %+v", target)
|
||||||
}
|
}
|
||||||
if target := inspect("custom"); target.Model != "custom-model" || target.BackendID != "openrouter" || target.Endpoint != "https://local.invalid/v1" {
|
if target := inspect("custom"); target.Model == nil || *target.Model != "custom-model" || target.Backend == nil || *target.Backend != "openrouter" || target.Endpoint != nil {
|
||||||
t.Fatalf("inherited = %+v", target)
|
t.Fatalf("inherited = %+v", target)
|
||||||
}
|
}
|
||||||
if target := inspect("gpt-luna-latest"); target.Endpoint != "https://local.invalid/v1" || target.APIKeyEnv != "" {
|
if backend, err := engine.InspectBackend(context.Background(), "openrouter"); err != nil || backend.Endpoint != "https://local.invalid/v1" || backend.APIKeyEnv != nil {
|
||||||
t.Fatalf("backend replacement = %+v", target)
|
t.Fatalf("backend replacement = %+v, %v", backend, err)
|
||||||
}
|
}
|
||||||
if target := inspect("openrouter/profile"); target.BackendID != "openrouter/local" || target.Model != "collection-model" {
|
if target := inspect("openrouter/profile"); target.Backend == nil || *target.Backend != "openrouter/local" || target.Model == nil || *target.Model != "collection-model" {
|
||||||
t.Fatalf("collection = %+v", target)
|
t.Fatalf("collection = %+v", target)
|
||||||
}
|
}
|
||||||
for _, tier := range []string{"low", "medium", "high", "xhigh", "max"} {
|
target := inspect("auto-low")
|
||||||
target := inspect("auto-" + tier)
|
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
||||||
if target.BackendID != "other" || target.Model != "other-model" || !strings.Contains(fmt.Sprint(target.ExtraParams), "cost_tier:"+tier) || !strings.Contains(fmt.Sprint(target.ExtraParams), "id:auto-router") {
|
if target.Backend == nil || *target.Backend != "other" || target.Model == nil || *target.Model != "other-model" || target.ExtraParams == nil || !reflect.DeepEqual(*target.ExtraParams, wantPlugins) {
|
||||||
t.Errorf("tier %s = %+v", tier, target)
|
t.Errorf("inherited tier = %+v", target)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,11 +69,11 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
|
|||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name, mode, id string
|
name, mode, id string
|
||||||
wantErr bool
|
wantErr bool
|
||||||
count int
|
hasAuto bool
|
||||||
}{
|
}{
|
||||||
{"duplicate append", "append", "auto", true, 0},
|
{"duplicate append", "append", "auto", true, false},
|
||||||
{"distinct append", "append", "custom", false, 18},
|
{"distinct append", "append", "custom", false, true},
|
||||||
{"replace", "replace", "custom", false, 1},
|
{"replace", "replace", "custom", false, false},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
|
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
|
||||||
@@ -92,7 +91,7 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles, err := engine.ListProfiles(context.Background())
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
if err != nil || len(profiles) != test.count {
|
if err != nil || !slices.Contains(profiles, "custom") || slices.Contains(profiles, "auto") != test.hasAuto {
|
||||||
t.Fatalf("profiles = %v, %v", profiles, err)
|
t.Fatalf("profiles = %v, %v", profiles, err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -51,8 +51,7 @@ func TestBundledProfilePreparationAndExecution(t *testing.T) {
|
|||||||
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
||||||
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
for _, test := range []struct{ id, model, tier string }{
|
for _, test := range []struct{ id, model, tier string }{
|
||||||
{"auto-low", "openrouter/auto", "low"}, {"auto-medium", "openrouter/auto", "medium"}, {"auto-high", "openrouter/auto", "high"}, {"auto-xhigh", "openrouter/auto", "xhigh"}, {"auto-max", "openrouter/auto", "max"},
|
{"auto-low", "openrouter/auto", "low"}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
||||||
{"auto", "openrouter/auto", ""}, {"free", "openrouter/free", ""}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
|
||||||
} {
|
} {
|
||||||
t.Run(test.id, func(t *testing.T) {
|
t.Run(test.id, func(t *testing.T) {
|
||||||
var body map[string]any
|
var body map[string]any
|
||||||
|
|||||||
@@ -51,23 +51,16 @@ func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
want := 0
|
if (len(profiles) > 0) != test.enabled || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
||||||
if test.enabled {
|
|
||||||
want = 17
|
|
||||||
}
|
|
||||||
if len(profiles) != want || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
|
||||||
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
||||||
}
|
}
|
||||||
if test.enabled {
|
if test.enabled {
|
||||||
wantProfiles := []string{"auto", "auto-high", "auto-low", "auto-max", "auto-medium", "auto-xhigh", "claude-fable-latest", "claude-haiku-latest", "claude-opus-latest", "claude-sonnet-latest", "free", "gemini-flash-latest", "gemini-pro-latest", "gpt-astra-latest", "gpt-luna-latest", "gpt-sol-latest", "gpt-terra-latest"}
|
if !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
if !reflect.DeepEqual(profiles, wantProfiles) || !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
|
||||||
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
||||||
}
|
}
|
||||||
for id, model := range map[string]string{"auto": "openrouter/auto", "auto-max": "openrouter/auto", "gpt-luna-latest": "~openai/gpt-luna-latest", "free": "openrouter/free"} {
|
profile, err := engine.InspectProfile(context.Background(), "auto")
|
||||||
profile, err := engine.InspectProfile(context.Background(), id)
|
if err != nil || profile.Backend == nil || *profile.Backend != "openrouter" {
|
||||||
if err != nil || profile.EffectiveModelParams.Model != model || profile.EffectiveModelParams.BackendID != "openrouter" {
|
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
||||||
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package promptkit_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"reflect"
|
"reflect"
|
||||||
@@ -13,6 +14,45 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type cancelingAuthenticationBody struct {
|
||||||
|
remaining int
|
||||||
|
cancel context.CancelFunc
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *cancelingAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n := min(len(p), b.remaining)
|
||||||
|
for i := range p[:n] {
|
||||||
|
p[i] = 'x'
|
||||||
|
}
|
||||||
|
b.remaining -= n
|
||||||
|
if b.remaining == 0 {
|
||||||
|
b.cancel()
|
||||||
|
return n, context.Canceled
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
func (b *cancelingAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestAuthenticationReadCancellationWinsOverOverflow(t *testing.T) {
|
||||||
|
for _, size := range []int{1024, (1 << 20) + 1} {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
body := &cancelingAuthenticationBody{remaining: size, cancel: cancel}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter"})
|
||||||
|
cancel()
|
||||||
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != promptkit.ExecutionFailureCanceled || report.Facts.CredentialsAccepted != nil || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || !body.closed {
|
||||||
|
t.Fatalf("size %d: report=%+v error=%v closed=%v", size, report, err, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
||||||
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
var urls []string
|
var urls []string
|
||||||
@@ -125,3 +165,17 @@ func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeAuthenticationResponseOverflow(t *testing.T) {
|
||||||
|
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(strings.Repeat("x", (1<<20)+1)))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter", APIKey: "private-key-marker"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Reason != promptkit.SmokeTestReasonAuthenticationResponseLimitExceeded || report.Failure != promptkit.ExecutionFailureOther || report.Facts.CredentialsAccepted != nil {
|
||||||
|
t.Fatalf("overflow = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ func TestCollectionInputLoadsNativeImportsAndLaterOverride(t *testing.T) {
|
|||||||
t.Fatalf("construct engine with native collection import: %v", err)
|
t.Fatalf("construct engine with native collection import: %v", err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -85,7 +85,7 @@ func TestCollectionInputLoadsEmbeddedImportsAndLaterOverride(t *testing.T) {
|
|||||||
t.Fatalf("construct engine with embedded collection import: %v", err)
|
t.Fatalf("construct engine with embedded collection import: %v", err)
|
||||||
}
|
}
|
||||||
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
inspection, err := engine.InspectPrompt(context.Background(), "docs/greeting")
|
||||||
if err != nil || inspection.PromptID != "docs/greeting" || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "overridden" {
|
if err != nil || inspection.PromptID != "docs/greeting" || inspection.Inputs == nil || len(*inspection.Inputs) != 1 || (*inspection.Inputs)[0].Name != "overridden" {
|
||||||
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
t.Fatalf("inspect imported and overlaid prompt = (%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ func TestFormerCatalogBackendIDsWorkWhenConfigured(t *testing.T) {
|
|||||||
"rakestrawhome-profile": rakestrawhomeID,
|
"rakestrawhome-profile": rakestrawhomeID,
|
||||||
} {
|
} {
|
||||||
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
inspection, err := engine.InspectProfile(context.Background(), profileID)
|
||||||
if err != nil || inspection.EffectiveModelParams.BackendID != backendID {
|
if err != nil || inspection.Backend == nil || *inspection.Backend != backendID {
|
||||||
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
t.Fatalf("inspect %q = (%+v, %v), want backend %q", profileID, inspection, err, backendID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -242,8 +242,9 @@ func TestPublicConstructionUsesProgrammaticBackendAndProfileSources(t *testing.T
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configured profile: %v", err)
|
t.Fatalf("inspect configured profile: %v", err)
|
||||||
}
|
}
|
||||||
if inspection.EffectiveModelParams.Endpoint != endpoint {
|
backendInspection, err := engine.InspectBackend(context.Background(), backendID)
|
||||||
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
|
if err != nil || inspection.Endpoint != nil || backendInspection.Endpoint != endpoint {
|
||||||
|
t.Fatalf("profile/backend = %+v / %+v, %v", inspection, backendInspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
153
convert.go
153
convert.go
@@ -2,9 +2,9 @@ package promptkit
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
@@ -229,37 +229,58 @@ func fromDomainExecutionTargetWithOwnedParams(target domain.ExecutionTarget, ext
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
|
func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
|
||||||
if inspection == nil {
|
out := &ProfileInspection{ProfileID: v.ID, Backend: copyStringPtr(v.Backend), Endpoint: copyStringPtr(v.Endpoint), Model: copyStringPtr(v.Model), Temperature: copyFloat64Ptr(v.Temperature), TopP: copyFloat64Ptr(v.TopP), MaxTokens: copyIntPtr(v.MaxTokens), TimeoutSeconds: copyIntPtr(v.TimeoutSeconds), ServiceTier: copyStringPtr(v.ServiceTier), ReasoningEffort: copyStringPtr(v.ReasoningEffort), APIKeyEnv: copyStringPtr(v.APIKeyEnv), APIKeyRequired: copyBoolPtr(v.APIKeyRequired)}
|
||||||
return nil
|
if v.ExtraParams != nil {
|
||||||
}
|
data := copyAnyMap(v.ExtraParams)
|
||||||
return &ProfileInspection{
|
out.ExtraParams = &data
|
||||||
ProfileID: inspection.ProfileID,
|
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
|
|
||||||
APIKeyRequired: inspection.APIKeyRequired,
|
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
|
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
||||||
if inspection == nil {
|
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||||
return nil
|
if v.ExtraParams != nil {
|
||||||
|
data := copyAnyMap(v.ExtraParams)
|
||||||
|
out.ExtraParams = &data
|
||||||
}
|
}
|
||||||
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
|
if v.SmokeTests != nil {
|
||||||
for i, input := range inspection.Inputs {
|
out.SmokeTests = &BackendSmokeTestConfig{DefaultProfile: copyStringPtr(v.SmokeTests.DefaultProfile)}
|
||||||
inputs[i] = PromptInputDefinition{
|
if v.SmokeTests.Reachability != nil {
|
||||||
Name: input.Name,
|
out.SmokeTests.Reachability = &ReachabilityProbeConfig{Endpoint: copyStringPtr(v.SmokeTests.Reachability.Endpoint)}
|
||||||
Required: input.Required,
|
}
|
||||||
ContentType: input.ContentType,
|
if v.SmokeTests.Authentication != nil {
|
||||||
Description: input.Description,
|
out.SmokeTests.Authentication = &AuthenticationProbeConfig{Kind: copyStringPtr(v.SmokeTests.Authentication.Kind), Endpoint: copyStringPtr(v.SmokeTests.Authentication.Endpoint)}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &PromptInspection{
|
return out
|
||||||
PromptID: inspection.PromptID,
|
}
|
||||||
PromptHash: inspection.PromptHash,
|
|
||||||
DefaultProfileID: inspection.DefaultProfileID,
|
func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
|
||||||
Inputs: inputs,
|
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: copyIntPtr(v.Output.RepairAttempts)}}
|
||||||
OutputContract: fromDomainOutputContract(inspection.OutputContract),
|
if v.Inputs != nil {
|
||||||
|
inputs := make([]PromptInputInspection, len(*v.Inputs))
|
||||||
|
for i, input := range *v.Inputs {
|
||||||
|
inputs[i] = PromptInputInspection{Name: input.Name, Required: copyBoolPtr(input.Required), ContentType: copyStringPtr(input.ContentType), Description: copyStringPtr(input.Description)}
|
||||||
|
}
|
||||||
|
out.Inputs = &inputs
|
||||||
}
|
}
|
||||||
|
out.Messages = make([]PromptMessageInspection, len(v.Messages))
|
||||||
|
for i, message := range v.Messages {
|
||||||
|
out.Messages[i] = PromptMessageInspection{Role: message.Role, Content: message.Content}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
out.Messages[i].CacheControl = &PromptCacheControlInspection{Type: CacheControlType(message.CacheControl.Type), TTL: copyStringPtr(message.CacheControl.TTL)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyBoolPtr(src *bool) *bool {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
value := *src
|
||||||
|
return &value
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
|
||||||
@@ -395,88 +416,10 @@ func copyAnyMap(src map[string]any) map[string]any {
|
|||||||
if src == nil {
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make(map[string]any, len(src))
|
return jsonvalue.CloneTrusted(src).(map[string]any)
|
||||||
for k, v := range src {
|
|
||||||
out[k] = copyAny(v)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyAny(value any) any {
|
func copyAny(value any) any { return jsonvalue.CloneTrusted(value) }
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
switch v := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
return copyAnyMap(v)
|
|
||||||
case []any:
|
|
||||||
out := make([]any, len(v))
|
|
||||||
for i, item := range v {
|
|
||||||
out[i] = copyAny(item)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case []string:
|
|
||||||
return copyStringSlice(v)
|
|
||||||
case []byte:
|
|
||||||
return copyBytes(v)
|
|
||||||
default:
|
|
||||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
|
||||||
if !value.IsValid() {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
switch value.Kind() {
|
|
||||||
case reflect.Interface:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
copied := copyReflectValue(value.Elem())
|
|
||||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
|
||||||
return copied
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
out.Set(copied)
|
|
||||||
return out
|
|
||||||
case reflect.Pointer:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type().Elem())
|
|
||||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
|
||||||
return out
|
|
||||||
case reflect.Map:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
|
||||||
iter := value.MapRange()
|
|
||||||
for iter.Next() {
|
|
||||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Slice:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Array:
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
default:
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyStringSlice(src []string) []string {
|
func copyStringSlice(src []string) []string {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
|
|||||||
26
doc.go
26
doc.go
@@ -4,17 +4,17 @@
|
|||||||
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
// Applications construct an [Engine] with [NewEngine], select ConfigKit
|
||||||
// sources for prompts, profiles, and backends, and
|
// sources for prompts, profiles, and backends, and
|
||||||
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
|
||||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.InspectBackend],
|
||||||
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.ListProfiles], [Engine.ListPrompts], [Engine.Prepare],
|
||||||
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
|
||||||
// validators, and the built-in OpenAI-compatible client remain internal
|
// validators, and the built-in OpenAI-compatible client remain internal
|
||||||
// implementation details.
|
// implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
|
||||||
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
|
// InspectRequirements, InspectProfile, InspectBackend, ListProfiles, ListPrompts,
|
||||||
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
// Prepare, PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
|
||||||
// bound admitted Run and RunPrepared calls and model generations where
|
// bound admitted Run and RunPrepared calls and model generations where
|
||||||
// configured, while different backend pools and unlimited backends continue
|
// configured, while different backend pools and unlimited backends continue
|
||||||
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
|
||||||
@@ -23,7 +23,8 @@
|
|||||||
// ConfigKit captures the selected configuration when NewEngine constructs an
|
// ConfigKit captures the selected configuration when NewEngine constructs an
|
||||||
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
|
||||||
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
|
||||||
// InspectRequirements, and InspectProfile return copied inspection values.
|
// InspectRequirements, InspectProfile, and InspectBackend return copied
|
||||||
|
// inspection values.
|
||||||
// ListProfiles and ListPrompts return caller-owned identity slices.
|
// ListProfiles and ListPrompts return caller-owned identity slices.
|
||||||
// Returned values and values passed to extension interfaces are likewise
|
// Returned values and values passed to extension interfaces are likewise
|
||||||
// isolated from engine state.
|
// isolated from engine state.
|
||||||
@@ -56,17 +57,22 @@
|
|||||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
// used by those values.
|
// used by those values.
|
||||||
//
|
//
|
||||||
// Construction, inspection, handle, and error values, including [Config],
|
// Resource inspection results support direct JSON and YAML encoding.
|
||||||
|
// [PromptInspection], [ProfileInspection], [BackendInspection], and their nested
|
||||||
|
// values define field names, nesting, and omission of absent optional fields.
|
||||||
|
// Formatting belongs to the selected encoder.
|
||||||
|
//
|
||||||
|
// Other construction, inspection, handle, and error values, including [Config],
|
||||||
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
|
||||||
// [ProfileConfig], [RunRequest],
|
// [ProfileConfig], [RunRequest],
|
||||||
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
|
// [ArtifactRef], [ExecutionTargetOverride],
|
||||||
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
|
// [PromptIdentity], [PromptInputDefinition],
|
||||||
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
|
||||||
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
// [CapacityError], [GenerationError], [RetryExhaustedError], and
|
||||||
// [ExecutionEvent], do not have stable JSON
|
// [ExecutionEvent], do not have stable JSON
|
||||||
// representations. Direct API keys are nevertheless excluded from JSON for
|
// representations. Direct API keys are nevertheless excluded from JSON for
|
||||||
// every public value.
|
// every public value.
|
||||||
|
//
|
||||||
// Provider-derived [GenerationError] accessor values are untrusted and can
|
// Provider-derived [GenerationError] accessor values are untrusted and can
|
||||||
// contain sensitive request or schema fragments. Applications must apply their
|
// contain sensitive request or schema fragments. Applications must apply their
|
||||||
// own disclosure policy before logging, displaying, or returning them.
|
// own disclosure policy before logging, displaying, or returning them.
|
||||||
|
|||||||
@@ -135,12 +135,16 @@ Replace former maps passed to `BackendSource` in the same way, setting each
|
|||||||
inspection paths now address list indexes, for example
|
inspection paths now address list indexes, for example
|
||||||
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
`configkit.Keys("profiles").Index(0).Key("model")`. Configuration inspection
|
||||||
and diffs expose resolved values, recorded normalization and provenance.
|
and diffs expose resolved values, recorded normalization and provenance.
|
||||||
`InspectProfile` additionally applies backend completion; request preparation
|
`InspectProfile` reports the selected resolved profile without backend completion.
|
||||||
can apply further overrides, so the views are not promised to be identical.
|
Use `Prepare` or `PreparedExecution.Details` for completed request settings,
|
||||||
|
including backend defaults and request overrides; see the
|
||||||
|
[resource inspection guidance](pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
||||||
`prompt_version` JSON members. Prompt hashes are opaque equality values and may
|
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
||||||
change when the removed field no longer participates in hashing.
|
values and may also change because redundant captured fields were removed.
|
||||||
|
Invalidate persisted caches or comparisons that span this change; hashes from
|
||||||
|
inspection and preparation of the same captured definition still agree.
|
||||||
|
|
||||||
## Identity Mapping
|
## Identity Mapping
|
||||||
|
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ engine, err := promptkit.NewEngine(promptkit.Config{
|
|||||||
|
|
||||||
For a single highest-priority document, set `Sources.Main` to a
|
For a single highest-priority document, set `Sources.Main` to a
|
||||||
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
`*configkit.Input`; omit it for ordinary layered construction. A single file
|
||||||
can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
|
can use `configkit.File` directly. `PromptSource`, `ProfileSource`, `BackendSource`, and raw
|
||||||
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
`configkit.Mapping` supply programmatic definitions. Schema and runtime
|
||||||
collaborator options remain available. Construction validates the final
|
collaborator options remain available. Construction validates the final
|
||||||
effective configuration and captures referenced prompt content; construct a
|
effective configuration and captures referenced prompt content; construct a
|
||||||
@@ -177,8 +177,17 @@ input := configkit.Input{Source: document}
|
|||||||
// Add input to a later underlay, or set Sources.Main = &input.
|
// Add input to a later underlay, or set Sources.Main = &input.
|
||||||
```
|
```
|
||||||
|
|
||||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
`PromptSource`, `ProfileSource`, and `BackendSource` are typed conveniences when
|
||||||
is not needed. Backend and profile definitions are supplied by the caller;
|
null is not needed. Prompt and backend helpers, like profiles, support sibling
|
||||||
|
inheritance via `Extends`; partial records can be completed by other layers.
|
||||||
|
For relative prompt `content_file` references, supply the ordinary ConfigKit
|
||||||
|
`Input.Filesystem` context for that source. The helpers do not read files or
|
||||||
|
validate final definitions. Profile and backend provider maps use ConfigKit Mapping input forms: built-in
|
||||||
|
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
|
||||||
|
typed containers explicitly to an appropriate supported value, choosing numeric
|
||||||
|
precision for your application. Mapping captures supported values immediately;
|
||||||
|
unsupported or cyclic values are reported when loaded. Runtime override values
|
||||||
|
follow their separate rules. Backend and profile definitions are supplied by the caller;
|
||||||
there is no registration-only backend namespace.
|
there is no registration-only backend namespace.
|
||||||
|
|
||||||
When configuration or message files change, build and validate a new engine,
|
When configuration or message files change, build and validate a new engine,
|
||||||
@@ -259,9 +268,10 @@ is not necessarily safe to share. Inspection uses the engine's captured result
|
|||||||
and does not reopen sources or resolve credentials. It retains effective values,
|
and does not reopen sources or resolve credentials. It retains effective values,
|
||||||
including recorded field normalization and queue defaulting, alongside source
|
including recorded field normalization and queue defaulting, alongside source
|
||||||
provenance.
|
provenance.
|
||||||
`InspectProfile` instead shows backend-completed execution settings for one profile, and `Prepare` or
|
`InspectPrompt`, `InspectProfile`, and `InspectBackend` instead show individual
|
||||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
resolved resources without backend-to-profile execution completion or provenance.
|
||||||
and validation resources. None of those views is a substitute for the others.
|
`Prepare` or `PreparedExecution.Details` shows completed request-specific settings,
|
||||||
|
rendered messages, inputs, and validation resources. None substitutes for the others.
|
||||||
|
|
||||||
To review changes between two independently captured configurations, compare
|
To review changes between two independently captured configurations, compare
|
||||||
their inspection views:
|
their inspection views:
|
||||||
@@ -375,8 +385,10 @@ if err != nil {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, input := range inspection.Inputs {
|
if inspection.Inputs != nil {
|
||||||
// Compare the declared input with application configuration.
|
for _, input := range *inspection.Inputs {
|
||||||
|
// Compare the declared input with application configuration.
|
||||||
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -702,10 +714,9 @@ composition rules.
|
|||||||
|
|
||||||
### Inspect A Profile Before Prompt Work
|
### Inspect A Profile Before Prompt Work
|
||||||
|
|
||||||
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
|
Use [`Engine.InspectProfile`](../../engine.go) to inspect a resolved profile
|
||||||
profile without constructing a synthetic prompt or placeholder inputs. It
|
without constructing a synthetic prompt or placeholder inputs. It reports the
|
||||||
resolves the profile's effective target but does not prepare or execute a
|
profile's own fields, not a backend-completed target:
|
||||||
prompt:
|
|
||||||
|
|
||||||
```go
|
```go
|
||||||
inspection, err := engine.InspectProfile(ctx, profileID)
|
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||||
@@ -713,23 +724,48 @@ if err != nil {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
if inspection.APIKeyRequired != nil && *inspection.APIKeyRequired {
|
||||||
if target.APIKeyEnv != "" {
|
// Arrange a usable direct or configured environment credential.
|
||||||
// This is a configured optional environment lookup source.
|
}
|
||||||
} else if inspection.APIKeyRequired {
|
if inspection.APIKeyEnv != nil {
|
||||||
// Arrange a direct credential before later execution.
|
// This is the configured environment lookup source, not its value.
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Use this configuration-time boundary when only the profile and its target need
|
Use [`InspectBackend`](../../engine.go) to inspect the referenced backend
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
separately. Its diagnostics are configured metadata, not live checks or derived
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
destinations; `ErrBackendNotFound` identifies an absent backend. References in
|
||||||
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
all three resource reports remain as authored. Use `Prepare` for completed
|
||||||
`APIKeyRequired` is the explicit local requirement. The
|
request settings and rendering, or prepared execution when that work must remain
|
||||||
|
tied to later execution. A reported `APIKeyEnv` identifies a configured source,
|
||||||
|
not its value; `APIKeyRequired` can also be present. The
|
||||||
[credential format reference](../formats.md#credentials) and the method's
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
contracts.
|
contracts.
|
||||||
|
|
||||||
|
Resource inspection pointers represent optional fields: nil is absent, while a
|
||||||
|
non-nil pointer preserves zero, false, or an empty string. Pointer-to-collection
|
||||||
|
fields distinguish omission from an empty map or list. Results and nested mutable
|
||||||
|
values belong to the caller. Ordinary generic encoders need no per-field adapter:
|
||||||
|
|
||||||
|
```go
|
||||||
|
jsonBytes, err := json.Marshal(inspection)
|
||||||
|
if err != nil { return err }
|
||||||
|
yamlBytes, err := yaml.Marshal(inspection) // go.yaml.in/yaml/v4
|
||||||
|
if err != nil { return err }
|
||||||
|
_, _ = jsonBytes, yamlBytes
|
||||||
|
```
|
||||||
|
|
||||||
|
For migration, replace `ProfileInspection.EffectiveModelParams` accesses with
|
||||||
|
the optional resource fields, and use `Prepare` or `PreparedExecution.Details`
|
||||||
|
where backend completion and request overrides are needed. Prompt inspection's
|
||||||
|
`DefaultProfileID`, `Inputs`, nested input metadata, and output fields now use
|
||||||
|
optional pointers where presence matters; check for nil before dereferencing.
|
||||||
|
Consumers such as CommandKit can render the results directly instead of copying
|
||||||
|
each field into a parallel report. See the result types' [GoDoc](../../resource_inspection.go)
|
||||||
|
for keys and ownership; resource inspection does not replace `InspectConfiguration`
|
||||||
|
provenance, `InspectRequirements`, or concrete-request `ValidateRequest`.
|
||||||
|
|
||||||
### Set A Per-Run Session And Reasoning
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
Supply a direct session ID when one prompt should be correlated with a
|
Supply a direct session ID when one prompt should be correlated with a
|
||||||
@@ -879,6 +915,12 @@ secret-handling policies.
|
|||||||
|
|
||||||
## Extension Interfaces
|
## Extension Interfaces
|
||||||
|
|
||||||
|
JSON Schema validation and native structured output use a self-contained root
|
||||||
|
document. Internal references are supported; references to other files or
|
||||||
|
network resources fail during preparation. `WithSchemaFile` selects one root
|
||||||
|
file, while `WithSchemaFS` and `SchemaDir` select a root source. Prepared
|
||||||
|
executions retain their captured schema even if the source later changes.
|
||||||
|
|
||||||
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||||
behavior does not fit the application. Their GoDoc defines concurrent use,
|
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||||
context handling, ownership of copied values, nil responses, and preservation
|
context handling, ownership of copied values, nil responses, and preservation
|
||||||
@@ -886,6 +928,11 @@ of collaborator errors. Implementations must honor cancellation, safely manage
|
|||||||
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||||
the application policy that motivated the injection.
|
the application policy that motivated the injection.
|
||||||
|
|
||||||
|
A successful `ArtifactReader.Read` transfers the returned artifact and mutable
|
||||||
|
body storage to Promptkit. Readers that previously reused a returned buffer
|
||||||
|
must instead return independent storage and stop mutating it after return;
|
||||||
|
private scratch buffers may still be reused when the returned bytes are copied.
|
||||||
|
|
||||||
## Handle Errors
|
## Handle Errors
|
||||||
|
|
||||||
Use `errors.Is` with the
|
Use `errors.Is` with the
|
||||||
|
|||||||
@@ -58,6 +58,14 @@ exist.
|
|||||||
|
|
||||||
## Maintainer Validation
|
## Maintainer Validation
|
||||||
|
|
||||||
|
Use the newest Go installation already available locally for ordinary
|
||||||
|
development checks. Put its `bin` directory first on `PATH` and set
|
||||||
|
`GOTOOLCHAIN=local` to keep that toolchain selected. For repository-isolated
|
||||||
|
checks, set `GOWORK=off`; consumers may use their own workspaces. The selected
|
||||||
|
validation patch does not by itself change the module's consumer minimum. See
|
||||||
|
the [release toolchain policy](policy/release.md#consumer-minimum-and-release-toolchain)
|
||||||
|
and [release procedure](release.md#prepare-the-candidate) for release selection.
|
||||||
|
|
||||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||||
successful checks for unchanged content merely to create a commit. During
|
successful checks for unchanged content merely to create a commit. During
|
||||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||||
|
|||||||
@@ -110,8 +110,8 @@ and static template syntax at construction. It does not reread configuration or
|
|||||||
message files on ordinary runs; construct a new engine to observe edits.
|
message files on ordinary runs; construct a new engine to observe edits.
|
||||||
Schema and request artifact reads still happen during preparation. Use
|
Schema and request artifact reads still happen during preparation. Use
|
||||||
`InspectConfiguration` for an explicit effective-configuration or provenance
|
`InspectConfiguration` for an explicit effective-configuration or provenance
|
||||||
report, `InspectProfile` for backend-completed profile settings, and preparation
|
report, resource inspection for resolved prompt, profile, and backend fields,
|
||||||
for request-specific rendered details. For identity discovery before inspection,
|
and preparation for completed request-specific details. For identity discovery before inspection,
|
||||||
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
see the consumer guide's [resource and collection listing](consumers/pkg-promptkit.md#discover-available-resources-and-collections).
|
||||||
|
|
||||||
## Prompt Definitions
|
## Prompt Definitions
|
||||||
@@ -315,17 +315,33 @@ only a model or backend does not clear unrelated explicitly inherited fields.
|
|||||||
| Profile field | Absent | Explicit value | Explicit null |
|
| Profile field | Absent | Explicit value | Explicit null |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
| `backend` | No backend | Exact backend ID; blank means none | No backend |
|
||||||
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Clear, without backend fallback |
|
| `endpoint` | Selected backend endpoint | Valid nonblank URL; empty clears | Selected backend endpoint |
|
||||||
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
| `model` | Unspecified | Nonblank model; empty clears | Unspecified |
|
||||||
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
| `temperature`, `top_p`, `max_tokens` | Unspecified | Valid value, including outbound zero | Suppress outbound control |
|
||||||
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
| `service_tier`, `reasoning_effort` | Unspecified | Normalized value; blank suppresses | Suppress outbound control |
|
||||||
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
| `timeout_seconds` | 600 seconds | Nonnegative; zero disables generation deadline | Reset to 600 seconds |
|
||||||
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Disable backend lookup |
|
| `api_key_env` | Backend lookup name, if any | Trimmed name; blank disables | Backend lookup name, if any |
|
||||||
| `api_key_required` | False | Boolean, including false | False |
|
| `api_key_required` | False | Boolean, including false | False |
|
||||||
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Suppress configured provider map |
|
| `extra_params` | Backend defaults | Recursively overlay profile map on backend map | Backend defaults |
|
||||||
|
|
||||||
|
An empty profile mapping does not clear backend map entries. After ConfigKit
|
||||||
|
composition and inheritance, a whole-map null and omission both permit backend
|
||||||
|
defaults. Nested maps merge recursively, while scalar and array values replace;
|
||||||
|
nulls within provider maps or arrays remain literal data.
|
||||||
|
When `api_key_required` is true and the profile source is absent or null,
|
||||||
|
execution suppresses the backend credential source; an explicitly configured
|
||||||
|
profile source remains in use.
|
||||||
|
|
||||||
|
For example, if the backend provides an endpoint, API-key environment name,
|
||||||
|
and provider map, a profile containing `endpoint: null`, `api_key_env: null`,
|
||||||
|
and `extra_params: null` now inherits those backend values after handoff (unless
|
||||||
|
the required-key rule suppresses the credential source). Earlier versions used
|
||||||
|
those nulls to suppress the corresponding fallback. Use `endpoint: ""` or
|
||||||
|
`api_key_env: ""` to clear those individual strings. There is no profile-level
|
||||||
|
whole-map suppression replacement; change the backend map through ordinary
|
||||||
|
ConfigKit composition when different provider defaults are needed. ConfigKit's
|
||||||
|
retained configuration and provenance continue to show the nulls.
|
||||||
|
|
||||||
An empty mapping does not clear inherited map entries; null clears the whole
|
|
||||||
map. Nested maps merge recursively, while scalar and array values replace.
|
|
||||||
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
Runtime nonempty `ExtraParams` overrides replace the entire configured map.
|
||||||
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
Request overrides otherwise follow the [public value GoDoc](../types.go).
|
||||||
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
For example, a runtime `extra_params["extends"]` key is ordinary provider
|
||||||
@@ -418,11 +434,17 @@ generic inheritance and layer mechanics, consult
|
|||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
Directory and `fs.FS` schema paths resolve within the configured root. Each
|
||||||
referenced nested schemas resolve relative to the owning schema document.
|
schema must be self-contained: in-document references, IDs, and anchors are
|
||||||
|
supported, but references requiring another file or network resource fail.
|
||||||
|
References must also work without the source filename as a base; use fragment
|
||||||
|
references or explicit schema IDs rather than relying on that filename.
|
||||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
||||||
invalid, or unresolvable schema causes an operational validation error.
|
invalid, or unresolvable schema causes an operational validation error.
|
||||||
Valid JSON that fails the schema instead yields a failed validation result.
|
Valid JSON that fails the schema instead yields a failed validation result.
|
||||||
|
Schema documents are detached without applying the provider extra-parameter
|
||||||
|
copy limits; this does not change the schema decoder, compiler, or
|
||||||
|
self-contained-resource rules.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
|
|||||||
@@ -16,14 +16,13 @@ Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
|||||||
Before the client is called, the engine resolves framework, backend, profile,
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
and request values into one execution target. Endpoint configuration is trimmed
|
and request values into one execution target. Endpoint configuration is trimmed
|
||||||
and must be an absolute HTTP or HTTPS URL with a host and without user
|
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||||
information, a query, or a fragment. A non-empty endpoint from the target
|
information, a query, or a fragment. The target's endpoint is validated again
|
||||||
overrides the client's configured base URL. The final selected endpoint is
|
before transport.
|
||||||
validated again before transport.
|
|
||||||
|
|
||||||
The completion URL is composed through parsed URL path operations. Nested base
|
The completion URL is composed through parsed URL path operations. Nested base
|
||||||
paths are retained, repeated trailing slashes are normalized, and the result
|
paths are retained, repeated trailing slashes are normalized, and the result
|
||||||
has exactly one appended `/chat/completions` suffix. Generation fails before
|
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||||
sending when neither source supplies a valid endpoint.
|
sending when the target does not supply a valid endpoint.
|
||||||
|
|
||||||
The target's backend ID is routing metadata for prepared values, results, and
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
injected clients. The built-in client does not derive the URL from that ID and
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
@@ -70,7 +69,9 @@ and numeric `usage`; unrelated fields and account limits are ignored. This is
|
|||||||
only a minimal identifying subset of OpenRouter's documented
|
only a minimal identifying subset of OpenRouter's documented
|
||||||
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||||
not validation of the full account document. Other statuses and unrecognized
|
not validation of the full account document. Other statuses and unrecognized
|
||||||
200 responses do not establish acceptance.
|
200 responses do not establish acceptance. A 200 response exceeding the bounded
|
||||||
|
recognition allowance reports `authentication_response_limit_exceeded`, leaving
|
||||||
|
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||||
RoundTripper boundary and any response associated with it. It includes the
|
RoundTripper boundary and any response associated with it. It includes the
|
||||||
@@ -87,8 +88,7 @@ transport for HTTP-only diagnostics.
|
|||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
target's model takes precedence over the client's configured model, and one
|
target's model must be available.
|
||||||
must be available.
|
|
||||||
|
|
||||||
Each ordinary message contains its `role` and string `content`. A
|
Each ordinary message contains its `role` and string `content`. A
|
||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
@@ -115,9 +115,10 @@ The client conditionally includes:
|
|||||||
strict flag, and schema document. Plain JSON validation does not add an
|
strict flag, and schema document. Plain JSON validation does not add an
|
||||||
object-only response constraint.
|
object-only response constraint.
|
||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
Configured profile maps recursively overlay backend defaults, while a nonempty
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
runtime override replaces the complete configured map. The [format reference](../formats.md#defaults-and-overrides)
|
||||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
owns these resolution rules. The resulting effective map is merged into the
|
||||||
|
top-level body. Empty keys, values that cannot be encoded, and
|
||||||
collisions with these reserved fields are rejected before any provider call:
|
collisions with these reserved fields are rejected before any provider call:
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
|
|||||||
@@ -68,9 +68,8 @@ Preparation and validation never hold an active permit.
|
|||||||
|
|
||||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||||
returns an idempotent release function. The coordinator uses it around
|
returns an idempotent release function. The coordinator uses it around
|
||||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
individual attempts. There is one queue and one active limit per limited backend
|
||||||
operation for focused internal callers and tests, but engine assembly no
|
pool.
|
||||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
|
||||||
|
|
||||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||||
@@ -121,7 +120,7 @@ validation, bounded admission, idempotent release, context handling, and
|
|||||||
unlimited admission. The
|
unlimited admission. The
|
||||||
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
unlimited calls, passthrough behavior, and panic release.
|
unlimited calls, and panic release using test-only one-shot calls.
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
admission, lease lifetime, failure release, and shared initial/repair
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
|
|||||||
@@ -26,10 +26,9 @@ client uses its endpoint, credential metadata, generation fields, and extra
|
|||||||
parameters. `BackendID` remains routing metadata
|
parameters. `BackendID` remains routing metadata
|
||||||
for the generation boundary and is not mapped into the provider payload.
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
Construction trims and validates a nonempty configured base URL and clones any
|
Construction clones any supplied `http.Client` so Promptkit can apply its
|
||||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
timeout default without mutating the caller's client. Generation preparation
|
||||||
mutating the caller's client. An empty configured base remains valid because a
|
then:
|
||||||
resolved request target may supply the endpoint. Generation preparation then:
|
|
||||||
|
|
||||||
1. validates shared execution-setting invariants and the final selected base
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
endpoint;
|
endpoint;
|
||||||
@@ -42,6 +41,10 @@ resolved request target may supply the endpoint. Generation preparation then:
|
|||||||
the complete-request failure remains a request failure;
|
the complete-request failure remains a request failure;
|
||||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||||
|
|
||||||
|
Provider values are detached with trusted copying before encoding. Custom JSON
|
||||||
|
encoding methods and per-value encoding-failure diagnostics operate on those
|
||||||
|
private copies rather than retained prepared or diagnostic metadata.
|
||||||
|
|
||||||
Each execution of that prepared request creates a fresh HTTP request, body
|
Each execution of that prepared request creates a fresh HTTP request, body
|
||||||
reader, and attempt timeout context. It closes the response body and returns
|
reader, and attempt timeout context. It closes the response body and returns
|
||||||
the HTTP status, decoded response or error, reported token values and presence,
|
the HTTP status, decoded response or error, reported token values and presence,
|
||||||
@@ -77,7 +80,8 @@ not retain headers, credentials, raw request or response bodies, or a redirect
|
|||||||
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||||
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||||
Reachability and named authentication probes use the same configured built-in
|
Reachability and named authentication probes use the same configured built-in
|
||||||
client but discard their response bodies after local recognition. The
|
client. The OpenRouter adapter owns endpoint derivation, bounded account-response
|
||||||
|
recognition, and body closure; the smoke coordinator interprets its evidence. The
|
||||||
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||||
owns their observable HTTP semantics.
|
owns their observable HTTP semantics.
|
||||||
|
|
||||||
@@ -105,14 +109,13 @@ public ownership and redaction semantics belong to the
|
|||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid generation
|
||||||
configuration, invalid generation requests, request execution failures,
|
requests, request execution failures, non-success provider statuses, and
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
malformed successful responses. Provider response bodies are never exposed in
|
||||||
response bodies are never exposed in raw form through non-success errors.
|
raw form through non-success errors.
|
||||||
|
|
||||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
A missing or invalid final selected endpoint is an invalid generation request
|
||||||
invalid final selected endpoint is an invalid generation request and is
|
and is rejected before transport.
|
||||||
rejected before transport.
|
|
||||||
|
|
||||||
Authentication resolves a trimmed direct key before a trimmed configured
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ lifecycle instrumentation spans the facade and runner boundaries.
|
|||||||
Prepared execution starts only after its private payload is atomically claimed;
|
Prepared execution starts only after its private payload is atomically claimed;
|
||||||
the facade receives that invocation's recorder privately and emits the terminal
|
the facade receives that invocation's recorder privately and emits the terminal
|
||||||
event after public error mapping. Failed claims and preparation emit nothing.
|
event after public error mapping. Failed claims and preparation emit nothing.
|
||||||
|
The public projection retains private operation-owner metadata separately from
|
||||||
|
inspectable child causes. Terminal classification uses that owner, preserving
|
||||||
|
context precedence and generation-specific transport/provider details rather
|
||||||
|
than mistaking a collaborator's nested lookup error for the parent's operation.
|
||||||
|
|
||||||
The facade masks inherited recorder and prepared-claim state when entering
|
The facade masks inherited recorder and prepared-claim state when entering
|
||||||
preparation or execution. Delegated calls from an injected collaborator retain
|
preparation or execution. Delegated calls from an injected collaborator retain
|
||||||
|
|||||||
@@ -11,19 +11,19 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, collection and resource mapping conveniences, injection options, public requests and results, resource and requirement inspection and identity inventories, concrete-request validation, explicit smoke diagnostics, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [smoke diagnostics](../../smoke_diagnostics.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
| `examples/go-library/prepare` | Demonstrates offline preparation from two explicitly selected synthetic collections, source-relative imported content, a scoped view, filtered listings, and operator inventory. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go), [collection inputs](../../examples/go-library/prepare/collections/support/collection.yaml) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||||
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Holds a defensive, immutable lookup of final backend definitions, including diagnostic metadata, plus capacity policies validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/configuration` | Loads root and namespaced collection resources through ConfigKit; records effective field presence, validates final values including exact references and optional diagnostic metadata, captures indexed prompt definitions and source-relative message content, compiled templates, input metadata snapshots, sorted backend/profile/prompt identities, resource namespace sets, effective collection declarations, backend and presence-aware profile snapshots, and the unchanged read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
| `internal/configuration` | Loads root and namespaced resources through ConfigKit; validates final values, captures indexed resource views and source-relative prompt content, compiled templates, requirement metadata, identity inventories, backend and presence-aware execution profiles, and the read-only provenance inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
|
||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
| `internal/engine` | Assembles caller-selected configuration into one snapshot, then constructs shared capacity, rendering, validation, generation, repair, the ordinary runner, and the smoke tester for each engine. Immutable namespace views share those collaborators and qualify caller-supplied execution, inspection, and diagnostic selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and identity listing directly to the captured configuration rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
|
|||||||
@@ -42,14 +42,13 @@ Ordinary preparation selects the full prompt definition and construction-time
|
|||||||
hash. It retains its early request-ID check before direct-session
|
hash. It retains its early request-ID check before direct-session
|
||||||
normalization. Referenced message content and static template syntax were
|
normalization. Referenced message content and static template syntax were
|
||||||
captured when the engine was built.
|
captured when the engine was built.
|
||||||
|
Rendering uses those captured parses, cloning them per call to bind request
|
||||||
|
input helpers; it does not parse templates during execution.
|
||||||
|
|
||||||
`InspectPrompt` selects captured metadata through an optional repository
|
Resource inspection bypasses the runner: the internal engine selects a captured
|
||||||
capability, falling back to the full definition and existing hash resolution
|
configuration resource, and the facade detaches its values for the caller.
|
||||||
for internal repositories without it. It returns owned metadata without
|
Prompt resource inspection includes captured message text without rendering or
|
||||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
touching profile, artifact, schema, admission, or model collaborators.
|
||||||
admission, or model collaborators. The root
|
|
||||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
|
||||||
exact contract.
|
|
||||||
|
|
||||||
`InspectRequirements` uses the same metadata selection to resolve the
|
`InspectRequirements` uses the same metadata selection to resolve the
|
||||||
request-profile/default identity, then obtains the captured requirement
|
request-profile/default identity, then obtains the captured requirement
|
||||||
@@ -57,24 +56,18 @@ snapshot by the selected prompt ID. The analyzer receives only
|
|||||||
those identity fields rather than an execution definition. It analyzes the
|
those identity fields rather than an execution definition. It analyzes the
|
||||||
snapshot with the supplied caller variables and normalized direct-session
|
snapshot with the supplied caller variables and normalized direct-session
|
||||||
bypass; it does not resolve target readiness or enter the preparation
|
bypass; it does not resolve target readiness or enter the preparation
|
||||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
pipeline. An unavailable captured snapshot is a prompt-load failure. The root
|
||||||
source fallback. The root
|
|
||||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||||
operation's exact contract.
|
operation's exact contract.
|
||||||
|
|
||||||
## Shared Profile Selection
|
## Shared Profile Selection
|
||||||
|
|
||||||
The runner uses one profile-selection and target-resolution boundary for
|
The runner selects a request profile or a prompt default and uses its captured,
|
||||||
ordinary preparation and exact profile inspection. Preparation first selects a
|
presence-aware target constructed from final profile and backend configuration.
|
||||||
request profile or a prompt default; inspection begins with its required
|
Request overrides apply afterward; preparation checks endpoint and model
|
||||||
explicit profile ID. Both use the captured, presence-aware target constructed
|
completeness only after those overrides. Resource profile inspection bypasses
|
||||||
from the final effective profile and backend configuration. Request overrides
|
this path and reports the profile before backend completion. Credential
|
||||||
apply afterward; preparation checks endpoint and model completeness only after
|
availability is checked when execution preparation requires it.
|
||||||
those overrides. Inspection can report a partial profile without requiring a
|
|
||||||
runnable target. Neither path checks credential availability or performs prompt,
|
|
||||||
artifact, schema, rendering, admission, or model-client work. The root
|
|
||||||
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
|
|
||||||
exact contract.
|
|
||||||
|
|
||||||
## Smoke-Test Diagnostics
|
## Smoke-Test Diagnostics
|
||||||
|
|
||||||
@@ -142,11 +135,10 @@ one execution-precedence and error-ordering implementation while allowing a
|
|||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations. `Prepare` discards the plan after returning its public
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
and discards it when the operation ends. Prepared execution transfers the
|
and discards it when the operation ends. Prepared execution retains one
|
||||||
completed preparation's owned message, input-hash, and copied-schema values
|
immutable, redacted metadata snapshot shared by execution and internal details.
|
||||||
into its private payload, while deep-copying mutable provider extras and
|
The facade detaches mutable public details. Claim completion or discard clears
|
||||||
creating an independent details snapshot. Claim completion or discard clears
|
execution-only payload references without modifying the retained metadata.
|
||||||
the execution payload without removing the details snapshot.
|
|
||||||
|
|
||||||
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||||
path as `Prepare` and discards the resulting internal prepared value without
|
path as `Prepare` and discards the resulting internal prepared value without
|
||||||
@@ -176,7 +168,7 @@ encoding of that session, every role and content value, and cache-control
|
|||||||
presence and values; its hexadecimal value is opaque.
|
presence and values; its hexadecimal value is opaque.
|
||||||
|
|
||||||
The configuration snapshot is read-only after engine construction. Concurrent
|
The configuration snapshot is read-only after engine construction. Concurrent
|
||||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
`Prepare` and `Run` calls use detached captured targets and keep invocation
|
||||||
state local.
|
state local.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ source semantics. Internal packages are not consumer APIs.
|
|||||||
## Typed Configuration Loading
|
## Typed Configuration Loading
|
||||||
|
|
||||||
`internal/configuration` is the single adapter from ConfigKit's final
|
`internal/configuration` is the single adapter from ConfigKit's final
|
||||||
effective configuration to execution-owned definitions. It performs one
|
effective configuration to execution-owned definitions and resource views. It performs one
|
||||||
layered load, captures effective presence (including explicit null), records
|
layered load, captures the needed effective presence, records
|
||||||
field-level normalization and derived defaults during resolution, and validates all effective
|
field-level normalization and derived defaults during resolution, and validates all effective
|
||||||
root and collection backend, profile, and prompt values. Replaced lower-layer
|
root and collection backend, profile, and prompt values. Replaced lower-layer
|
||||||
domain values are not separately validated. It indexes root resources under
|
domain values are not separately validated. It indexes root resources under
|
||||||
@@ -85,6 +85,8 @@ returning and retains no source capability for public inspection.
|
|||||||
|
|
||||||
Prompt materialization shares a lazy metadata snapshot across root and collection
|
Prompt materialization shares a lazy metadata snapshot across root and collection
|
||||||
resources during construction, avoiding a full metadata copy per collection.
|
resources during construction, avoiding a full metadata copy per collection.
|
||||||
|
Input metadata, message content and templates, and output contracts are captured
|
||||||
|
separately before the definition, inspection, and hash are assembled.
|
||||||
|
|
||||||
Normalization and defaults are recorded at individual resolved field paths,
|
Normalization and defaults are recorded at individual resolved field paths,
|
||||||
including message roles and cache controls. List indexes identify fields for
|
including message roles and cache controls. List indexes identify fields for
|
||||||
@@ -92,13 +94,13 @@ presence, validation, sensitivity, and resource context; full IDs select
|
|||||||
captured runtime indexes. Unchanged content-file fields retain their origins.
|
captured runtime indexes. Unchanged content-file fields retain their origins.
|
||||||
Backend-to-profile and request-time completion remain separate runtime work.
|
Backend-to-profile and request-time completion remain separate runtime work.
|
||||||
|
|
||||||
The runner selects full definitions from this index for preparation and uses
|
The runner selects full definitions from this index for preparation and retains
|
||||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
lightweight prompt metadata for `InspectRequirements`; the requirement snapshot
|
||||||
internal requirement snapshot lookup uses the selected prompt ID alone.
|
lookup uses the selected prompt ID alone. Separate configuration-owned resource
|
||||||
These paths do not rescan configuration or reopen message files. `InspectPrompt`
|
inspection projects the resolved record and captured messages without rendering
|
||||||
returns declared metadata without rendering or resolving profile, input, or
|
or resolving profile, input, or schema resources. These paths do not rescan
|
||||||
schema resources, and its public shape and prompt hashes do not include the
|
configuration or reopen message files. Presence metadata does not change the
|
||||||
captured presence state. Runtime rendering still checks request variables and
|
opaque definition hash. Runtime rendering still checks request variables and
|
||||||
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
|
||||||
cover resource origins, context containment, capture, and selection.
|
cover resource origins, context containment, capture, and selection.
|
||||||
|
|
||||||
@@ -146,11 +148,13 @@ change rendering behavior.
|
|||||||
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||||
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
||||||
The configuration adapter validates final effective resource references and
|
The configuration adapter validates final effective resource references and
|
||||||
field values, then builds target
|
field values, then captures backend definitions and profile target snapshots
|
||||||
snapshots with backend defaults and presence/suppression state. A profile can
|
with backend defaults and outbound-control presence. Null profile endpoint,
|
||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
credential source, and whole provider map permit backend fallback after
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
composition, subject to the required-credential rule. A profile can remain
|
||||||
provider work. Preparation applies runtime overrides and then checks target
|
partial until runtime overrides complete it. Resource inspection selects indexed
|
||||||
|
resolved backend and profile records without execution completion or prompt,
|
||||||
|
artifact, schema, credential lookup, or provider work. Preparation applies runtime overrides and then checks target
|
||||||
completeness. Identity-only profile selection can identify a partial captured
|
completeness. Identity-only profile selection can identify a partial captured
|
||||||
profile without copying its target or requiring readiness. The
|
profile without copying its target or requiring readiness. The
|
||||||
[target tests](../../internal/configuration/targets_test.go) cover effective
|
[target tests](../../internal/configuration/targets_test.go) cover effective
|
||||||
@@ -216,12 +220,14 @@ instead of first making a response-sized string copy.
|
|||||||
|
|
||||||
Every preparation operation creates one operation-local validation plan. None,
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
basic, and JSON modes retain the effective output contract without source
|
basic, and JSON modes retain the effective output contract without source
|
||||||
access. JSON Schema mode loads the root document once, resolves and compiles
|
access. JSON Schema mode loads the root document once, compiles its declared
|
||||||
each transitive reference, and retains the compiled validator. Schema compiler
|
schema locations, and retains the compiled root validator. External resources
|
||||||
resources use canonical escaped file or private-scheme URLs; loaders decode
|
are rejected without file or network loading; in-document references remain
|
||||||
their paths once and enforce the configured source boundary. The
|
supported. Root resources use canonical escaped file or private-scheme URLs. The
|
||||||
provider-facing structured-output metadata uses the root document captured by
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
the same plan.
|
the same plan. Its detached schema document is not revalidated against the
|
||||||
|
runtime provider-value copier's traversal limits. Smoke inference uses a
|
||||||
|
separate JSON-only prepared plan before checking the prepared schema plan.
|
||||||
|
|
||||||
Schema preparation and execution remain synchronous. Promptkit checks
|
Schema preparation and execution remain synchronous. Promptkit checks
|
||||||
cancellation before and after source resolution, JSON decoding, compilation,
|
cancellation before and after source resolution, JSON decoding, compilation,
|
||||||
|
|||||||
80
docs/releases/v0.19.0.md
Normal file
80
docs/releases/v0.19.0.md
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
# Promptkit v0.19.0
|
||||||
|
|
||||||
|
This release completes two rounds of audit remediation, correcting error and
|
||||||
|
diagnostic reporting and simplifying configuration, validation, and value
|
||||||
|
ownership. It includes intentional compatibility changes for custom artifact
|
||||||
|
readers, programmatic configuration sources, and schemas. The minimum supported
|
||||||
|
Go version remains 1.27.0; ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.18.0 should review these changes:
|
||||||
|
|
||||||
|
- A successful `ArtifactReader.Read` transfers ownership of the returned
|
||||||
|
artifact and its mutable body storage to Promptkit. Custom readers must stop
|
||||||
|
mutating returned values through retained aliases. Readers that reuse scratch
|
||||||
|
buffers must return independent storage. Promptkit retains its defensive
|
||||||
|
body copy, but that copy does not make concurrent buffer reuse safe. See the
|
||||||
|
[extension guidance](../consumers/pkg-promptkit.md#extension-interfaces).
|
||||||
|
- Provider maps passed through `ProfileSource` and `BackendSource` now use
|
||||||
|
ConfigKit's native Mapping input forms directly: built-in scalars, `[]any`,
|
||||||
|
and `map[string]any` recursively. Convert `json.Number` and typed containers
|
||||||
|
explicitly to supported values, choosing numeric precision appropriate to
|
||||||
|
the application. The former recursive compatibility conversion is removed.
|
||||||
|
Runtime execution overrides retain their separate accepted-value behavior.
|
||||||
|
- Schemas used for local validation and native structured output must be
|
||||||
|
self-contained. Move externally referenced definitions into the root document
|
||||||
|
and use internal references or explicit schema IDs. References requiring
|
||||||
|
another file or network resource, or relying on the source filename as a base,
|
||||||
|
fail during preparation. Promptkit does not bundle external schemas. See the
|
||||||
|
[schema reference](../formats.md#schemas).
|
||||||
|
|
||||||
|
Prepared schemas no longer inherit the depth and node limits used when copying
|
||||||
|
runtime provider values. Schemas previously rejected solely by those incidental
|
||||||
|
limits may now be accepted; decoder/compiler checks and the self-contained
|
||||||
|
schema requirement still apply.
|
||||||
|
|
||||||
|
No configuration-format migration is required from v0.18.0. After addressing
|
||||||
|
any affected integrations:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.19.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Errors And Diagnostics
|
||||||
|
|
||||||
|
Public errors and terminal observations now preserve the category of the
|
||||||
|
operation that failed even when an injected collaborator returns an error from
|
||||||
|
another Promptkit operation. Underlying causes remain available through
|
||||||
|
`errors.Is` and `errors.As`, without repeatedly duplicating nested cause trees.
|
||||||
|
|
||||||
|
Built-in generation preserves response-body read failures separately from
|
||||||
|
malformed response content, and failures during request preparation are
|
||||||
|
distinguished from dispatched attempts.
|
||||||
|
|
||||||
|
OpenRouter authentication diagnostics bound response recognition to 1 MiB.
|
||||||
|
An oversized HTTP 200 response reports
|
||||||
|
`SmokeTestReasonAuthenticationResponseLimitExceeded`, leaving credential
|
||||||
|
acceptance unknown. Cancellation during the read remains an interruption.
|
||||||
|
Consumers that enumerate diagnostic reasons should handle the new reason;
|
||||||
|
see the [diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||||
|
|
||||||
|
## Internal Simplification And Ownership
|
||||||
|
|
||||||
|
Prompt lookup and rendering now consistently use captured configuration, and
|
||||||
|
smoke syntax and schema checks share the prepared output-validation lifecycle.
|
||||||
|
Unused routing defaults, alternate private workflows, and duplicate wire
|
||||||
|
representations have been removed.
|
||||||
|
|
||||||
|
Provider-setting resolution, inspection, prepared details, and smoke reports
|
||||||
|
avoid redundant validation and copying of already accepted values. Prepared
|
||||||
|
execution shares one private redacted metadata snapshot while preserving
|
||||||
|
independent caller-visible values. Provider values are detached before custom
|
||||||
|
JSON encoders run, protecting retained metadata from encoder-side mutation.
|
||||||
|
|
||||||
|
Requirement analysis, namespace-filtered listings, and repair-feedback
|
||||||
|
construction also avoid unnecessary traversal and temporary allocation. These
|
||||||
|
changes preserve existing request isolation and execution behavior without
|
||||||
|
introducing new performance guarantees.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/docs/oss/attribution.md)
|
||||||
55
docs/releases/v0.20.0.md
Normal file
55
docs/releases/v0.20.0.md
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
# Promptkit v0.20.0
|
||||||
|
|
||||||
|
This minor release adds generically renderable resource inspection and typed
|
||||||
|
prompt source construction. The Go consumer minimum remains 1.27.0 and
|
||||||
|
ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Migration
|
||||||
|
|
||||||
|
`InspectProfile` now returns the resolved profile resource instead of the
|
||||||
|
backend-completed `EffectiveModelParams`. Replace accesses to that field with
|
||||||
|
the optional profile fields; use `Prepare` or `PreparedExecution.Details` for
|
||||||
|
completed request settings. `InspectPrompt` now includes captured messages and
|
||||||
|
uses optional pointers for presence-sensitive fields, including default profile,
|
||||||
|
inputs, input metadata, and output settings. Check pointers before dereferencing.
|
||||||
|
All three resource reports support ordinary JSON and YAML rendering without
|
||||||
|
per-field adapters. See the [consumer guidance](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
||||||
|
|
||||||
|
After configuration composition and inheritance, profile null endpoint,
|
||||||
|
API-key source, and whole provider map now permit backend defaults during
|
||||||
|
execution as well as appearing absent in resource inspection. Use explicit
|
||||||
|
empty strings to clear endpoint or API-key source. There is no replacement for
|
||||||
|
profile-level suppression of an entire backend provider map; edit backend
|
||||||
|
defaults through composition instead. The required-key rule still applies.
|
||||||
|
See [defaults and overrides](../formats.md#defaults-and-overrides) for the
|
||||||
|
migration example and retained ConfigKit null provenance.
|
||||||
|
|
||||||
|
## Resource Inspection And Sources
|
||||||
|
|
||||||
|
`InspectBackend` reports the resolved backend and configured diagnostics without
|
||||||
|
contacting a provider or deriving diagnostic destinations. Prompt and profile
|
||||||
|
inspection likewise report selected resources independently of request readiness;
|
||||||
|
configuration provenance, requirement analysis, and preparation remain separate.
|
||||||
|
`PromptSource` adds typed programmatic prompt definitions, and `BackendConfig`
|
||||||
|
supports sibling inheritance. ConfigKit continues to own source composition,
|
||||||
|
snapshotting, and relative message-file contexts.
|
||||||
|
|
||||||
|
The source helpers' [GoDoc](../../prompts.go) and
|
||||||
|
[backend configuration API](../../backends.go) describe programmatic construction.
|
||||||
|
|
||||||
|
## Refinements
|
||||||
|
|
||||||
|
Package GoDoc and migration guidance now consistently describe the resource
|
||||||
|
inspection contract, and the full package overview is visible in generated
|
||||||
|
documentation. Prompt inspection avoids repeating the same metadata lookup for
|
||||||
|
each declared input.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
After adapting affected inspection consumers and reviewing profile null usage:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.20.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.20.0/docs/oss/attribution.md)
|
||||||
@@ -38,8 +38,17 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
No unselected ideas are currently recorded here. Deliberately postponed ideas
|
### Schema Bundling
|
||||||
remain in the [deferred catalog](deferred.md).
|
|
||||||
|
Allow consumers to author reusable schemas across multiple files and package
|
||||||
|
them into a single self-contained schema for local validation and provider
|
||||||
|
submission. This could restore multi-file authoring convenience while retaining
|
||||||
|
the [self-contained schema requirement](../formats.md#schemas).
|
||||||
|
|
||||||
|
Bundling would need to preserve reference semantics, identifiers, anchors, and
|
||||||
|
recursion. A complete bundle would not guarantee that a provider supports every
|
||||||
|
schema feature. This is a possible future enhancement; implementation is not
|
||||||
|
currently approved.
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
58
engine.go
58
engine.go
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
internalengine "go.maximumdirect.net/promptkit/internal/engine"
|
internalengine "go.maximumdirect.net/promptkit/internal/engine"
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
@@ -30,6 +31,8 @@ var (
|
|||||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||||
// every configured profile source. It does not also match ErrProfileLoad.
|
// every configured profile source. It does not also match ErrProfileLoad.
|
||||||
ErrProfileNotFound = errors.New("profile not found")
|
ErrProfileNotFound = errors.New("profile not found")
|
||||||
|
// ErrBackendNotFound identifies a backend absent from resource inspection.
|
||||||
|
ErrBackendNotFound = errors.New("backend not found")
|
||||||
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||||
// nor the selected prompt's default profile is present. Such an error also
|
// nor the selected prompt's default profile is present. Such an error also
|
||||||
// matches ErrInvalidRequest.
|
// matches ErrInvalidRequest.
|
||||||
@@ -81,12 +84,12 @@ var (
|
|||||||
ErrValidation = errors.New("failed to validate output")
|
ErrValidation = errors.New("failed to validate output")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Engine inspects prompts and profiles, enumerates prompt requirements, and
|
// Engine inspects resources, enumerates prompt requirements, and
|
||||||
// prepares and runs Promptkit prompt requests.
|
// prepares and runs Promptkit prompt requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
// An Engine is safe for concurrent calls to [Engine.WithNamespaces],
|
||||||
// [Engine.InspectConfiguration],
|
// [Engine.InspectConfiguration],
|
||||||
// [Engine.InspectPrompt],
|
// [Engine.InspectPrompt], [Engine.InspectBackend],
|
||||||
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
|
||||||
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
// [Engine.Run], [Engine.RunPrepared], [Engine.SmokeTest], [Engine.ListProfiles],
|
||||||
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
// [Engine.ListPrompts], [Engine.ListBackends], [Engine.ListNamespaces], and
|
||||||
@@ -213,7 +216,8 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
|||||||
//
|
//
|
||||||
// Prompt schema_path values refer to the file's base name. path must name an
|
// Prompt schema_path values refer to the file's base name. path must name an
|
||||||
// existing non-directory file when NewEngine applies the option. This option
|
// existing non-directory file when NewEngine applies the option. This option
|
||||||
// replaces Config.SchemaDir and earlier schema-source options.
|
// replaces Config.SchemaDir and earlier schema-source options. The selected
|
||||||
|
// schema must be self-contained; external resources are not loaded.
|
||||||
func WithSchemaFile(path string) Option {
|
func WithSchemaFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
return options.internal.SetSchemaFile(path)
|
return options.internal.SetSchemaFile(path)
|
||||||
@@ -266,15 +270,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
// qualified IDs pass through.
|
// qualified IDs pass through.
|
||||||
//
|
//
|
||||||
// A successful result proves that the selected definition and any referenced
|
// A successful result proves that the selected definition and any referenced
|
||||||
// message content files were structurally loaded. Inputs are returned in
|
// message content files were structurally loaded. Inputs and captured,
|
||||||
// definition order. DefaultProfileID is declared metadata only and is not
|
// unrendered messages are returned in definition order. DefaultProfileID is
|
||||||
|
// declared metadata only and is not
|
||||||
// resolved. OutputContract is the normalized declared contract, with a JSON
|
// resolved. OutputContract is the normalized declared contract, with a JSON
|
||||||
// Schema path when declared but without loading or compiling that schema.
|
// Schema path when declared but without loading or compiling that schema.
|
||||||
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
|
||||||
// the selected captured definition; its spelling, length,
|
// the selected captured definition; its spelling, length,
|
||||||
// encoding, algorithm, and security properties are not contracts.
|
// encoding, algorithm, and security properties are not contracts.
|
||||||
//
|
//
|
||||||
// This method does not return prompt bodies, templates, source paths, schemas,
|
// This method does not return parsed templates, source paths, schemas,
|
||||||
// rendered messages, or execution settings. It does not resolve a profile or
|
// rendered messages, or execution settings. It does not resolve a profile or
|
||||||
// credential, read artifacts or schemas, render, validate, admit capacity,
|
// credential, read artifacts or schemas, render, validate, admit capacity,
|
||||||
// contact a provider, or generate model output. The returned PromptInspection
|
// contact a provider, or generate model output. The returned PromptInspection
|
||||||
@@ -298,21 +303,19 @@ func (e *Engine) InspectPrompt(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainPromptInspection(inspection), nil
|
return fromPromptResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectProfile resolves one explicit profile without selecting a prompt or
|
// InspectProfile resolves one explicit profile without selecting a prompt or
|
||||||
// starting execution work.
|
// starting execution work.
|
||||||
//
|
//
|
||||||
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
|
||||||
// in the captured effective configuration. It applies the framework timeout
|
// in the captured effective configuration. It reports the resolved profile
|
||||||
// baseline, selected
|
// resource without backend completion, execution defaults, or request overrides.
|
||||||
// backend, and then selected profile to EffectiveModelParams without a request
|
|
||||||
// override. BackendID is empty for an endpoint-only profile.
|
|
||||||
//
|
//
|
||||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
// APIKeyEnv in the returned resource is an environment-variable name, never its
|
||||||
// value. APIKeyRequired instead reports a direct credential requirement and is
|
// value. APIKeyRequired, when present and true, reports that a credential is
|
||||||
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
// required; an explicit profile environment source can also be present. InspectProfile neither derives
|
||||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
// absent or blank named environment variable is not an error.
|
// absent or blank named environment variable is not an error.
|
||||||
//
|
//
|
||||||
@@ -336,7 +339,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainProfileInspection(inspection), nil
|
return fromProfileResource(inspection), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InspectBackend selects a resolved backend by exact ID from the captured
|
||||||
|
// configuration, qualifying a slashless ID with the backend namespace. It
|
||||||
|
// reports configured resource fields and diagnostics, not credential values,
|
||||||
|
// derived diagnostic destinations, or runtime capacity occupancy. References
|
||||||
|
// are not resolved and no network request is made. The result and its mutable
|
||||||
|
// values are caller-owned. Source changes require a new engine.
|
||||||
|
//
|
||||||
|
// An invalid engine matches ErrInvalidConfig, an invalid ID matches
|
||||||
|
// ErrInvalidRequest, and an absent backend matches ErrBackendNotFound (not
|
||||||
|
// ErrInvalidRequest). Cancellation remains detectable via errors.Is. No
|
||||||
|
// partial result is returned on error.
|
||||||
|
func (e *Engine) InspectBackend(ctx context.Context, backendID string) (*BackendInspection, error) {
|
||||||
|
if e == nil || e.internal == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
inspection, err := e.internal.InspectBackend(ctx, backendID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrBackendNotFound, err)
|
||||||
|
}
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromBackendResource(inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
|
|||||||
34
errors.go
34
errors.go
@@ -13,10 +13,44 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// publicOperationError keeps classification separate from inspectable causes.
|
||||||
|
// Cache wording so formatting nested operations does not repeatedly rebuild it.
|
||||||
|
type publicOperationError struct {
|
||||||
|
owner error
|
||||||
|
cause error
|
||||||
|
message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *publicOperationError) Error() string { return e.message }
|
||||||
|
func (e *publicOperationError) Unwrap() error { return e.cause }
|
||||||
|
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
|
||||||
|
|
||||||
func mapPublicError(err error) error {
|
func mapPublicError(err error) error {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var operation *usecase.OperationError
|
||||||
|
if errors.As(err, &operation) && operation != nil {
|
||||||
|
owner := publicErrorFor(operation.Category)
|
||||||
|
if owner != nil {
|
||||||
|
// Project the underlying specialized error separately, retaining both
|
||||||
|
// the outer operation and the original cause for Is/As callers.
|
||||||
|
mapped := mapPublicError(operation.Cause)
|
||||||
|
if errors.Is(operation.Cause, llm.ErrTransport) {
|
||||||
|
mapped = errors.Join(ErrTransport, mapped)
|
||||||
|
}
|
||||||
|
if errors.Is(operation.Cause, llm.ErrMalformedResponse) {
|
||||||
|
mapped = errors.Join(ErrMalformedResponse, mapped)
|
||||||
|
}
|
||||||
|
// The projection already retains ordinary causes; adding the original
|
||||||
|
// again would duplicate its entire subtree at each enclosing operation.
|
||||||
|
return &publicOperationError{owner: owner, cause: mapped, message: fmt.Sprintf("%v: %v", owner, mapped)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mapUnownedPublicError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapUnownedPublicError(err error) error {
|
||||||
var internalCapacityError *usecase.CapacityError
|
var internalCapacityError *usecase.CapacityError
|
||||||
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
|
|||||||
@@ -80,3 +80,12 @@ func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
|||||||
t.Fatalf("request encoding category = %v", err)
|
t.Fatalf("request encoding category = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorRetainsOwnerWithSpecializedCapacity(t *testing.T) {
|
||||||
|
internal := &usecase.CapacityError{BackendID: "limited"}
|
||||||
|
err := mapPublicError(&usecase.OperationError{Category: usecase.ErrLLMGenerate, Cause: internal})
|
||||||
|
var capacity *CapacityError
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) || !errors.As(err, &capacity) || capacity.BackendID != "limited" || !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("mapped owned capacity = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -110,10 +110,10 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if profile.EffectiveModelParams.TimeoutSeconds != 30 {
|
if profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 30 {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"later profile override was not applied: timeout is %d seconds",
|
"later profile override was not applied: timeout is %v",
|
||||||
profile.EffectiveModelParams.TimeoutSeconds,
|
profile.TimeoutSeconds,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
namespaces, err := scoped.ListNamespaces(ctx)
|
namespaces, err := scoped.ListNamespaces(ctx)
|
||||||
@@ -156,7 +156,7 @@ func run(ctx context.Context, exampleDir string, out io.Writer) error {
|
|||||||
SelectedProfile: prepared.SelectedProfileID,
|
SelectedProfile: prepared.SelectedProfileID,
|
||||||
Model: prepared.EffectiveModelParams.Model,
|
Model: prepared.EffectiveModelParams.Model,
|
||||||
MessageCount: len(prepared.Messages),
|
MessageCount: len(prepared.Messages),
|
||||||
ProfileTimeout: profile.EffectiveModelParams.TimeoutSeconds,
|
ProfileTimeout: *profile.TimeoutSeconds,
|
||||||
Namespaces: namespaces,
|
Namespaces: namespaces,
|
||||||
Collections: collectionSummaries,
|
Collections: collectionSummaries,
|
||||||
ScopedPrompts: scopedPromptIDs,
|
ScopedPrompts: scopedPromptIDs,
|
||||||
|
|||||||
2
go.mod
2
go.mod
@@ -5,12 +5,12 @@ go 1.27.0
|
|||||||
require (
|
require (
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
go.maximumdirect.net/configkit v1.1.0
|
go.maximumdirect.net/configkit v1.1.0
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.maximumdirect.net/releasekit v0.9.0 // indirect
|
go.maximumdirect.net/releasekit v0.9.0 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
|
||||||
golang.org/x/mod v0.40.0 // indirect
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ package artifact
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"fmt"
|
|
||||||
"mime"
|
"mime"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -153,9 +151,12 @@ func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef,
|
|||||||
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||||
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||||
}
|
}
|
||||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(want)})
|
||||||
if artifact.Hash != wantHash {
|
if err != nil {
|
||||||
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
b.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,9 +3,7 @@ package artifact
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -284,9 +282,12 @@ func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
|||||||
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||||
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||||
}
|
}
|
||||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(tc.body)})
|
||||||
if artifact.Hash != wantHash {
|
if err != nil {
|
||||||
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
t.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,17 +27,13 @@ type Registry struct {
|
|||||||
// NewEffectiveRegistry stores already-validated effective backend definitions
|
// NewEffectiveRegistry stores already-validated effective backend definitions
|
||||||
// under their exact configuration keys. Configuration validation and
|
// under their exact configuration keys. Configuration validation and
|
||||||
// normalization precede this ownership boundary.
|
// normalization precede this ownership boundary.
|
||||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
func NewEffectiveRegistry(definitions map[string]domain.Backend) *Registry {
|
||||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||||
for id, definition := range definitions {
|
for id, definition := range definitions {
|
||||||
definition.ID = id
|
definition.ID = id
|
||||||
cloned, err := domain.CloneBackend(definition)
|
registry.backends[id] = domain.CloneBackend(definition)
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("backend %q: %w", id, err)
|
|
||||||
}
|
|
||||||
registry.backends[id] = cloned
|
|
||||||
}
|
}
|
||||||
return registry, nil
|
return registry
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBackend returns a defensive copy of the backend registered with id.
|
// GetBackend returns a defensive copy of the backend registered with id.
|
||||||
@@ -49,11 +45,7 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
}
|
}
|
||||||
cloned, err := domain.CloneBackend(definition)
|
return domain.CloneBackend(definition), nil
|
||||||
if err != nil {
|
|
||||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
|
||||||
}
|
|
||||||
return cloned, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||||
|
|||||||
@@ -22,16 +22,13 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
|||||||
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
registry := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||||
" local.v1 ": {
|
" local.v1 ": {
|
||||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
ConcurrencyLimit: 2, QueueCapacity: 3,
|
||||||
SmokeTests: smokeTests,
|
SmokeTests: smokeTests,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
params["nested"].(map[string]any)["enabled"] = false
|
params["nested"].(map[string]any)["enabled"] = false
|
||||||
defaultProfile = "mutated-profile"
|
defaultProfile = "mutated-profile"
|
||||||
reachabilityEndpoint = "https://mutated.example/ready"
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
@@ -79,10 +76,7 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
||||||
registry, err := backend.NewEffectiveRegistry(nil)
|
registry := backend.NewEffectiveRegistry(nil)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
t.Fatalf("missing backend error = %v", err)
|
t.Fatalf("missing backend error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
package capacity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
|
||||||
)
|
|
||||||
|
|
||||||
type client struct {
|
|
||||||
manager *Manager
|
|
||||||
next llm.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient wraps next with configured active-generation limits. A nil manager
|
|
||||||
// leaves next unchanged.
|
|
||||||
func NewClient(manager *Manager, next llm.Client) llm.Client {
|
|
||||||
if manager == nil {
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
return &client{
|
|
||||||
manager: manager,
|
|
||||||
next: next,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *client) Generate(
|
|
||||||
ctx context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
if c.manager.getPool(req.Target.BackendID) == nil {
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,6 @@ package capacity
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -24,6 +23,18 @@ type clientFunc func(
|
|||||||
domain.GenerateRequest,
|
domain.GenerateRequest,
|
||||||
) (*domain.GenerateResponse, error)
|
) (*domain.GenerateResponse, error)
|
||||||
|
|
||||||
|
// newTestClient exercises one acquisition around a controlled call.
|
||||||
|
func newTestClient(manager *Manager, next llm.Client) llm.Client {
|
||||||
|
return clientFunc(func(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
release, err := manager.Acquire(ctx, req.Target.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
return next.Generate(ctx, req)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (f clientFunc) Generate(
|
func (f clientFunc) Generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -167,7 +178,7 @@ func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
|
|||||||
"second": secondRelease,
|
"second": secondRelease,
|
||||||
"third": thirdRelease,
|
"third": thirdRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
first := generateAsync(client, context.Background(), "limited", "first")
|
first := generateAsync(client, context.Background(), "limited", "first")
|
||||||
if got := receiveStarted(t, next.started); got != "first" {
|
if got := receiveStarted(t, next.started); got != "first" {
|
||||||
@@ -212,7 +223,7 @@ func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
|
|||||||
releases[string(rune('a'+i))] = gate
|
releases[string(rune('a'+i))] = gate
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
results := make([]<-chan generateResult, 0, len(releases))
|
results := make([]<-chan generateResult, 0, len(releases))
|
||||||
for id := range releases {
|
for id := range releases {
|
||||||
@@ -257,7 +268,7 @@ func TestClientRemovesCanceledWaiters(t *testing.T) {
|
|||||||
"three": make(chan struct{}),
|
"three": make(chan struct{}),
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
if got := receiveStarted(t, next.started); got != "holder" {
|
if got := receiveStarted(t, next.started); got != "holder" {
|
||||||
@@ -321,7 +332,7 @@ func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
waitForActiveCount(t, manager, "limited", 1)
|
waitForActiveCount(t, manager, "limited", 1)
|
||||||
@@ -394,7 +405,7 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
"alpha": alphaRelease,
|
"alpha": alphaRelease,
|
||||||
"beta": betaRelease,
|
"beta": betaRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
||||||
beta := generateAsync(client, context.Background(), "beta", "beta")
|
beta := generateAsync(client, context.Background(), "beta", "beta")
|
||||||
@@ -423,63 +434,6 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if got := NewClient(nil, next); got != next {
|
|
||||||
t.Fatal("nil manager did not return the wrapped client unchanged")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
|
|
||||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
|
||||||
"limited": {ConcurrencyLimit: 1},
|
|
||||||
})
|
|
||||||
request := domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{
|
|
||||||
SessionID: "session",
|
|
||||||
Messages: []domain.RenderedMessage{
|
|
||||||
{Role: "user", Content: "content"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Target: domain.ExecutionTarget{
|
|
||||||
BackendID: "limited",
|
|
||||||
Model: "model",
|
|
||||||
ExtraParams: map[string]any{"key": "value"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
response := &domain.GenerateResponse{
|
|
||||||
Content: "output",
|
|
||||||
Usage: domain.TokenUsage{TotalTokens: 7},
|
|
||||||
}
|
|
||||||
collaboratorErr := errors.New("collaborator failure")
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
response *domain.GenerateResponse
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
{name: "successful response", response: response},
|
|
||||||
{name: "nil response"},
|
|
||||||
{name: "collaborator error", response: response, err: collaboratorErr},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
var captured domain.GenerateRequest
|
|
||||||
next := clientFunc(func(
|
|
||||||
_ context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
captured = req
|
|
||||||
return tc.response, tc.err
|
|
||||||
})
|
|
||||||
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
|
|
||||||
if !reflect.DeepEqual(captured, request) {
|
|
||||||
t.Fatalf("request changed: %#v", captured)
|
|
||||||
}
|
|
||||||
if gotResponse != tc.response || gotErr != tc.err {
|
|
||||||
t.Fatalf("response=(%p, %v), want (%p, %v)",
|
|
||||||
gotResponse, gotErr, tc.response, tc.err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||||
@@ -496,7 +450,7 @@ func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: "recovered"}, nil
|
return &domain.GenerateResponse{Content: "recovered"}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
request := domain.GenerateRequest{
|
request := domain.GenerateRequest{
|
||||||
Target: domain.ExecutionTarget{BackendID: "limited"},
|
Target: domain.ExecutionTarget{BackendID: "limited"},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ package configuration
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBundledOpenRouterUnderlay(t *testing.T) {
|
func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||||
@@ -15,9 +17,12 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backend, err := state.GetBackend("openrouter")
|
backend, err := state.GetBackend("openrouter")
|
||||||
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||||
t.Fatalf("backend = %+v, %v", backend, err)
|
t.Fatalf("backend = %+v, %v", backend, err)
|
||||||
}
|
}
|
||||||
|
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
|
||||||
|
t.Fatalf("bundle inventories: backends=%v prompts=%v collections=%v", state.backendIDs, state.promptIDs, state.collections)
|
||||||
|
}
|
||||||
models := map[string]string{
|
models := map[string]string{
|
||||||
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
|
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
|
||||||
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
|
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
|
||||||
@@ -31,14 +36,16 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for id, model := range models {
|
for id, model := range models {
|
||||||
profile, err := state.GetProfile(context.Background(), id)
|
profile, err := state.GetProfile(context.Background(), id)
|
||||||
if err != nil || profile.ResolvedTarget.Model != model || profile.ResolvedTarget.BackendID != "openrouter" {
|
if err != nil {
|
||||||
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
||||||
}
|
}
|
||||||
|
want := defaults.ExecutionTargetDefault()
|
||||||
|
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
|
||||||
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
||||||
plugins := fmt.Sprint(profile.ResolvedTarget.ExtraParams)
|
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
|
||||||
if !strings.Contains(plugins, "cost_tier:"+tier) || !strings.Contains(plugins, "id:auto-router") {
|
}
|
||||||
t.Fatalf("profile %s plugins = %+v", id, profile.ResolvedTarget.ExtraParams)
|
if !reflect.DeepEqual(*profile.ResolvedTarget, want) {
|
||||||
}
|
t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") {
|
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
|
|||||||
@@ -2,36 +2,66 @@ package configuration
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// IdentityFilter selects all identities, one namespace, or unqualified IDs.
|
||||||
|
type IdentityFilter struct {
|
||||||
|
All bool
|
||||||
|
Namespace string
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectIdentities[T any](ctx context.Context, entries []T, filter IdentityFilter, id func(T) string) ([]T, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if filter.All {
|
||||||
|
result := append([]T(nil), entries...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if filter.Namespace != "" {
|
||||||
|
prefix := filter.Namespace + "/"
|
||||||
|
start := sort.Search(len(entries), func(i int) bool { return id(entries[i]) >= prefix })
|
||||||
|
end := start + sort.Search(len(entries)-start, func(i int) bool { return !strings.HasPrefix(id(entries[start+i]), prefix) })
|
||||||
|
result := append([]T(nil), entries[start:end]...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
var result []T
|
||||||
|
for _, entry := range entries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !strings.Contains(id(entry), "/") {
|
||||||
|
result = append(result, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
// CollectionIdentity is the captured metadata for one final collection entry.
|
// CollectionIdentity is the captured metadata for one final collection entry.
|
||||||
type CollectionIdentity struct {
|
type CollectionIdentity struct {
|
||||||
Namespace string
|
Namespace string
|
||||||
ReleaseVersion *string
|
ReleaseVersion *string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
func (state *loaded) ListProfiles(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.profileIDs, filter, func(id string) string { return id })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
profiles := append([]string(nil), state.profileIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return profiles, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
|
func (state *loaded) ListBackends(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.backendIDs, filter, func(id string) string { return id })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
backends := append([]string(nil), state.backendIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return backends, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
@@ -72,13 +102,6 @@ func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity,
|
|||||||
return collections, nil
|
return collections, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
func (state *loaded) ListPrompts(ctx context.Context, filter IdentityFilter) ([]domain.PromptIdentity, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.promptIDs, filter, func(entry domain.PromptIdentity) string { return entry.ID })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return prompts, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -32,7 +32,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -41,7 +41,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||||
}
|
}
|
||||||
|
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -65,36 +65,36 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles = append(profiles, "caller profile")
|
profiles = append(profiles, "caller profile")
|
||||||
profiles[0] = "changed profile"
|
profiles[0] = "changed profile"
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backends[0] = "changed backend"
|
backends[0] = "changed backend"
|
||||||
|
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
prompts[0].ID = "changed prompt"
|
prompts[0].ID = "changed prompt"
|
||||||
|
|
||||||
profiles, err = state.ListProfiles(context.Background())
|
profiles, err = state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||||
}
|
}
|
||||||
backends, err = state.ListBackends(context.Background())
|
backends, err = state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
||||||
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
||||||
}
|
}
|
||||||
prompts, err = state.ListPrompts(context.Background())
|
prompts, err = state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -104,20 +104,66 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadedListsSelectNamespaces(t *testing.T) {
|
||||||
|
ids := []string{"a", "a-b/one", "a/one", "a/two", "b", "z/one"}
|
||||||
|
state := &loaded{profileIDs: append([]string(nil), ids...), backendIDs: append([]string(nil), ids...)}
|
||||||
|
for _, id := range ids {
|
||||||
|
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
|
||||||
|
}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
filter IdentityFilter
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"all", IdentityFilter{All: true}, ids},
|
||||||
|
{"unqualified", IdentityFilter{}, []string{"a", "b"}},
|
||||||
|
{"namespace", IdentityFilter{Namespace: "a"}, []string{"a/one", "a/two"}},
|
||||||
|
{"prefix sharing", IdentityFilter{Namespace: "a-b"}, []string{"a-b/one"}},
|
||||||
|
{"missing", IdentityFilter{Namespace: "absent"}, nil},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
profiles, err := state.ListProfiles(context.Background(), tc.filter)
|
||||||
|
if err != nil || !reflect.DeepEqual(profiles, tc.want) {
|
||||||
|
t.Fatalf("profiles = %v, %v; want %v", profiles, err, tc.want)
|
||||||
|
}
|
||||||
|
backends, err := state.ListBackends(context.Background(), tc.filter)
|
||||||
|
if err != nil || !reflect.DeepEqual(backends, tc.want) {
|
||||||
|
t.Fatalf("backends = %v, %v; want %v", backends, err, tc.want)
|
||||||
|
}
|
||||||
|
prompts, err := state.ListPrompts(context.Background(), tc.filter)
|
||||||
|
if err != nil || len(prompts) != len(tc.want) {
|
||||||
|
t.Fatalf("prompts = %v, %v; want %v", prompts, err, tc.want)
|
||||||
|
}
|
||||||
|
for i := range prompts {
|
||||||
|
if prompts[i].ID != tc.want[i] {
|
||||||
|
t.Fatalf("prompt ID = %q; want %q", prompts[i].ID, tc.want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(profiles) > 0 {
|
||||||
|
profiles[0], backends[0], prompts[0].ID = "changed", "changed", "changed"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(state.profileIDs, ids) || !reflect.DeepEqual(state.backendIDs, ids) || state.promptIDs[0].ID != ids[0] {
|
||||||
|
t.Fatal("caller mutation changed captured identities")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadedListsEmptyInventory(t *testing.T) {
|
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||||
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(profiles) != 0 {
|
if err != nil || len(profiles) != 0 {
|
||||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||||
}
|
}
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(backends) != 0 {
|
if err != nil || len(backends) != 0 {
|
||||||
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
||||||
}
|
}
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(prompts) != 0 {
|
if err != nil || len(prompts) != 0 {
|
||||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||||
}
|
}
|
||||||
@@ -134,17 +180,19 @@ func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(ctx)
|
for _, filter := range []IdentityFilter{{All: true}, {}, {Namespace: "example"}} {
|
||||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
profiles, err := state.ListProfiles(ctx, filter)
|
||||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||||
}
|
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||||
backends, err := state.ListBackends(ctx)
|
}
|
||||||
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
backends, err := state.ListBackends(ctx, filter)
|
||||||
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
||||||
}
|
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
||||||
prompts, err := state.ListPrompts(ctx)
|
}
|
||||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
prompts, err := state.ListPrompts(ctx, filter)
|
||||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||||
|
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -119,11 +119,12 @@ const (
|
|||||||
|
|
||||||
type loaded struct {
|
type loaded struct {
|
||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
backends map[string]map[string]fieldState
|
|
||||||
profiles map[string]map[string]fieldState
|
|
||||||
promptMetadata map[string][]promptInputPresence
|
promptMetadata map[string][]promptInputPresence
|
||||||
|
promptRepair map[string]bool
|
||||||
prompts map[string]*promptEntry
|
prompts map[string]*promptEntry
|
||||||
registry *backend.Registry
|
registry *backend.Registry
|
||||||
|
backendRecords map[string]backendConfig
|
||||||
|
profileRecords map[string]profileConfig
|
||||||
indexedProfiles map[string]domain.ExecutionProfile
|
indexedProfiles map[string]domain.ExecutionProfile
|
||||||
backendIDs []string
|
backendIDs []string
|
||||||
profileIDs []string
|
profileIDs []string
|
||||||
@@ -212,9 +213,6 @@ func indexedProfiles(config *document) map[string]profileConfig {
|
|||||||
|
|
||||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
|
|
||||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
|
||||||
|
|
||||||
// Load captures one immutable effective configuration for internal consumers.
|
// Load captures one immutable effective configuration for internal consumers.
|
||||||
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
return load(main, underlays)
|
return load(main, underlays)
|
||||||
@@ -237,9 +235,7 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
Sensitive: secretProviderPath,
|
Sensitive: secretProviderPath,
|
||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
captureFields(resources.backends, func(v backendConfig) string { return v.ID }, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
|
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata, state.promptRepair)
|
||||||
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
|
||||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
|
||||||
}
|
}
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||||
@@ -257,9 +253,8 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
state := &loaded{
|
state := &loaded{
|
||||||
backends: make(map[string]map[string]fieldState),
|
|
||||||
profiles: make(map[string]map[string]fieldState),
|
|
||||||
promptMetadata: make(map[string][]promptInputPresence),
|
promptMetadata: make(map[string][]promptInputPresence),
|
||||||
|
promptRepair: make(map[string]bool),
|
||||||
}
|
}
|
||||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||||
Underlays: underlays,
|
Underlays: underlays,
|
||||||
@@ -324,11 +319,12 @@ func checkConfigurationVersion(version configkit.Version, validation *configkit.
|
|||||||
}
|
}
|
||||||
|
|
||||||
type promptInputPresence struct {
|
type promptInputPresence struct {
|
||||||
|
required fieldState
|
||||||
contentType fieldState
|
contentType fieldState
|
||||||
description fieldState
|
description fieldState
|
||||||
}
|
}
|
||||||
|
|
||||||
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
|
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence, repair map[string]bool) {
|
||||||
for promptIndex, prompt := range prompts {
|
for promptIndex, prompt := range prompts {
|
||||||
inputs := prompt.Inputs
|
inputs := prompt.Inputs
|
||||||
states := make([]promptInputPresence, len(inputs))
|
states := make([]promptInputPresence, len(inputs))
|
||||||
@@ -336,11 +332,13 @@ func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, name
|
|||||||
for index := range inputs {
|
for index := range inputs {
|
||||||
path := base.Index(index)
|
path := base.Index(index)
|
||||||
states[index] = promptInputPresence{
|
states[index] = promptInputPresence{
|
||||||
|
required: captureFieldState(path.Key("required"), resolution),
|
||||||
contentType: captureFieldState(path.Key("content_type"), resolution),
|
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||||
description: captureFieldState(path.Key("description"), resolution),
|
description: captureFieldState(path.Key("description"), resolution),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
destination[indexedResourceID(namespace, prompt.ID)] = states
|
destination[indexedResourceID(namespace, prompt.ID)] = states
|
||||||
|
repair[indexedResourceID(namespace, prompt.ID)] = captureFieldState(basePath.Index(promptIndex).Key("output").Key("repair_attempts"), resolution) == fieldValue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -356,24 +354,6 @@ func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func captureFields[T any](entries []T, identity func(T) string, basePath configkit.Path, namespace string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
|
|
||||||
for index, entry := range entries {
|
|
||||||
states := make(map[string]fieldState, len(fields))
|
|
||||||
for _, field := range fields {
|
|
||||||
info, exists := resolution.EntryInfo(basePath.Index(index).Key(field))
|
|
||||||
switch {
|
|
||||||
case !exists || !info.Explicit:
|
|
||||||
states[field] = fieldAbsent
|
|
||||||
case info.Null:
|
|
||||||
states[field] = fieldNull
|
|
||||||
default:
|
|
||||||
states[field] = fieldValue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
destination[indexedResourceID(namespace, identity(entry))] = states
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// eligibleResourceList leaves invalid identities to the existing validation
|
// eligibleResourceList leaves invalid identities to the existing validation
|
||||||
// boundaries, without mutating a list that cannot be recorded by identity.
|
// boundaries, without mutating a list that cannot be recorded by identity.
|
||||||
func eligibleResourceList[T any](entries []T, identity func(T) string) bool {
|
func eligibleResourceList[T any](entries []T, identity func(T) string) bool {
|
||||||
@@ -419,101 +399,110 @@ func validateEffective(config *document, validation *configkit.Validation) {
|
|||||||
}
|
}
|
||||||
for _, resources := range sets {
|
for _, resources := range sets {
|
||||||
for index, backend := range resources.backends {
|
for index, backend := range resources.backends {
|
||||||
base := resources.bases.backends.Index(index)
|
validateBackendRecord(backend, resources.bases.backends.Index(index), validation)
|
||||||
validateBackendEndpoints(backend, base, validation)
|
|
||||||
if !domain.IsValidLocalResourceKey(backend.ID) {
|
|
||||||
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
|
||||||
}
|
|
||||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
|
||||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
|
||||||
}
|
|
||||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
|
||||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
|
||||||
}
|
|
||||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
|
||||||
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
|
||||||
}
|
|
||||||
if backend.QueueCapacity != nil {
|
|
||||||
if *backend.QueueCapacity < 0 {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
|
||||||
}
|
|
||||||
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
|
||||||
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
|
||||||
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
|
||||||
smokePath := base.Key("smoke_tests")
|
|
||||||
if smokeTests.DefaultProfile != nil {
|
|
||||||
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
|
||||||
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
|
||||||
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
|
||||||
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if authentication := smokeTests.Authentication; authentication != nil {
|
|
||||||
kindPath := smokePath.Key("authentication").Key("kind")
|
|
||||||
if authentication.Kind == nil {
|
|
||||||
validation.Error(kindPath, "authentication kind is required")
|
|
||||||
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
|
||||||
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for index, profile := range resources.profiles {
|
for index, profile := range resources.profiles {
|
||||||
base := resources.bases.profiles.Index(index)
|
validateProfileRecord(profile, resources.bases.profiles.Index(index), backendIDs, validation)
|
||||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
|
||||||
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
|
|
||||||
validation.Error(base.Key("endpoint"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !domain.IsValidLocalResourceKey(profile.ID) {
|
|
||||||
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
|
||||||
}
|
|
||||||
if profile.Backend != nil && *profile.Backend != "" {
|
|
||||||
if !domain.IsValidResourceID(*profile.Backend) {
|
|
||||||
validation.Error(base.Key("backend"), "backend reference is malformed")
|
|
||||||
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
|
||||||
validation.Error(base.Key("backend"), "backend is not defined")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
|
||||||
validation.Error(base.Key("model"), "model must not be blank")
|
|
||||||
}
|
|
||||||
if profile.Temperature != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
|
||||||
validation.Error(base.Key("temperature"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.TopP != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
|
||||||
validation.Error(base.Key("top_p"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.MaxTokens != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
|
||||||
validation.Error(base.Key("max_tokens"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.TimeoutSeconds != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
|
||||||
validation.Error(base.Key("timeout_seconds"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
|
||||||
}
|
}
|
||||||
for index, prompt := range resources.prompts {
|
for index, prompt := range resources.prompts {
|
||||||
base := resources.bases.prompts.Index(index)
|
validatePromptRecord(prompt, resources.bases.prompts.Index(index), validation)
|
||||||
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
}
|
||||||
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
}
|
||||||
}
|
}
|
||||||
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
|
||||||
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
func validateBackendRecord(backend backendConfig, base configkit.Path, validation *configkit.Validation) {
|
||||||
|
validateBackendEndpoints(backend, base, validation)
|
||||||
|
if !domain.IsValidLocalResourceKey(backend.ID) {
|
||||||
|
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||||
|
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||||
|
}
|
||||||
|
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||||
|
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||||
|
}
|
||||||
|
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
||||||
|
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
||||||
|
}
|
||||||
|
if backend.QueueCapacity != nil {
|
||||||
|
if *backend.QueueCapacity < 0 {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
||||||
|
}
|
||||||
|
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
||||||
|
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||||
|
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||||
|
smokePath := base.Key("smoke_tests")
|
||||||
|
if smokeTests.DefaultProfile != nil {
|
||||||
|
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
||||||
|
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if authentication := smokeTests.Authentication; authentication != nil {
|
||||||
|
kindPath := smokePath.Key("authentication").Key("kind")
|
||||||
|
if authentication.Kind == nil {
|
||||||
|
validation.Error(kindPath, "authentication kind is required")
|
||||||
|
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||||
|
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateProfileRecord(profile profileConfig, base configkit.Path, backendIDs map[string]struct{}, validation *configkit.Validation) {
|
||||||
|
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||||
|
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
|
||||||
|
validation.Error(base.Key("endpoint"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !domain.IsValidLocalResourceKey(profile.ID) {
|
||||||
|
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if profile.Backend != nil && *profile.Backend != "" {
|
||||||
|
if !domain.IsValidResourceID(*profile.Backend) {
|
||||||
|
validation.Error(base.Key("backend"), "backend reference is malformed")
|
||||||
|
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
||||||
|
validation.Error(base.Key("backend"), "backend is not defined")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
||||||
|
validation.Error(base.Key("model"), "model must not be blank")
|
||||||
|
}
|
||||||
|
if profile.Temperature != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
||||||
|
validation.Error(base.Key("temperature"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.TopP != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
||||||
|
validation.Error(base.Key("top_p"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.MaxTokens != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
||||||
|
validation.Error(base.Key("max_tokens"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.TimeoutSeconds != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
||||||
|
validation.Error(base.Key("timeout_seconds"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePromptRecord(prompt promptConfig, base configkit.Path, validation *configkit.Validation) {
|
||||||
|
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
||||||
|
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
||||||
|
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -488,15 +488,12 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
|||||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||||
t.Fatalf("effective values were not retained: %+v", profile)
|
t.Fatalf("effective values were not retained: %+v", profile)
|
||||||
}
|
}
|
||||||
if loaded.profiles["base/child.v1"]["temperature"] != fieldValue || loaded.profiles["base/child.v1"]["top_p"] != fieldNull || loaded.profiles["base/child.v1"]["api_key_required"] != fieldValue {
|
|
||||||
t.Fatalf("effective presence was lost: %+v", loaded.profiles["base/child.v1"])
|
|
||||||
}
|
|
||||||
provider := profile.ExtraParams["provider"].(map[string]any)
|
provider := profile.ExtraParams["provider"].(map[string]any)
|
||||||
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
if provider["first"] != int64(1) || provider["second"] != int64(3) {
|
||||||
t.Fatalf("recursive overlay failed: %+v", provider)
|
t.Fatalf("recursive overlay failed: %+v", provider)
|
||||||
}
|
}
|
||||||
backend := loaded.result.Config.Backends[0]
|
backend := loaded.result.Config.Backends[0]
|
||||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 {
|
||||||
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
||||||
}
|
}
|
||||||
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
||||||
@@ -582,9 +579,6 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
|
|||||||
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
|
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
|
||||||
t.Fatalf("resolved backend was not normalized: %+v", composed)
|
t.Fatalf("resolved backend was not normalized: %+v", composed)
|
||||||
}
|
}
|
||||||
if loaded.backends["local"]["queue_capacity"] != fieldNull {
|
|
||||||
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
|
|
||||||
}
|
|
||||||
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
|
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
|
||||||
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
t.Fatalf("endpoint origin was not preserved: %+v", field)
|
t.Fatalf("endpoint origin was not preserved: %+v", field)
|
||||||
@@ -763,7 +757,7 @@ func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backend, err := loaded.GetBackend("local")
|
backend, err := loaded.GetBackend("local")
|
||||||
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 || loaded.backends["local"]["queue_capacity"] != fieldAbsent {
|
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 {
|
||||||
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -779,9 +773,8 @@ func TestLoadCapturesProfilePresence(t *testing.T) {
|
|||||||
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
if profile.Temperature == nil || *profile.Temperature != 0 || profile.TopP != nil || profile.APIKeyRequired == nil || *profile.APIKeyRequired {
|
||||||
t.Fatalf("profile values were not preserved: %+v", profile)
|
t.Fatalf("profile values were not preserved: %+v", profile)
|
||||||
}
|
}
|
||||||
fields := loaded.profiles["local"]
|
if profile.Model != nil {
|
||||||
if fields["temperature"] != fieldValue || fields["top_p"] != fieldNull || fields["api_key_required"] != fieldValue || fields["model"] != fieldAbsent {
|
t.Fatalf("omitted model was not preserved: %+v", profile)
|
||||||
t.Fatalf("profile presence was not preserved: %+v", fields)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type promptEntry struct {
|
type promptEntry struct {
|
||||||
|
resource promptConfig
|
||||||
definition domain.PromptDefinition
|
definition domain.PromptDefinition
|
||||||
inspection domain.PromptInspection
|
inspection domain.PromptInspection
|
||||||
requirementInputs []prompt.InputMetadata
|
requirementInputs []prompt.InputMetadata
|
||||||
@@ -72,104 +73,21 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
definition.SessionID = *raw.SessionID
|
definition.SessionID = *raw.SessionID
|
||||||
}
|
}
|
||||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
inputs, inputMetadata, err := state.capturePromptInputs(id, raw.Inputs, base)
|
||||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
if err != nil {
|
||||||
entry := &promptEntry{
|
return nil, err
|
||||||
messages: make([]*template.Template, 0, len(raw.Messages)),
|
|
||||||
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
|
|
||||||
}
|
}
|
||||||
for index, input := range raw.Inputs {
|
messages, compiledMessages, err := state.capturePromptMessages(raw.Messages, base, lookupOrigin)
|
||||||
path := base.Key("inputs").Index(index)
|
if err != nil {
|
||||||
if input.Name == "" {
|
return nil, err
|
||||||
return nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
|
||||||
}
|
|
||||||
if seenInputs[input.Name] {
|
|
||||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
|
||||||
}
|
|
||||||
seenInputs[input.Name] = true
|
|
||||||
contentType := ""
|
|
||||||
if input.ContentType != nil {
|
|
||||||
contentType = *input.ContentType
|
|
||||||
}
|
|
||||||
description := ""
|
|
||||||
if input.Description != nil {
|
|
||||||
description = *input.Description
|
|
||||||
}
|
|
||||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
|
||||||
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
|
||||||
})
|
|
||||||
presence := state.promptMetadata[id][index]
|
|
||||||
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
|
||||||
if presence.contentType == fieldValue {
|
|
||||||
metadata.ContentType = stringPointer(contentType)
|
|
||||||
}
|
|
||||||
if presence.description == fieldValue {
|
|
||||||
metadata.Description = stringPointer(description)
|
|
||||||
}
|
|
||||||
entry.requirementInputs = append(entry.requirementInputs, metadata)
|
|
||||||
}
|
}
|
||||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
contract, err := capturePromptOutput(raw.Output, base)
|
||||||
for index, message := range raw.Messages {
|
if err != nil {
|
||||||
path := base.Key("messages").Index(index)
|
return nil, err
|
||||||
if message.Role == nil {
|
|
||||||
return nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
|
||||||
}
|
|
||||||
if _, err := domain.NormalizeMessageRole(*message.Role); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
|
||||||
}
|
|
||||||
if (message.Content != nil) == (message.ContentFile != nil) {
|
|
||||||
return nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
|
||||||
}
|
|
||||||
content := ""
|
|
||||||
if message.Content != nil {
|
|
||||||
content = *message.Content
|
|
||||||
} else {
|
|
||||||
var err error
|
|
||||||
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var control *domain.CacheControl
|
|
||||||
if message.CacheControl != nil {
|
|
||||||
if message.CacheControl.Type == nil {
|
|
||||||
return nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
|
||||||
}
|
|
||||||
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
|
||||||
if message.CacheControl.TTL != nil {
|
|
||||||
control.TTL = *message.CacheControl.TTL
|
|
||||||
}
|
|
||||||
if _, err := domain.NormalizeCacheControl(control); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
|
||||||
}
|
|
||||||
entry.messages = append(entry.messages, compiled)
|
|
||||||
definition.Templates = append(definition.Templates, domain.PromptMessageTemplate{
|
|
||||||
Role: *message.Role, Content: content, CacheControl: control,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
if raw.Output == nil {
|
definition.Inputs = inputs
|
||||||
return nil, fmt.Errorf("%s: output is required", base.Key("output"))
|
definition.Templates = messages
|
||||||
}
|
entry := &promptEntry{resource: raw, requirementInputs: inputMetadata, messages: compiledMessages}
|
||||||
if raw.Output.Format == nil || raw.Output.ValidationMode == nil {
|
|
||||||
return nil, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
|
||||||
}
|
|
||||||
contract := domain.OutputContract{
|
|
||||||
Format: domain.OutputFormat(*raw.Output.Format),
|
|
||||||
ValidationMode: domain.ValidationMode(*raw.Output.ValidationMode),
|
|
||||||
RepairAttempts: raw.Output.RepairAttempts,
|
|
||||||
}
|
|
||||||
if raw.Output.SchemaPath != nil {
|
|
||||||
contract.SchemaPath = *raw.Output.SchemaPath
|
|
||||||
}
|
|
||||||
if err := domain.ValidateOutputContract(contract); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
|
||||||
}
|
|
||||||
definition.OutputFormat = contract.Format
|
|
||||||
definition.Validation = contract
|
definition.Validation = contract
|
||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
var err error
|
var err error
|
||||||
@@ -192,6 +110,113 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
return entry, nil
|
return entry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (state *loaded) capturePromptInputs(id string, rawInputs []promptInputConfig, base configkit.Path) ([]domain.PromptInput, []prompt.InputMetadata, error) {
|
||||||
|
seenInputs := make(map[string]bool, len(rawInputs))
|
||||||
|
inputs := make([]domain.PromptInput, 0, len(rawInputs))
|
||||||
|
metadata := make([]prompt.InputMetadata, 0, len(rawInputs))
|
||||||
|
for index, input := range rawInputs {
|
||||||
|
path := base.Key("inputs").Index(index)
|
||||||
|
if input.Name == "" {
|
||||||
|
return nil, nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
||||||
|
}
|
||||||
|
if seenInputs[input.Name] {
|
||||||
|
return nil, nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||||
|
}
|
||||||
|
seenInputs[input.Name] = true
|
||||||
|
contentType := ""
|
||||||
|
if input.ContentType != nil {
|
||||||
|
contentType = *input.ContentType
|
||||||
|
}
|
||||||
|
description := ""
|
||||||
|
if input.Description != nil {
|
||||||
|
description = *input.Description
|
||||||
|
}
|
||||||
|
inputs = append(inputs, domain.PromptInput{
|
||||||
|
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||||
|
})
|
||||||
|
presence := state.promptMetadata[id][index]
|
||||||
|
item := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||||
|
if presence.contentType == fieldValue {
|
||||||
|
item.ContentType = stringPointer(contentType)
|
||||||
|
}
|
||||||
|
if presence.description == fieldValue {
|
||||||
|
item.Description = stringPointer(description)
|
||||||
|
}
|
||||||
|
metadata = append(metadata, item)
|
||||||
|
}
|
||||||
|
return inputs, metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) capturePromptMessages(rawMessages []promptMessageConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) ([]domain.PromptMessageTemplate, []*template.Template, error) {
|
||||||
|
templates := make([]domain.PromptMessageTemplate, 0, len(rawMessages))
|
||||||
|
compiledMessages := make([]*template.Template, 0, len(rawMessages))
|
||||||
|
for index, message := range rawMessages {
|
||||||
|
path := base.Key("messages").Index(index)
|
||||||
|
if message.Role == nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
||||||
|
}
|
||||||
|
if _, err := domain.NormalizeMessageRole(*message.Role); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
||||||
|
}
|
||||||
|
if (message.Content != nil) == (message.ContentFile != nil) {
|
||||||
|
return nil, nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
||||||
|
}
|
||||||
|
content := ""
|
||||||
|
if message.Content != nil {
|
||||||
|
content = *message.Content
|
||||||
|
} else {
|
||||||
|
var err error
|
||||||
|
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var control *domain.CacheControl
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
if message.CacheControl.Type == nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
||||||
|
}
|
||||||
|
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
||||||
|
if message.CacheControl.TTL != nil {
|
||||||
|
control.TTL = *message.CacheControl.TTL
|
||||||
|
}
|
||||||
|
if _, err := domain.NormalizeCacheControl(control); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
||||||
|
}
|
||||||
|
compiledMessages = append(compiledMessages, compiled)
|
||||||
|
templates = append(templates, domain.PromptMessageTemplate{
|
||||||
|
Role: *message.Role, Content: content, CacheControl: control,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return templates, compiledMessages, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func capturePromptOutput(output *promptOutputConfig, base configkit.Path) (domain.OutputContract, error) {
|
||||||
|
if output == nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: output is required", base.Key("output"))
|
||||||
|
}
|
||||||
|
if output.Format == nil || output.ValidationMode == nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
||||||
|
}
|
||||||
|
contract := domain.OutputContract{
|
||||||
|
Format: domain.OutputFormat(*output.Format),
|
||||||
|
ValidationMode: domain.ValidationMode(*output.ValidationMode),
|
||||||
|
RepairAttempts: output.RepairAttempts,
|
||||||
|
}
|
||||||
|
if output.SchemaPath != nil {
|
||||||
|
contract.SchemaPath = *output.SchemaPath
|
||||||
|
}
|
||||||
|
if err := domain.ValidateOutputContract(contract); err != nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
||||||
|
}
|
||||||
|
return contract, nil
|
||||||
|
}
|
||||||
|
|
||||||
func stringPointer(value string) *string {
|
func stringPointer(value string) *string {
|
||||||
copy := value
|
copy := value
|
||||||
return ©
|
return ©
|
||||||
@@ -292,8 +317,8 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id string) (*domain.
|
|||||||
return ownPromptDefinition(entry.definition), entry.hash, nil
|
return ownPromptDefinition(entry.definition), entry.hash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptInspection returns an independently owned inspection of one
|
// GetPromptInspection returns lightweight metadata used for requirement
|
||||||
// captured prompt without copying its execution templates.
|
// selection, independently of the full resource inspection view.
|
||||||
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -305,12 +330,6 @@ func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domai
|
|||||||
return ownPromptInspection(entry.inspection), nil
|
return ownPromptInspection(entry.inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
|
||||||
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
|
|
||||||
definition, _, err := state.GetPromptSnapshot(ctx, id)
|
|
||||||
return definition, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
||||||
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
||||||
if state == nil {
|
if state == nil {
|
||||||
|
|||||||
@@ -84,6 +84,10 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
files["catalog/body.md"].Data = []byte("Changed")
|
files["catalog/body.md"].Data = []byte("Changed")
|
||||||
files["app/body.md"].Data = []byte("Changed replacement")
|
files["app/body.md"].Data = []byte("Changed replacement")
|
||||||
files["other/body.md"].Data = []byte("Changed other collection")
|
files["other/body.md"].Data = []byte("Changed other collection")
|
||||||
|
resource, err := state.InspectPrompt(context.Background(), "docs/child")
|
||||||
|
if err != nil || resource.Messages[0].Content != "Captured {{.name}}" || resource.Messages[0].CacheControl == nil || resource.Messages[0].CacheControl.TTL == nil || *resource.Messages[0].CacheControl.TTL != "1h" || resource.Inputs == nil || (*resource.Inputs)[0].Required != nil {
|
||||||
|
t.Fatalf("captured resource = %+v, %v", resource, err)
|
||||||
|
}
|
||||||
again, againInspection, err := state.prompt("docs/child")
|
again, againInspection, err := state.prompt("docs/child")
|
||||||
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
||||||
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
||||||
@@ -99,6 +103,25 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptResourcePresence(t *testing.T) {
|
||||||
|
state, err := load(mapping("prompt resources", map[string]any{"prompts": []any{
|
||||||
|
map[string]any{"id": "base", "default_profile": "missing", "inputs": []any{map[string]any{"name": "x", "required": false}, map[string]any{"name": "y"}},
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}, map[string]any{"role": "assistant", "content": "{{.missing}}", "cache_control": map[string]any{"type": "ephemeral"}}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}},
|
||||||
|
}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := state.InspectPrompt(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts == nil || *got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
|
||||||
|
t.Fatalf("prompt resource = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
||||||
files := fstest.MapFS{
|
files := fstest.MapFS{
|
||||||
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
||||||
|
|||||||
140
internal/configuration/resources.go
Normal file
140
internal/configuration/resources.go
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProfileResource is the resolved profile before execution completion.
|
||||||
|
// Its fields borrow immutable configuration captured by the adapter.
|
||||||
|
type ProfileResource struct {
|
||||||
|
ID string
|
||||||
|
Backend, Endpoint, Model *string
|
||||||
|
Temperature, TopP *float64
|
||||||
|
MaxTokens, TimeoutSeconds *int
|
||||||
|
ServiceTier, ReasoningEffort *string
|
||||||
|
APIKeyEnv *string
|
||||||
|
APIKeyRequired *bool
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackendResource is the resolved backend and its configured diagnostics.
|
||||||
|
// Its fields borrow immutable configuration captured by the adapter.
|
||||||
|
type BackendResource struct {
|
||||||
|
ID, Endpoint string
|
||||||
|
APIKeyEnv *string
|
||||||
|
ConcurrencyLimit, QueueCapacity *int
|
||||||
|
ExtraParams map[string]any
|
||||||
|
SmokeTests *domain.BackendSmokeTestConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptResource is the resolved prompt with already captured message text.
|
||||||
|
type PromptResource struct {
|
||||||
|
ID, Hash string
|
||||||
|
DefaultProfile, Description, SessionID *string
|
||||||
|
Inputs *[]PromptInputResource
|
||||||
|
Messages []PromptMessageResource
|
||||||
|
Output PromptOutputResource
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptInputResource struct {
|
||||||
|
Name string
|
||||||
|
Required *bool
|
||||||
|
ContentType, Description *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptMessageResource struct {
|
||||||
|
Role, Content string
|
||||||
|
CacheControl *PromptCacheControlResource
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptCacheControlResource struct {
|
||||||
|
Type domain.CacheControlType
|
||||||
|
TTL *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptOutputResource struct {
|
||||||
|
Format domain.OutputFormat
|
||||||
|
ValidationMode domain.ValidationMode
|
||||||
|
SchemaPath *string
|
||||||
|
RepairAttempts *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entry, err := state.lookupPromptEntry(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, promptLookupError(err)
|
||||||
|
}
|
||||||
|
raw := entry.resource
|
||||||
|
result := &PromptResource{ID: id, Hash: entry.hash, DefaultProfile: raw.DefaultProfile, Description: raw.Description, SessionID: raw.SessionID}
|
||||||
|
if raw.Inputs != nil {
|
||||||
|
inputs := make([]PromptInputResource, len(raw.Inputs))
|
||||||
|
presence := state.promptMetadata[id]
|
||||||
|
for i, input := range raw.Inputs {
|
||||||
|
inputs[i] = PromptInputResource{Name: input.Name, ContentType: input.ContentType, Description: input.Description}
|
||||||
|
if presence[i].required == fieldValue {
|
||||||
|
required := input.Required
|
||||||
|
inputs[i].Required = &required
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.Inputs = &inputs
|
||||||
|
}
|
||||||
|
result.Messages = make([]PromptMessageResource, len(entry.definition.Templates))
|
||||||
|
for i, message := range entry.definition.Templates {
|
||||||
|
result.Messages[i] = PromptMessageResource{Role: message.Role, Content: message.Content}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, SchemaPath: raw.Output.SchemaPath}
|
||||||
|
if state.promptRepair[id] {
|
||||||
|
value := raw.Output.RepairAttempts
|
||||||
|
result.Output.RepairAttempts = &value
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) InspectProfile(ctx context.Context, id string) (*ProfileResource, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if state == nil {
|
||||||
|
return nil, fmt.Errorf("%w: %q", profile.ErrProfileNotFound, id)
|
||||||
|
}
|
||||||
|
config, ok := state.profileRecords[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: %q", profile.ErrProfileNotFound, id)
|
||||||
|
}
|
||||||
|
return &ProfileResource{
|
||||||
|
ID: id, Backend: config.Backend, Endpoint: config.Endpoint, Model: config.Model,
|
||||||
|
Temperature: config.Temperature, TopP: config.TopP, MaxTokens: config.MaxTokens,
|
||||||
|
TimeoutSeconds: config.TimeoutSeconds, ServiceTier: config.ServiceTier,
|
||||||
|
ReasoningEffort: config.ReasoningEffort, APIKeyEnv: config.APIKeyEnv,
|
||||||
|
APIKeyRequired: config.APIKeyRequired, ExtraParams: config.ExtraParams,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendResource, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if state == nil {
|
||||||
|
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
config, ok := state.backendRecords[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
return &BackendResource{
|
||||||
|
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv,
|
||||||
|
ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity,
|
||||||
|
ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
67
internal/configuration/resources_test.go
Normal file
67
internal/configuration/resources_test.go
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/backend"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolvedBackendAndProfileResources(t *testing.T) {
|
||||||
|
state, err := load(mapping("resources", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "concurrency_limit": 2,
|
||||||
|
"extra_params": map[string]any{"backend": true}, "smoke_tests": map[string]any{"default_profile": "missing", "reachability": map[string]any{}}}},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "parent", "backend": "local", "temperature": 0.4},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "temperature": 0, "endpoint": "", "extra_params": map[string]any{}},
|
||||||
|
},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resource, err := state.InspectProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resource.ID != "child" || resource.Backend == nil || *resource.Backend != "local" || resource.Endpoint == nil || *resource.Endpoint != "" || resource.Temperature == nil || *resource.Temperature != 0 || resource.Model != nil || resource.ExtraParams == nil || len(resource.ExtraParams) != 0 {
|
||||||
|
t.Fatalf("profile resource = %+v", resource)
|
||||||
|
}
|
||||||
|
target, err := state.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil || target.ResolvedTarget.ExtraParams["backend"] != true {
|
||||||
|
t.Fatalf("execution target = %+v, %v", target, err)
|
||||||
|
}
|
||||||
|
resourceBackend, err := state.InspectBackend(context.Background(), "local")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resourceBackend.Endpoint != "https://example.test/v1" || resourceBackend.QueueCapacity == nil || *resourceBackend.QueueCapacity != 1024 || resourceBackend.SmokeTests == nil || resourceBackend.SmokeTests.DefaultProfile == nil || *resourceBackend.SmokeTests.DefaultProfile != "missing" || resourceBackend.SmokeTests.Reachability == nil || resourceBackend.SmokeTests.Reachability.Endpoint != nil {
|
||||||
|
t.Fatalf("backend resource = %+v", resourceBackend)
|
||||||
|
}
|
||||||
|
if _, err := state.InspectBackend(context.Background(), "missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
t.Fatalf("backend error = %v", err)
|
||||||
|
}
|
||||||
|
if _, err := state.InspectProfile(context.Background(), "missing"); !errors.Is(err, profile.ErrProfileNotFound) {
|
||||||
|
t.Fatalf("profile error = %v", err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := state.InspectBackend(ctx, "local"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("cancellation = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendResourcePreservesInheritedDiagnostics(t *testing.T) {
|
||||||
|
state, err := load(mapping("backend diagnostics", map[string]any{"backends": []any{
|
||||||
|
map[string]any{"id": "parent", "endpoint": "https://provider.test/v1", "smoke_tests": map[string]any{"reachability": map[string]any{"endpoint": "https://probe.test/ready"}, "authentication": map[string]any{"kind": "none"}}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}},
|
||||||
|
}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := state.InspectBackend(context.Background(), "child")
|
||||||
|
if err != nil || got.ID != "child" || got.ConcurrencyLimit != nil || got.QueueCapacity != nil || got.SmokeTests == nil || got.SmokeTests.Reachability == nil || got.SmokeTests.Reachability.Endpoint == nil || *got.SmokeTests.Reachability.Endpoint != "https://probe.test/ready" || got.SmokeTests.Authentication == nil || got.SmokeTests.Authentication.Kind == nil || *got.SmokeTests.Authentication.Kind != "none" || got.SmokeTests.Authentication.Endpoint != nil {
|
||||||
|
t.Fatalf("inherited diagnostics = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,112 +14,117 @@ import (
|
|||||||
func (state *loaded) buildExecutionProfiles() error {
|
func (state *loaded) buildExecutionProfiles() error {
|
||||||
backendConfigs := indexedBackends(&state.result.Config)
|
backendConfigs := indexedBackends(&state.result.Config)
|
||||||
profileConfigs := indexedProfiles(&state.result.Config)
|
profileConfigs := indexedProfiles(&state.result.Config)
|
||||||
|
state.backendRecords = backendConfigs
|
||||||
|
state.profileRecords = profileConfigs
|
||||||
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
||||||
for _, id := range sortedKeys(backendConfigs) {
|
for _, id := range sortedKeys(backendConfigs) {
|
||||||
config := backendConfigs[id]
|
config := backendConfigs[id]
|
||||||
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
definitions[id] = captureBackend(id, config)
|
||||||
if config.SmokeTests != nil {
|
|
||||||
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
|
||||||
if config.SmokeTests.Reachability != nil {
|
|
||||||
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
|
||||||
}
|
|
||||||
if config.SmokeTests.Authentication != nil {
|
|
||||||
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
|
||||||
Kind: config.SmokeTests.Authentication.Kind,
|
|
||||||
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
definition.SmokeTests = smokeTests
|
|
||||||
}
|
|
||||||
if config.Endpoint != nil {
|
|
||||||
definition.Endpoint = *config.Endpoint
|
|
||||||
}
|
|
||||||
if config.APIKeyEnv != nil {
|
|
||||||
definition.APIKeyEnv = *config.APIKeyEnv
|
|
||||||
}
|
|
||||||
if config.ConcurrencyLimit != nil {
|
|
||||||
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
|
||||||
}
|
|
||||||
if config.QueueCapacity != nil {
|
|
||||||
definition.QueueCapacity = *config.QueueCapacity
|
|
||||||
definition.QueueCapacitySet = definition.ConcurrencyLimit > 0
|
|
||||||
}
|
|
||||||
definitions[id] = definition
|
|
||||||
}
|
}
|
||||||
registry, err := backend.NewEffectiveRegistry(definitions)
|
state.registry = backend.NewEffectiveRegistry(definitions)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
state.registry = registry
|
|
||||||
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
||||||
for _, id := range sortedKeys(profileConfigs) {
|
for _, id := range sortedKeys(profileConfigs) {
|
||||||
config := profileConfigs[id]
|
config := profileConfigs[id]
|
||||||
fields := state.profiles[id]
|
profile, err := captureExecutionProfile(id, config, state.registry)
|
||||||
target := defaults.ExecutionTargetDefault()
|
if err != nil {
|
||||||
if config.Backend != nil && *config.Backend != "" {
|
return err
|
||||||
selected, err := registry.GetBackend(*config.Backend)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("profile %q backend: %w", id, err)
|
|
||||||
}
|
|
||||||
target.BackendID = selected.ID
|
|
||||||
target.Endpoint = selected.Endpoint
|
|
||||||
target.APIKeyEnv = selected.APIKeyEnv
|
|
||||||
target.ExtraParams = selected.ExtraParams
|
|
||||||
}
|
}
|
||||||
presence := domain.ExecutionTargetPresence{}
|
|
||||||
if fields["endpoint"] != fieldAbsent {
|
|
||||||
target.Endpoint = valueOrZero(config.Endpoint)
|
|
||||||
}
|
|
||||||
if fields["model"] != fieldAbsent {
|
|
||||||
target.Model = valueOrZero(config.Model)
|
|
||||||
}
|
|
||||||
if fields["temperature"] != fieldAbsent {
|
|
||||||
target.Temperature = valueOrZero(config.Temperature)
|
|
||||||
presence.Temperature = fields["temperature"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["top_p"] != fieldAbsent {
|
|
||||||
target.TopP = valueOrZero(config.TopP)
|
|
||||||
presence.TopP = fields["top_p"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["max_tokens"] != fieldAbsent {
|
|
||||||
target.MaxTokens = valueOrZero(config.MaxTokens)
|
|
||||||
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["timeout_seconds"] == fieldValue {
|
|
||||||
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
|
||||||
presence.TimeoutSeconds = true
|
|
||||||
}
|
|
||||||
if fields["service_tier"] != fieldAbsent {
|
|
||||||
target.ServiceTier = valueOrZero(config.ServiceTier)
|
|
||||||
}
|
|
||||||
if fields["reasoning_effort"] != fieldAbsent {
|
|
||||||
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
|
||||||
}
|
|
||||||
if fields["api_key_env"] != fieldAbsent {
|
|
||||||
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
|
||||||
}
|
|
||||||
if fields["api_key_required"] != fieldAbsent {
|
|
||||||
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
|
||||||
}
|
|
||||||
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
|
||||||
target.APIKeyEnv = ""
|
|
||||||
}
|
|
||||||
switch fields["extra_params"] {
|
|
||||||
case fieldNull:
|
|
||||||
target.ExtraParams = nil
|
|
||||||
case fieldValue:
|
|
||||||
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("profile %q extra parameters: %w", id, err)
|
|
||||||
}
|
|
||||||
target.ExtraParams = merged
|
|
||||||
}
|
|
||||||
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
|
||||||
state.indexedProfiles[id] = profile
|
state.indexedProfiles[id] = profile
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func captureBackend(id string, config backendConfig) domain.Backend {
|
||||||
|
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
||||||
|
if config.SmokeTests != nil {
|
||||||
|
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
||||||
|
if config.SmokeTests.Reachability != nil {
|
||||||
|
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication != nil {
|
||||||
|
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
||||||
|
Kind: config.SmokeTests.Authentication.Kind,
|
||||||
|
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
definition.SmokeTests = smokeTests
|
||||||
|
}
|
||||||
|
if config.Endpoint != nil {
|
||||||
|
definition.Endpoint = *config.Endpoint
|
||||||
|
}
|
||||||
|
if config.APIKeyEnv != nil {
|
||||||
|
definition.APIKeyEnv = *config.APIKeyEnv
|
||||||
|
}
|
||||||
|
if config.ConcurrencyLimit != nil {
|
||||||
|
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
||||||
|
}
|
||||||
|
if config.QueueCapacity != nil {
|
||||||
|
definition.QueueCapacity = *config.QueueCapacity
|
||||||
|
}
|
||||||
|
return definition
|
||||||
|
}
|
||||||
|
|
||||||
|
func captureExecutionProfile(id string, config profileConfig, registry *backend.Registry) (domain.ExecutionProfile, error) {
|
||||||
|
target := defaults.ExecutionTargetDefault()
|
||||||
|
if config.Backend != nil && *config.Backend != "" {
|
||||||
|
selected, err := registry.GetBackend(*config.Backend)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w", id, err)
|
||||||
|
}
|
||||||
|
target.BackendID = selected.ID
|
||||||
|
target.Endpoint = selected.Endpoint
|
||||||
|
target.APIKeyEnv = selected.APIKeyEnv
|
||||||
|
target.ExtraParams = selected.ExtraParams
|
||||||
|
}
|
||||||
|
presence := domain.ExecutionTargetPresence{}
|
||||||
|
if config.Endpoint != nil {
|
||||||
|
target.Endpoint = valueOrZero(config.Endpoint)
|
||||||
|
}
|
||||||
|
if config.Model != nil {
|
||||||
|
target.Model = valueOrZero(config.Model)
|
||||||
|
}
|
||||||
|
if config.Temperature != nil {
|
||||||
|
target.Temperature = valueOrZero(config.Temperature)
|
||||||
|
presence.Temperature = true
|
||||||
|
}
|
||||||
|
if config.TopP != nil {
|
||||||
|
target.TopP = valueOrZero(config.TopP)
|
||||||
|
presence.TopP = true
|
||||||
|
}
|
||||||
|
if config.MaxTokens != nil {
|
||||||
|
target.MaxTokens = valueOrZero(config.MaxTokens)
|
||||||
|
presence.MaxTokens = true
|
||||||
|
}
|
||||||
|
if config.TimeoutSeconds != nil {
|
||||||
|
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
||||||
|
presence.TimeoutSeconds = true
|
||||||
|
}
|
||||||
|
if config.ServiceTier != nil {
|
||||||
|
target.ServiceTier = valueOrZero(config.ServiceTier)
|
||||||
|
}
|
||||||
|
if config.ReasoningEffort != nil {
|
||||||
|
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
||||||
|
}
|
||||||
|
if config.APIKeyEnv != nil {
|
||||||
|
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
||||||
|
}
|
||||||
|
if config.APIKeyRequired != nil {
|
||||||
|
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
||||||
|
}
|
||||||
|
if target.APIKeyRequired && config.APIKeyEnv == nil {
|
||||||
|
target.APIKeyEnv = ""
|
||||||
|
}
|
||||||
|
if config.ExtraParams != nil {
|
||||||
|
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionProfile{}, fmt.Errorf("profile %q extra parameters: %w", id, err)
|
||||||
|
}
|
||||||
|
target.ExtraParams = merged
|
||||||
|
}
|
||||||
|
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
||||||
|
return profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
func valueOrZero[T any](value *T) T {
|
func valueOrZero[T any](value *T) T {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
var zero T
|
var zero T
|
||||||
@@ -138,24 +143,33 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
|||||||
if merged == nil {
|
if merged == nil {
|
||||||
merged = make(map[string]any)
|
merged = make(map[string]any)
|
||||||
}
|
}
|
||||||
|
if err := overlayOwnedProviderMap(merged, upper); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return merged, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlayOwnedProviderMap(merged, upper map[string]any) error {
|
||||||
for key, value := range upper {
|
for key, value := range upper {
|
||||||
if child, ok := value.(map[string]any); ok {
|
if child, ok := value.(map[string]any); ok {
|
||||||
if existing, ok := merged[key].(map[string]any); ok {
|
if existing, ok := merged[key].(map[string]any); ok {
|
||||||
combined, err := overlayProviderMap(existing, child)
|
if existing == nil {
|
||||||
if err != nil {
|
existing = make(map[string]any)
|
||||||
return nil, err
|
merged[key] = existing
|
||||||
|
}
|
||||||
|
if err := overlayOwnedProviderMap(existing, child); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
merged[key] = combined
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
copied, err := jsonvalue.Copy(value)
|
copied, err := jsonvalue.Copy(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return err
|
||||||
}
|
}
|
||||||
merged[key] = copied
|
merged[key] = copied
|
||||||
}
|
}
|
||||||
return merged, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
||||||
@@ -180,11 +194,7 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
target := *profile.ResolvedTarget
|
target := *profile.ResolvedTarget
|
||||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
target.ExtraParams = params
|
|
||||||
profile.ResolvedTarget = &target
|
profile.ResolvedTarget = &target
|
||||||
return &profile, nil
|
return &profile, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,6 +42,28 @@ func TestLoadIndexesIdentifiedTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOverlayProviderMapOwnsInputsAndReplacesValues(t *testing.T) {
|
||||||
|
lower := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"old"}, "null": "old"}, "disjoint": []any{"original"}}
|
||||||
|
upper := map[string]any{"nested": map[string]any{"replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}}
|
||||||
|
got, err := overlayProviderMap(lower, upper)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}, "disjoint": []any{"original"}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("overlay = %#v", got)
|
||||||
|
}
|
||||||
|
lower["disjoint"].([]any)[0] = "changed"
|
||||||
|
upper["nested"].(map[string]any)["replace"].([]any)[0] = "changed"
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("overlay shares input storage: %#v", got)
|
||||||
|
}
|
||||||
|
got["nested"].(map[string]any)["added"].(map[string]any)["value"] = "changed"
|
||||||
|
if upper["nested"].(map[string]any)["added"].(map[string]any)["value"] != "upper" {
|
||||||
|
t.Fatal("result mutation reached upper")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
||||||
state, err := load(mapping("partial profile", map[string]any{
|
state, err := load(mapping("partial profile", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "partial"}},
|
"profiles": []any{map[string]any{"id": "partial"}},
|
||||||
@@ -53,7 +75,7 @@ func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
runner := usecase.NewRunner(state, state, nil, prompt.NewGoRenderer(), nil, nil, nil)
|
runner := usecase.NewRunner(state, state, nil, prompt.NewCompiledRenderer(state), nil, nil, nil)
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "example", ProfileID: "partial",
|
PromptID: "example", ProfileID: "partial",
|
||||||
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
|
||||||
@@ -97,7 +119,7 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
selected, err := state.registry.GetBackend("second")
|
selected, err := state.registry.GetBackend("second")
|
||||||
if err != nil || selected.QueueCapacity != 0 || !selected.QueueCapacitySet {
|
if err != nil || selected.QueueCapacity != 0 || selected.ConcurrencyLimit != 2 {
|
||||||
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
||||||
}
|
}
|
||||||
child, err := state.GetProfile(context.Background(), "child")
|
child, err := state.GetProfile(context.Background(), "child")
|
||||||
@@ -110,7 +132,7 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
target := child.ResolvedTarget
|
target := child.ResolvedTarget
|
||||||
if target.BackendID != "second" || target.Endpoint != "https://second.example/v1" || target.Model != "model" ||
|
if target.BackendID != "second" || target.Endpoint != "https://second.example/v1" || target.Model != "model" ||
|
||||||
target.Temperature != 0 || target.TopP != 0 || target.MaxTokens != 0 ||
|
target.Temperature != 0 || target.TopP != 0 || target.MaxTokens != 0 ||
|
||||||
target.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds || target.APIKeyEnv != "" || target.APIKeyRequired {
|
target.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds || target.APIKeyEnv != "SECOND_KEY" || target.APIKeyRequired {
|
||||||
t.Fatalf("effective child target is wrong: %+v", target)
|
t.Fatalf("effective child target is wrong: %+v", target)
|
||||||
}
|
}
|
||||||
if !child.TargetPresence.Temperature || !child.TargetPresence.MaxTokens || child.TargetPresence.TopP || child.TargetPresence.TimeoutSeconds {
|
if !child.TargetPresence.Temperature || !child.TargetPresence.MaxTokens || child.TargetPresence.TopP || child.TargetPresence.TimeoutSeconds {
|
||||||
@@ -132,8 +154,8 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
noMap, _ := state.GetProfile(context.Background(), "no-provider-map")
|
noMap, _ := state.GetProfile(context.Background(), "no-provider-map")
|
||||||
if noMap.ResolvedTarget.ExtraParams != nil {
|
if !reflect.DeepEqual(noMap.ResolvedTarget.ExtraParams, selected.ExtraParams) {
|
||||||
t.Fatalf("explicit null provider map did not suppress backend defaults: %+v", noMap.ResolvedTarget.ExtraParams)
|
t.Fatalf("explicit null provider map did not inherit backend defaults: %+v", noMap.ResolvedTarget.ExtraParams)
|
||||||
}
|
}
|
||||||
noTimeout, _ := state.GetProfile(context.Background(), "no-timeout")
|
noTimeout, _ := state.GetProfile(context.Background(), "no-timeout")
|
||||||
if noTimeout.ResolvedTarget.TimeoutSeconds != 0 || !noTimeout.TargetPresence.TimeoutSeconds {
|
if noTimeout.ResolvedTarget.TimeoutSeconds != 0 || !noTimeout.TargetPresence.TimeoutSeconds {
|
||||||
@@ -161,8 +183,8 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
t.Fatalf("endpoint-only profile was not retained: %+v", endpointOnly.ResolvedTarget)
|
t.Fatalf("endpoint-only profile was not retained: %+v", endpointOnly.ResolvedTarget)
|
||||||
}
|
}
|
||||||
clearedEndpoint, _ := state.GetProfile(context.Background(), "cleared-endpoint")
|
clearedEndpoint, _ := state.GetProfile(context.Background(), "cleared-endpoint")
|
||||||
if clearedEndpoint.ResolvedTarget.Endpoint != "" || clearedEndpoint.ResolvedTarget.BackendID != "first" {
|
if clearedEndpoint.ResolvedTarget.Endpoint != "https://first.example/v1" || clearedEndpoint.ResolvedTarget.BackendID != "first" {
|
||||||
t.Fatalf("explicit null recovered backend endpoint: %+v", clearedEndpoint.ResolvedTarget)
|
t.Fatalf("explicit null did not recover backend endpoint: %+v", clearedEndpoint.ResolvedTarget)
|
||||||
}
|
}
|
||||||
child.ResolvedTarget.ExtraParams["source"] = "mutated"
|
child.ResolvedTarget.ExtraParams["source"] = "mutated"
|
||||||
again, _ := state.GetProfile(context.Background(), "child")
|
again, _ := state.GetProfile(context.Background(), "child")
|
||||||
@@ -171,6 +193,48 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProfileNullFallbackAndExplicitValues(t *testing.T) {
|
||||||
|
state, err := load(mapping("profile fallback", map[string]any{
|
||||||
|
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://backend.example/v1", "api_key_env": "BACKEND_KEY", "extra_params": map[string]any{"backend": true}}},
|
||||||
|
"profiles": []any{
|
||||||
|
map[string]any{"id": "parent", "backend": "backend", "endpoint": "https://parent.example/v1", "extra_params": map[string]any{"parent": true}},
|
||||||
|
map[string]any{"id": "inherited-null", "extends": map[string]any{"id": "parent"}, "endpoint": nil, "extra_params": nil, "api_key_env": nil},
|
||||||
|
map[string]any{"id": "omitted", "backend": "backend"},
|
||||||
|
map[string]any{"id": "null", "backend": "backend", "endpoint": nil, "api_key_env": nil, "extra_params": nil},
|
||||||
|
map[string]any{"id": "empty", "backend": "backend", "endpoint": "", "api_key_env": "", "extra_params": map[string]any{}},
|
||||||
|
map[string]any{"id": "required-null", "backend": "backend", "api_key_required": true, "api_key_env": nil},
|
||||||
|
map[string]any{"id": "required-omitted", "backend": "backend", "api_key_required": true},
|
||||||
|
map[string]any{"id": "required-explicit", "backend": "backend", "api_key_required": true, "api_key_env": "PROFILE_KEY"},
|
||||||
|
},
|
||||||
|
}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, id := range []string{"inherited-null", "omitted", "null", "empty", "required-null", "required-omitted", "required-explicit"} {
|
||||||
|
profile, err := state.GetProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
target := profile.ResolvedTarget
|
||||||
|
wantEndpoint, wantKey := "https://backend.example/v1", "BACKEND_KEY"
|
||||||
|
switch id {
|
||||||
|
case "empty":
|
||||||
|
wantEndpoint, wantKey = "", ""
|
||||||
|
case "required-null", "required-omitted":
|
||||||
|
wantKey = ""
|
||||||
|
case "required-explicit":
|
||||||
|
wantKey = "PROFILE_KEY"
|
||||||
|
}
|
||||||
|
if target.Endpoint != wantEndpoint || target.APIKeyEnv != wantKey || !reflect.DeepEqual(target.ExtraParams, map[string]any{"backend": true}) {
|
||||||
|
t.Errorf("%s target = %+v", id, target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
child := state.result.Config.Profiles[1]
|
||||||
|
if child.Endpoint != nil || child.ExtraParams != nil || child.APIKeyEnv != nil {
|
||||||
|
t.Fatalf("inherited nulls were not retained in resolved values: %+v", child)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
||||||
payloads := make(chan map[string]any, 1)
|
payloads := make(chan map[string]any, 1)
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||||
@@ -197,10 +261,7 @@ func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
||||||
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
||||||
|
|||||||
@@ -145,11 +145,10 @@ type PromptDefinition struct {
|
|||||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||||
Inputs []PromptInput `yaml:"inputs"`
|
Inputs []PromptInput `yaml:"inputs"`
|
||||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
Templates []PromptMessageTemplate `yaml:"templates"`
|
||||||
OutputFormat OutputFormat `yaml:"output_format"`
|
|
||||||
Validation OutputContract `yaml:"validation"`
|
Validation OutputContract `yaml:"validation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptInspection is the resolved result of exact prompt inspection.
|
// PromptInspection is lightweight metadata for prompt requirement selection.
|
||||||
type PromptInspection struct {
|
type PromptInspection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
@@ -170,7 +169,6 @@ type PromptInput struct {
|
|||||||
type PromptMessageTemplate struct {
|
type PromptMessageTemplate struct {
|
||||||
Role string `yaml:"role"`
|
Role string `yaml:"role"`
|
||||||
Content string `yaml:"content"`
|
Content string `yaml:"content"`
|
||||||
ContentFile string `yaml:"content_file"`
|
|
||||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -184,7 +182,6 @@ type Backend struct {
|
|||||||
SmokeTests *BackendSmokeTestConfig
|
SmokeTests *BackendSmokeTestConfig
|
||||||
ConcurrencyLimit int
|
ConcurrencyLimit int
|
||||||
QueueCapacity int
|
QueueCapacity int
|
||||||
QueueCapacitySet bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendCapacityPolicy describes normalized run and generation capacity for
|
// BackendCapacityPolicy describes normalized run and generation capacity for
|
||||||
@@ -242,13 +239,6 @@ type ExecutionTarget struct {
|
|||||||
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProfileInspection is the resolved result of exact profile inspection.
|
|
||||||
type ProfileInspection struct {
|
|
||||||
ProfileID string
|
|
||||||
EffectiveModelParams ExecutionTarget
|
|
||||||
APIKeyRequired bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
// OutputContract defines the requirements for the output artifact.
|
||||||
type OutputContract struct {
|
type OutputContract struct {
|
||||||
Format OutputFormat `yaml:"format"`
|
Format OutputFormat `yaml:"format"`
|
||||||
|
|||||||
@@ -31,14 +31,6 @@ func qualifyResourceID(id, namespace string) (string, bool) {
|
|||||||
return namespace + "/" + id, true
|
return namespace + "/" + id, true
|
||||||
}
|
}
|
||||||
|
|
||||||
func resourceNamespaceFromID(id string) (string, bool) {
|
|
||||||
if !validResourceID(id) || !strings.Contains(id, "/") {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
namespace, _, _ := strings.Cut(id, "/")
|
|
||||||
return namespace, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
||||||
// valid namespace/local-ID pair.
|
// valid namespace/local-ID pair.
|
||||||
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
||||||
|
|||||||
@@ -86,25 +86,3 @@ func TestQualifyResourceID(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResourceNamespaceFromID(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
id string
|
|
||||||
namespace string
|
|
||||||
wantOK bool
|
|
||||||
}{
|
|
||||||
{name: "qualified", id: "team/item", namespace: "team", wantOK: true},
|
|
||||||
{name: "same-prefix namespace stays distinct", id: "team-extra/item", namespace: "team-extra", wantOK: true},
|
|
||||||
{name: "unqualified", id: "item", wantOK: false},
|
|
||||||
{name: "malformed", id: "team/sub/item", wantOK: false},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
got, ok := resourceNamespaceFromID(test.id)
|
|
||||||
if got != test.namespace || ok != test.wantOK {
|
|
||||||
t.Fatalf("resourceNamespaceFromID(%q) = (%q, %t), want (%q, %t)", test.id, got, ok, test.namespace, test.wantOK)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,5 +18,5 @@ func NormalizeSessionID(raw string) (string, error) {
|
|||||||
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
||||||
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
||||||
}
|
}
|
||||||
return normalized, nil
|
return strings.Clone(normalized), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,24 +112,21 @@ type SmokeTestAttempt struct {
|
|||||||
type SmokeTestReason string
|
type SmokeTestReason string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
SmokeTestReasonNone SmokeTestReason = ""
|
SmokeTestReasonNone SmokeTestReason = ""
|
||||||
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
||||||
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
||||||
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
||||||
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
||||||
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
||||||
|
SmokeTestReasonAuthenticationResponseLimitExceeded SmokeTestReason = "authentication_response_limit_exceeded"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CloneBackend returns a copy of a backend, including its mutable settings.
|
// CloneBackend returns a copy of a backend, including its mutable settings.
|
||||||
func CloneBackend(backend Backend) (Backend, error) {
|
func CloneBackend(backend Backend) Backend {
|
||||||
cloned := backend
|
cloned := backend
|
||||||
var err error
|
cloned.ExtraParams = jsonvalue.CloneTrusted(backend.ExtraParams).(map[string]any)
|
||||||
cloned.ExtraParams, err = jsonvalue.CopyMap(backend.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return Backend{}, err
|
|
||||||
}
|
|
||||||
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
||||||
return cloned, nil
|
return cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
||||||
@@ -153,105 +150,6 @@ func CloneBackendSmokeTestConfig(config *BackendSmokeTestConfig) *BackendSmokeTe
|
|||||||
return &cloned
|
return &cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneSmokeTestRequest returns a copy that owns its optional settings.
|
|
||||||
func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
|
|
||||||
cloned := request
|
|
||||||
cloned.TransientRetries = cloneInt(request.TransientRetries)
|
|
||||||
if request.Execution == nil {
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
execution := *request.Execution
|
|
||||||
execution.Temperature = cloneFloat64(request.Execution.Temperature)
|
|
||||||
execution.MaxTokens = cloneInt(request.Execution.MaxTokens)
|
|
||||||
execution.TopP = cloneFloat64(request.Execution.TopP)
|
|
||||||
execution.TimeoutSeconds = cloneInt(request.Execution.TimeoutSeconds)
|
|
||||||
execution.ReasoningEffort = cloneString(request.Execution.ReasoningEffort)
|
|
||||||
var err error
|
|
||||||
execution.ExtraParams, err = jsonvalue.CopyMap(request.Execution.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return SmokeTestRequest{}, err
|
|
||||||
}
|
|
||||||
cloned.Execution = &execution
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CloneSmokeTestReport returns a copy with independent optional facts and maps.
|
|
||||||
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
|
|
||||||
cloned := report
|
|
||||||
cloned.Applicable = cloneBool(report.Applicable)
|
|
||||||
cloned.Facts = cloneSmokeTestFacts(report.Facts)
|
|
||||||
cloned.Timing = cloneSmokeTestTiming(report.Timing)
|
|
||||||
var err error
|
|
||||||
cloned.Target.EffectiveSettings.ExtraParams, err = jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return SmokeTestReport{}, err
|
|
||||||
}
|
|
||||||
if report.Attempts != nil {
|
|
||||||
cloned.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
|
|
||||||
for index, attempt := range report.Attempts {
|
|
||||||
cloned.Attempts[index] = attempt
|
|
||||||
cloned.Attempts[index].HTTPStatus = cloneInt(attempt.HTTPStatus)
|
|
||||||
cloned.Attempts[index].Facts = cloneSmokeTestFacts(attempt.Facts)
|
|
||||||
cloned.Attempts[index].Timing = cloneSmokeTestTiming(attempt.Timing)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneSmokeTestFacts(facts SmokeTestFacts) SmokeTestFacts {
|
|
||||||
facts.CredentialsSupplied = cloneBool(facts.CredentialsSupplied)
|
|
||||||
facts.EndpointReachable = cloneBool(facts.EndpointReachable)
|
|
||||||
facts.CredentialsAccepted = cloneBool(facts.CredentialsAccepted)
|
|
||||||
facts.ResponseReceived = cloneBool(facts.ResponseReceived)
|
|
||||||
facts.GenerationCompleted = cloneBool(facts.GenerationCompleted)
|
|
||||||
facts.TextReturned = cloneBool(facts.TextReturned)
|
|
||||||
facts.ExpectedTextMatched = cloneBool(facts.ExpectedTextMatched)
|
|
||||||
facts.StructuredOutputRequested = cloneBool(facts.StructuredOutputRequested)
|
|
||||||
facts.ValidJSON = cloneBool(facts.ValidJSON)
|
|
||||||
facts.SchemaMatched = cloneBool(facts.SchemaMatched)
|
|
||||||
return facts
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneSmokeTestTiming(timing SmokeTestTiming) SmokeTestTiming {
|
|
||||||
timing.CapacityWait = cloneDuration(timing.CapacityWait)
|
|
||||||
timing.RequestDuration = cloneDuration(timing.RequestDuration)
|
|
||||||
timing.RetryDelay = cloneDuration(timing.RetryDelay)
|
|
||||||
timing.EvaluationDuration = cloneDuration(timing.EvaluationDuration)
|
|
||||||
return timing
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneBool(value *bool) *bool {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneDuration(value *time.Duration) *time.Duration {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneFloat64(value *float64) *float64 {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneInt(value *int) *int {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneString(value *string) *string {
|
func cloneString(value *string) *string {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
package domain
|
package domain
|
||||||
|
|
||||||
import (
|
import "testing"
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
||||||
profile := "profile"
|
profile := "profile"
|
||||||
@@ -22,10 +19,7 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cloned, err := CloneBackend(backend)
|
cloned := CloneBackend(backend)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.SmokeTests.DefaultProfile = "changed"
|
*cloned.SmokeTests.DefaultProfile = "changed"
|
||||||
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
||||||
*cloned.SmokeTests.Authentication.Kind = "changed"
|
*cloned.SmokeTests.Authentication.Kind = "changed"
|
||||||
@@ -41,72 +35,12 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCloneSmokeTestRequestOwnsOptionalSettings(t *testing.T) {
|
func TestCloneBackendPreservesProviderMapPresence(t *testing.T) {
|
||||||
temperature := 0.25
|
if cloned := CloneBackend(Backend{}); cloned.ExtraParams != nil {
|
||||||
retries := 2
|
t.Fatalf("nil provider map became %#v", cloned.ExtraParams)
|
||||||
reasoning := "low"
|
|
||||||
request := SmokeTestRequest{
|
|
||||||
APIKey: "secret",
|
|
||||||
TransientRetries: &retries,
|
|
||||||
Execution: &ExecutionTargetOverride{
|
|
||||||
Temperature: &temperature,
|
|
||||||
ReasoningEffort: &reasoning,
|
|
||||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
cloned := CloneBackend(Backend{ExtraParams: map[string]any{"empty": map[string]any{}, "null": nil}})
|
||||||
cloned, err := CloneSmokeTestRequest(request)
|
if cloned.ExtraParams == nil || cloned.ExtraParams["empty"].(map[string]any) == nil || cloned.ExtraParams["null"] != nil {
|
||||||
if err != nil {
|
t.Fatalf("provider map presence changed: %#v", cloned.ExtraParams)
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.TransientRetries = 3
|
|
||||||
*cloned.Execution.Temperature = 0.5
|
|
||||||
*cloned.Execution.ReasoningEffort = "high"
|
|
||||||
cloned.Execution.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
|
|
||||||
if retries != 2 || temperature != 0.25 || reasoning != "low" {
|
|
||||||
t.Fatal("cloned request shared optional scalar pointers")
|
|
||||||
}
|
|
||||||
if got := request.Execution.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
|
||||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
|
|
||||||
applicable := true
|
|
||||||
reachable := true
|
|
||||||
status := 200
|
|
||||||
wait := 3 * time.Millisecond
|
|
||||||
report := SmokeTestReport{
|
|
||||||
Applicable: &applicable,
|
|
||||||
Target: SmokeTestTarget{
|
|
||||||
EffectiveSettings: ExecutionTarget{
|
|
||||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
|
||||||
Attempts: []SmokeTestAttempt{{
|
|
||||||
HTTPStatus: &status,
|
|
||||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
|
||||||
Timing: SmokeTestTiming{CapacityWait: &wait},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
|
|
||||||
cloned, err := CloneSmokeTestReport(report)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.Applicable = false
|
|
||||||
*cloned.Facts.EndpointReachable = false
|
|
||||||
*cloned.Attempts[0].HTTPStatus = 503
|
|
||||||
*cloned.Attempts[0].Facts.EndpointReachable = false
|
|
||||||
*cloned.Attempts[0].Timing.CapacityWait = time.Second
|
|
||||||
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
|
|
||||||
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond {
|
|
||||||
t.Fatal("cloned report shared optional scalar pointers")
|
|
||||||
}
|
|
||||||
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
|
||||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ type ConfigurationSources struct {
|
|||||||
type Options struct {
|
type Options struct {
|
||||||
llmClient llm.Client
|
llmClient llm.Client
|
||||||
artifactReader artifact.Reader
|
artifactReader artifact.Reader
|
||||||
validator validate.Validator
|
validator validate.ValidationPreparer
|
||||||
validatorSource bool
|
validatorSource bool
|
||||||
artifactSource bool
|
artifactSource bool
|
||||||
disableBundledOpenRouter bool
|
disableBundledOpenRouter bool
|
||||||
@@ -73,7 +73,7 @@ func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
|
|||||||
|
|
||||||
// SetSchemaFile selects a single schema file.
|
// SetSchemaFile selects a single schema file.
|
||||||
func (o *Options) SetSchemaFile(path string) error {
|
func (o *Options) SetSchemaFile(path string) error {
|
||||||
fsys, root, _, err := fileSource(path)
|
fsys, root, err := fileSource(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -87,6 +87,7 @@ type Engine struct {
|
|||||||
runner *usecase.Runner
|
runner *usecase.Runner
|
||||||
smokeTester *usecase.SmokeTester
|
smokeTester *usecase.SmokeTester
|
||||||
inspection configuration.Inspection
|
inspection configuration.Inspection
|
||||||
|
resources resourceInspection
|
||||||
listing identityInventory
|
listing identityInventory
|
||||||
namespaces resourceNamespaces
|
namespaces resourceNamespaces
|
||||||
}
|
}
|
||||||
@@ -98,13 +99,19 @@ type resourceNamespaces struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type identityInventory interface {
|
type identityInventory interface {
|
||||||
ListBackends(context.Context) ([]string, error)
|
ListBackends(context.Context, configuration.IdentityFilter) ([]string, error)
|
||||||
ListProfiles(context.Context) ([]string, error)
|
ListProfiles(context.Context, configuration.IdentityFilter) ([]string, error)
|
||||||
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
ListPrompts(context.Context, configuration.IdentityFilter) ([]domain.PromptIdentity, error)
|
||||||
ListNamespaces(context.Context, []string) ([]string, error)
|
ListNamespaces(context.Context, []string) ([]string, error)
|
||||||
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type resourceInspection interface {
|
||||||
|
InspectPrompt(context.Context, string) (*configuration.PromptResource, error)
|
||||||
|
InspectProfile(context.Context, string) (*configuration.ProfileResource, error)
|
||||||
|
InspectBackend(context.Context, string) (*configuration.BackendResource, error)
|
||||||
|
}
|
||||||
|
|
||||||
// ListingFilterMode selects namespace membership for one identity listing.
|
// ListingFilterMode selects namespace membership for one identity listing.
|
||||||
type ListingFilterMode uint8
|
type ListingFilterMode uint8
|
||||||
|
|
||||||
@@ -157,10 +164,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
var client llm.Client
|
var client llm.Client
|
||||||
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
||||||
if options.llmClient == nil {
|
if options.llmClient == nil {
|
||||||
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
builtIn := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
||||||
if newErr != nil {
|
|
||||||
return nil, newErr
|
|
||||||
}
|
|
||||||
diagnosticHTTPClient = builtIn
|
diagnosticHTTPClient = builtIn
|
||||||
client = generation.NewBuiltIn(manager, builtIn)
|
client = generation.NewBuiltIn(manager, builtIn)
|
||||||
} else {
|
} else {
|
||||||
@@ -176,6 +180,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
runner: runner,
|
runner: runner,
|
||||||
smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
|
smokeTester: usecase.NewSmokeTester(runner, diagnosticHTTPClient),
|
||||||
inspection: snapshot.Inspection(),
|
inspection: snapshot.Inspection(),
|
||||||
|
resources: snapshot,
|
||||||
listing: snapshot,
|
listing: snapshot,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -208,32 +213,35 @@ func validateSourceFS(fsys fs.FS, root string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, string, error) {
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
if strings.TrimSpace(name) == "" {
|
if strings.TrimSpace(name) == "" {
|
||||||
return nil, "", "", fmt.Errorf("source path is required")
|
return nil, "", fmt.Errorf("source path is required")
|
||||||
}
|
}
|
||||||
directory := filepath.Dir(name)
|
directory := filepath.Dir(name)
|
||||||
base := filepath.Base(name)
|
base := filepath.Base(name)
|
||||||
if base == "." || base == string(filepath.Separator) {
|
if base == "." || base == string(filepath.Separator) {
|
||||||
return nil, "", "", fmt.Errorf("source path must name a file")
|
return nil, "", fmt.Errorf("source path must name a file")
|
||||||
}
|
}
|
||||||
info, err := os.Stat(name)
|
info, err := os.Stat(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
return nil, "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
||||||
}
|
}
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return nil, "", "", fmt.Errorf("source path %q must be a file", name)
|
return nil, "", fmt.Errorf("source path %q must be a file", name)
|
||||||
}
|
}
|
||||||
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
return os.DirFS(directory), filepath.ToSlash(base), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*configuration.PromptResource, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: prompt id is required", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
var err error
|
var err error
|
||||||
id, err = e.qualifyResourceID(id, e.namespaces.prompt, "prompt")
|
id, err = e.qualifyResourceID(id, e.namespaces.prompt, "prompt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return e.runner.InspectPrompt(ctx, id)
|
return e.resources.InspectPrompt(ctx, id)
|
||||||
}
|
}
|
||||||
func (e *Engine) InspectRequirements(
|
func (e *Engine) InspectRequirements(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@@ -250,13 +258,27 @@ func (e *Engine) InspectRequirements(
|
|||||||
}
|
}
|
||||||
return e.runner.InspectRequirements(ctx, request)
|
return e.runner.InspectRequirements(ctx, request)
|
||||||
}
|
}
|
||||||
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
|
func (e *Engine) InspectProfile(ctx context.Context, id string) (*configuration.ProfileResource, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: profile id is required", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
var err error
|
var err error
|
||||||
id, err = e.qualifyResourceID(id, e.namespaces.profile, "profile")
|
id, err = e.qualifyResourceID(id, e.namespaces.profile, "profile")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return e.runner.InspectProfile(ctx, id)
|
return e.resources.InspectProfile(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) InspectBackend(ctx context.Context, id string) (*configuration.BackendResource, error) {
|
||||||
|
if strings.TrimSpace(id) == "" {
|
||||||
|
return nil, fmt.Errorf("%w: backend id is required", usecase.ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
id, err := e.qualifyResourceID(id, e.namespaces.backend, "backend")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return e.resources.InspectBackend(ctx, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListBackends(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
func (e *Engine) ListBackends(ctx context.Context, filters ...ListingFilter) ([]string, error) {
|
||||||
@@ -268,24 +290,11 @@ func (e *Engine) ListProfiles(ctx context.Context, filters ...ListingFilter) ([]
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
||||||
mode, namespace, err := resolveListingFilter(filters, e.namespaces.prompt)
|
filter, err := inventoryFilter(filters, e.namespaces.prompt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
prompts, err := e.listing.ListPrompts(ctx)
|
return e.listing.ListPrompts(ctx, filter)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var filtered []domain.PromptIdentity
|
|
||||||
for _, prompt := range prompts {
|
|
||||||
if matchesListingNamespace(prompt.ID, mode, namespace) {
|
|
||||||
filtered = append(filtered, prompt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return filtered, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
@@ -307,26 +316,21 @@ func filterIdentityList(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
filters []ListingFilter,
|
filters []ListingFilter,
|
||||||
scope string,
|
scope string,
|
||||||
list func(context.Context) ([]string, error),
|
list func(context.Context, configuration.IdentityFilter) ([]string, error),
|
||||||
) ([]string, error) {
|
) ([]string, error) {
|
||||||
|
filter, err := inventoryFilter(filters, scope)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return list(ctx, filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func inventoryFilter(filters []ListingFilter, scope string) (configuration.IdentityFilter, error) {
|
||||||
mode, namespace, err := resolveListingFilter(filters, scope)
|
mode, namespace, err := resolveListingFilter(filters, scope)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return configuration.IdentityFilter{}, err
|
||||||
}
|
}
|
||||||
ids, err := list(ctx)
|
return configuration.IdentityFilter{All: mode == ListingAll, Namespace: namespace}, nil
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var filtered []string
|
|
||||||
for _, id := range ids {
|
|
||||||
if matchesListingNamespace(id, mode, namespace) {
|
|
||||||
filtered = append(filtered, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return filtered, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
||||||
@@ -355,19 +359,6 @@ func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterM
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func matchesListingNamespace(id string, mode ListingFilterMode, namespace string) bool {
|
|
||||||
switch mode {
|
|
||||||
case ListingNamed:
|
|
||||||
idNamespace, _, qualified := strings.Cut(id, "/")
|
|
||||||
return qualified && idNamespace == namespace
|
|
||||||
case ListingUnqualified:
|
|
||||||
return !strings.Contains(id, "/")
|
|
||||||
case ListingAll:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||||
request, err := e.qualifyRunRequest(request)
|
request, err := e.qualifyRunRequest(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -67,15 +67,6 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
|
|||||||
return files, err
|
return files, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// RelativePath computes a clean relative path from root to path.
|
|
||||||
func RelativePath(root string, filePath string) string {
|
|
||||||
rel, err := filepath.Rel(root, filePath)
|
|
||||||
if err != nil {
|
|
||||||
return filepath.Clean(filePath)
|
|
||||||
}
|
|
||||||
return filepath.Clean(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
||||||
// nonblank leading and trailing whitespace.
|
// nonblank leading and trailing whitespace.
|
||||||
func CleanFSRoot(root string) string {
|
func CleanFSRoot(root string) string {
|
||||||
|
|||||||
@@ -81,16 +81,6 @@ func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRelativePathNested(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
|
||||||
got := RelativePath(root, path)
|
|
||||||
want := filepath.Join("nested", "profiles", "local.yaml")
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("expected relative path %q, got %q", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanFSRoot(t *testing.T) {
|
func TestCleanFSRoot(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -85,6 +85,12 @@ func (c *Coordinator) GenerateWithEvidence(
|
|||||||
return c.generate(ctx, req, true)
|
return c.generate(ctx, req, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreparationError identifies a built-in request preparation failure.
|
||||||
|
type PreparationError struct{ Cause error }
|
||||||
|
|
||||||
|
func (e *PreparationError) Error() string { return e.Cause.Error() }
|
||||||
|
func (e *PreparationError) Unwrap() error { return e.Cause }
|
||||||
|
|
||||||
func (c *Coordinator) generate(
|
func (c *Coordinator) generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -95,7 +101,7 @@ func (c *Coordinator) generate(
|
|||||||
}
|
}
|
||||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err, GenerationEvidence{}
|
return nil, &PreparationError{Cause: err}, GenerationEvidence{PreparationFailed: true}
|
||||||
}
|
}
|
||||||
|
|
||||||
evidence := GenerationEvidence{}
|
evidence := GenerationEvidence{}
|
||||||
@@ -120,66 +126,37 @@ func (c *Coordinator) generate(
|
|||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
ordinal := attempts + 1
|
ordinal := attempts + 1
|
||||||
outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
result := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
||||||
if dispatched {
|
if result.Dispatched {
|
||||||
attempts++
|
attempts++
|
||||||
}
|
}
|
||||||
if captureEvidence {
|
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
|
||||||
attempt := AttemptEvidence{
|
if result.Outcome.Err == nil {
|
||||||
StartedAt: startedAt.UTC(),
|
|
||||||
Dispatched: dispatched,
|
|
||||||
ResponseReceived: outcome.Evidence.ResponseReceived,
|
|
||||||
HTTP: cloneHTTPEvidence(outcome.Evidence),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
RequestDuration: generationDuration,
|
|
||||||
Failure: ClassifyFailure(outcome.Err),
|
|
||||||
ResponseDecoded: outcome.ResponseDecoded,
|
|
||||||
GenerationCompleted: outcome.GenerationCompleted,
|
|
||||||
}
|
|
||||||
if outcome.StructuredOutputRequested != nil {
|
|
||||||
requested := *outcome.StructuredOutputRequested
|
|
||||||
attempt.StructuredOutputRequested = &requested
|
|
||||||
}
|
|
||||||
if dispatched {
|
|
||||||
usage.Add(outcome.Usage, outcome.UsagePresence)
|
|
||||||
var attemptUsage observation.UsageAccumulator
|
|
||||||
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
|
|
||||||
attempt.Usage = attemptUsage.Observed()
|
|
||||||
}
|
|
||||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
|
||||||
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
|
||||||
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
|
|
||||||
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
|
||||||
}
|
|
||||||
if outcome.Err == nil {
|
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(outcome.Response, err)
|
return finish(result.Outcome.Response, err)
|
||||||
}
|
}
|
||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
return finish(outcome.Response, nil)
|
return finish(result.Outcome.Response, nil)
|
||||||
}
|
}
|
||||||
if !dispatched || !eligibleStatus(outcome) {
|
if !result.Dispatched || !eligibleStatus(result.Outcome) {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(outcome.Response, outcome.Err)
|
return finish(result.Outcome.Response, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
return finish(nil, outcome.Err)
|
return finish(nil, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
if retriesUsed == req.TransientRetries {
|
if retriesUsed == req.TransientRetries {
|
||||||
if retriesUsed == 0 {
|
if retriesUsed == 0 {
|
||||||
return finish(nil, outcome.Err)
|
return finish(nil, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err})
|
return finish(nil, &ExhaustedError{attempts: attempts, cause: result.Outcome.Err})
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
retryOrdinal := retriesUsed + 1
|
retryOrdinal := retriesUsed + 1
|
||||||
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
delay := retryDelay(c.backoff(retryOrdinal), result.Outcome.RetryNotBefore, c.now())
|
||||||
waitStart := c.now()
|
waitStart := c.now()
|
||||||
waitErr := c.wait(ctx, delay)
|
waitErr := c.wait(ctx, delay)
|
||||||
waitDuration := elapsed(waitStart, c.now())
|
waitDuration := elapsed(waitStart, c.now())
|
||||||
@@ -200,127 +177,133 @@ func (c *Coordinator) generate(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) generateInjected(
|
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest, captureEvidence bool) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
ctx context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
captureEvidence bool,
|
|
||||||
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
|
||||||
evidence := GenerationEvidence{}
|
evidence := GenerationEvidence{}
|
||||||
var usage observation.UsageAccumulator
|
var usage observation.UsageAccumulator
|
||||||
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Usage = usage.Observed()
|
|
||||||
}
|
|
||||||
return response, err, evidence
|
|
||||||
}
|
|
||||||
recorder := observation.FromContext(ctx)
|
|
||||||
generation := req.Generation
|
generation := req.Generation
|
||||||
if generation == 0 {
|
if generation == 0 {
|
||||||
generation = 1
|
generation = 1
|
||||||
}
|
}
|
||||||
start := c.now()
|
result := c.executeInjected(ctx, req)
|
||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
|
||||||
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
|
||||||
if err != nil {
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
|
||||||
StartedAt: start.UTC(),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
|
||||||
}
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
release()
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
|
||||||
StartedAt: start.UTC(),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
|
||||||
}
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
|
||||||
dispatchStart := c.now()
|
|
||||||
response, err := c.injected.Generate(ctx, req)
|
|
||||||
duration := elapsed(dispatchStart, c.now())
|
|
||||||
if err == nil && response == nil {
|
|
||||||
err = llm.ErrMalformedResponse
|
|
||||||
}
|
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
attempt := AttemptEvidence{
|
evidence.Usage = usage.Observed()
|
||||||
StartedAt: start.UTC(),
|
|
||||||
Dispatched: true,
|
|
||||||
ResponseReceived: response != nil,
|
|
||||||
RequestDuration: duration,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
GenerationCompleted: err == nil && response != nil,
|
|
||||||
}
|
|
||||||
requested := req.StructuredOutput != nil
|
|
||||||
attempt.StructuredOutputRequested = &requested
|
|
||||||
var reportedUsage domain.TokenUsage
|
|
||||||
var reportedPresence *domain.TokenUsagePresence
|
|
||||||
if err == nil && response != nil {
|
|
||||||
reportedUsage = response.Usage
|
|
||||||
reportedPresence = response.UsagePresence
|
|
||||||
}
|
|
||||||
usage.Add(reportedUsage, reportedPresence)
|
|
||||||
var attemptUsage observation.UsageAccumulator
|
|
||||||
attemptUsage.Add(reportedUsage, reportedPresence)
|
|
||||||
attempt.Usage = attemptUsage.Observed()
|
|
||||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
|
||||||
}
|
}
|
||||||
release()
|
if result.Outcome.Err != nil {
|
||||||
if recorder != nil {
|
return nil, result.Outcome.Err, evidence
|
||||||
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
|
||||||
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
|
|
||||||
if err == nil {
|
|
||||||
data.Usage = response.Usage
|
|
||||||
data.UsagePresence = response.UsagePresence
|
|
||||||
}
|
|
||||||
recorder.Attempt(data)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(response, err)
|
return result.Outcome.Response, err, evidence
|
||||||
}
|
}
|
||||||
return finish(nil, err)
|
return nil, err, evidence
|
||||||
}
|
}
|
||||||
return finish(response, nil)
|
return result.Outcome.Response, nil, evidence
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) {
|
func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRequest) attemptResult {
|
||||||
start := c.now()
|
result := attemptResult{StartedAt: c.now()}
|
||||||
startedAt = start
|
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||||
release, err := c.manager.Acquire(ctx, backendID)
|
result.CapacityWait = c.capacityWait(req.Target.BackendID, result.StartedAt)
|
||||||
capacityWait = c.capacityWait(backendID, start)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
start := c.now()
|
||||||
|
result.Dispatched = true
|
||||||
|
requested := req.StructuredOutput != nil
|
||||||
|
result.Outcome.StructuredOutputRequested = &requested
|
||||||
|
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
|
||||||
|
result.RequestDuration = elapsed(start, c.now())
|
||||||
|
if result.Outcome.Err == nil && result.Outcome.Response == nil {
|
||||||
|
result.Outcome.Err = llm.ErrMalformedResponse
|
||||||
|
}
|
||||||
|
if result.Outcome.Err == nil {
|
||||||
|
result.Outcome.Usage = result.Outcome.Response.Usage
|
||||||
|
result.Outcome.UsagePresence = result.Outcome.Response.UsagePresence
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type attemptResult struct {
|
||||||
|
Outcome llm.AttemptOutcome
|
||||||
|
Dispatched bool
|
||||||
|
CapacityWait time.Duration
|
||||||
|
RequestDuration time.Duration
|
||||||
|
StartedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordAttempt runs after the attempt has released its permit, including when
|
||||||
|
// the recorder calls application code. Injected clients expose no HTTP facts.
|
||||||
|
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
|
||||||
|
outcome := result.Outcome
|
||||||
|
failure := ClassifyFailure(outcome.Err)
|
||||||
|
if capture {
|
||||||
|
attempt := AttemptEvidence{
|
||||||
|
StartedAt: result.StartedAt.UTC(), Dispatched: result.Dispatched,
|
||||||
|
CapacityWait: result.CapacityWait, RequestDuration: result.RequestDuration,
|
||||||
|
Failure: failure,
|
||||||
|
}
|
||||||
|
if injected {
|
||||||
|
attempt.ResponseReceived = outcome.Response != nil
|
||||||
|
attempt.GenerationCompleted = outcome.Err == nil && outcome.Response != nil
|
||||||
|
if result.Dispatched {
|
||||||
|
attempt.CapacityWait = 0
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
attempt.ResponseReceived = outcome.Evidence.ResponseReceived
|
||||||
|
attempt.HTTP = cloneHTTPEvidence(outcome.Evidence)
|
||||||
|
attempt.ResponseDecoded = outcome.ResponseDecoded
|
||||||
|
attempt.GenerationCompleted = outcome.GenerationCompleted
|
||||||
|
}
|
||||||
|
if outcome.StructuredOutputRequested != nil {
|
||||||
|
requested := *outcome.StructuredOutputRequested
|
||||||
|
attempt.StructuredOutputRequested = &requested
|
||||||
|
}
|
||||||
|
if result.Dispatched {
|
||||||
|
usage.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
var single observation.UsageAccumulator
|
||||||
|
single.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
attempt.Usage = single.Observed()
|
||||||
|
}
|
||||||
|
evidence.Attempts = append(evidence.Attempts, attempt)
|
||||||
|
}
|
||||||
|
if recorder != nil {
|
||||||
|
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||||
|
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
|
||||||
|
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
|
||||||
|
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) attemptResult {
|
||||||
|
result := attemptResult{StartedAt: c.now()}
|
||||||
|
release, err := c.manager.Acquire(ctx, backendID)
|
||||||
|
result.CapacityWait = c.capacityWait(backendID, result.StartedAt)
|
||||||
|
if err != nil {
|
||||||
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
dispatchStart := c.now()
|
dispatchStart := c.now()
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
outcome = prepared.ExecuteWithEvidence(ctx)
|
result.Outcome = prepared.ExecuteWithEvidence(ctx)
|
||||||
dispatched = outcome.Evidence.DispatchInvoked
|
result.Dispatched = result.Outcome.Evidence.DispatchInvoked
|
||||||
} else {
|
} else {
|
||||||
outcome = prepared.Execute(ctx)
|
result.Outcome = prepared.Execute(ctx)
|
||||||
dispatched = true
|
result.Dispatched = true
|
||||||
}
|
}
|
||||||
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt
|
result.RequestDuration = elapsed(dispatchStart, c.now())
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
||||||
|
|||||||
@@ -40,10 +40,7 @@ func testCoordinator(t *testing.T, transport transportFunc) *Coordinator {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
coordinator := NewBuiltIn(manager, client)
|
coordinator := NewBuiltIn(manager, client)
|
||||||
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
||||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
// GenerationEvidence contains invocation-local request evidence without raw
|
// GenerationEvidence contains invocation-local request evidence without raw
|
||||||
// provider bodies, headers, credentials, or generated content.
|
// provider bodies, headers, credentials, or generated content.
|
||||||
type GenerationEvidence struct {
|
type GenerationEvidence struct {
|
||||||
|
// PreparationFailed identifies a built-in request rejected before attempts.
|
||||||
|
PreparationFailed bool
|
||||||
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
||||||
// Injected generation leaves it unknown.
|
// Injected generation leaves it unknown.
|
||||||
InitialEndpoint string
|
InitialEndpoint string
|
||||||
|
|||||||
@@ -144,6 +144,32 @@ func (c fixedClient) Generate(context.Context, domain.GenerateRequest) (*domain.
|
|||||||
return c.response, c.err
|
return c.response, c.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInjectedObserverRunsAfterPermitRelease(t *testing.T) {
|
||||||
|
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP request"); return nil, nil })
|
||||||
|
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{}})
|
||||||
|
called := false
|
||||||
|
recorder, err := observation.New(func(event domain.ExecutionEvent) {
|
||||||
|
if event.Kind != domain.ExecutionAttempt {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
called = true
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
release, err := coordinator.manager.Acquire(ctx, "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("observer could not acquire permit: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := coordinator.Generate(observation.WithRecorder(context.Background(), recorder), testRequest(0)); err != nil || !called {
|
||||||
|
t.Fatalf("generation error = %v, observer called = %v", err, called)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
||||||
manager, err := capacity.NewManager(nil)
|
manager, err := capacity.NewManager(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
97
internal/jsonvalue/trusted.go
Normal file
97
internal/jsonvalue/trusted.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import "reflect"
|
||||||
|
|
||||||
|
func cloneTrustedMap(src map[string]any) map[string]any {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = CloneTrusted(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneTrusted detaches a previously accepted value without revalidating it.
|
||||||
|
func CloneTrusted(value any) any {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch v := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return cloneTrustedMap(v)
|
||||||
|
case []any:
|
||||||
|
out := make([]any, len(v))
|
||||||
|
for i, item := range v {
|
||||||
|
out[i] = CloneTrusted(item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case []string:
|
||||||
|
if v == nil {
|
||||||
|
return []string(nil)
|
||||||
|
}
|
||||||
|
return append([]string{}, v...)
|
||||||
|
case []byte:
|
||||||
|
if v == nil {
|
||||||
|
return []byte(nil)
|
||||||
|
}
|
||||||
|
return append([]byte{}, v...)
|
||||||
|
default:
|
||||||
|
return cloneTrustedReflect(reflect.ValueOf(value)).Interface()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneTrustedReflect(value reflect.Value) reflect.Value {
|
||||||
|
if !value.IsValid() {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Interface:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
copied := cloneTrustedReflect(value.Elem())
|
||||||
|
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||||
|
return copied
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
out.Set(copied)
|
||||||
|
return out
|
||||||
|
case reflect.Pointer:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type().Elem())
|
||||||
|
out.Elem().Set(cloneTrustedReflect(value.Elem()))
|
||||||
|
return out
|
||||||
|
case reflect.Map:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||||
|
iter := value.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
out.SetMapIndex(cloneTrustedReflect(iter.Key()), cloneTrustedReflect(iter.Value()))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Slice:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Array:
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
27
internal/jsonvalue/trusted_test.go
Normal file
27
internal/jsonvalue/trusted_test.go
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type trustedNamed string
|
||||||
|
|
||||||
|
func TestCloneTrustedPreservesTypesAndDetaches(t *testing.T) {
|
||||||
|
named := trustedNamed("value")
|
||||||
|
original := map[string]any{"": map[string]any{"number": json.Number("9007199254740993")}, "typed": []string{"original"}, "pointer": &named, "nilmap": map[string]any(nil), "nilslice": []string(nil), "empty": []any{}}
|
||||||
|
copy := CloneTrusted(original).(map[string]any)
|
||||||
|
if !reflect.DeepEqual(copy, original) {
|
||||||
|
t.Fatalf("clone = %#v", copy)
|
||||||
|
}
|
||||||
|
copy[""].(map[string]any)["number"] = json.Number("1")
|
||||||
|
copy["typed"].([]string)[0] = "changed"
|
||||||
|
*copy["pointer"].(*trustedNamed) = "changed"
|
||||||
|
if original[""].(map[string]any)["number"] != json.Number("9007199254740993") || original["typed"].([]string)[0] != "original" || named != "value" {
|
||||||
|
t.Fatal("clone shares mutable storage")
|
||||||
|
}
|
||||||
|
if copy["nilmap"].(map[string]any) != nil || copy["nilslice"].([]string) != nil || copy["empty"].([]any) == nil {
|
||||||
|
t.Fatal("nil/empty shape changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,10 +25,7 @@ func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
|
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
|
||||||
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
|
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -88,7 +85,7 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
||||||
var authorization string
|
var authorization string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
authorization = request.Header.Get("Authorization")
|
authorization = request.Header.Get("Authorization")
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -98,9 +95,6 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, evidence, err := client.GetWithAuthenticationEvidence(
|
response, evidence, err := client.GetWithAuthenticationEvidence(
|
||||||
context.Background(), "https://example.invalid/key", 0, tc.target,
|
context.Background(), "https://example.invalid/key", 0, tc.target,
|
||||||
@@ -269,10 +263,7 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
|
|||||||
|
|
||||||
transport := http.DefaultTransport
|
transport := http.DefaultTransport
|
||||||
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
|
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
const calls = 8
|
const calls = 8
|
||||||
evidences := make([]HTTPExecutionEvidence, calls)
|
evidences := make([]HTTPExecutionEvidence, calls)
|
||||||
@@ -307,15 +298,12 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
|
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
|
||||||
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
<-request.Context().Done()
|
<-request.Context().Done()
|
||||||
return nil, request.Context().Err()
|
return nil, request.Context().Err()
|
||||||
}),
|
}),
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
|
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
|
||||||
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
|
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
|||||||
@@ -17,10 +17,10 @@ import (
|
|||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
|
||||||
ErrInvalidRequest = errors.New("invalid generate request")
|
ErrInvalidRequest = errors.New("invalid generate request")
|
||||||
ErrRequestFailed = errors.New("llm request failed")
|
ErrRequestFailed = errors.New("llm request failed")
|
||||||
ErrTransport = errors.New("llm transport failed")
|
ErrTransport = errors.New("llm transport failed")
|
||||||
@@ -43,28 +43,15 @@ func (e *requestFailedError) Unwrap() []error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type OpenAICompatibleConfig struct {
|
type OpenAICompatibleConfig struct {
|
||||||
BaseURL string
|
|
||||||
Model string
|
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
type OpenAICompatibleClient struct {
|
type OpenAICompatibleClient struct {
|
||||||
baseURL string
|
httpClient *http.Client
|
||||||
defaultModel string
|
|
||||||
httpClient *http.Client
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) *OpenAICompatibleClient {
|
||||||
baseURL := ""
|
|
||||||
if strings.TrimSpace(cfg.BaseURL) != "" {
|
|
||||||
var err error
|
|
||||||
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout := cfg.Timeout
|
timeout := cfg.Timeout
|
||||||
if timeout <= 0 {
|
if timeout <= 0 {
|
||||||
timeout = defaults.LLMRequestTimeoutDefault
|
timeout = defaults.LLMRequestTimeoutDefault
|
||||||
@@ -82,10 +69,8 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &OpenAICompatibleClient{
|
return &OpenAICompatibleClient{
|
||||||
baseURL: baseURL,
|
httpClient: client,
|
||||||
defaultModel: cfg.Model,
|
}
|
||||||
httpClient: client,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
@@ -132,11 +117,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
selectedEndpoint := req.Target.Endpoint
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(req.Target.Endpoint)
|
||||||
if strings.TrimSpace(selectedEndpoint) == "" {
|
|
||||||
selectedEndpoint = c.baseURL
|
|
||||||
}
|
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -145,12 +126,10 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
// Custom JSON encoders may mutate their receivers. Both payload encoding
|
||||||
if err != nil {
|
// and encoding-failure diagnostics must use values detached from metadata.
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
req.Target.ExtraParams = jsonvalue.CloneTrusted(req.Target.ExtraParams).(map[string]any)
|
||||||
}
|
wirePayload, err := openAIChatRequestPayload(req)
|
||||||
|
|
||||||
wirePayload, err := openAIChatRequestPayload(wireReq)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -179,7 +158,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return &PreparedGeneration{
|
return &PreparedGeneration{
|
||||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||||
structuredOutputRequested: wireReq.ResponseFormat != nil,
|
structuredOutputRequested: wirePayload["response_format"] != nil,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -370,40 +349,69 @@ func tokenCount(value *int) int {
|
|||||||
return *value
|
return *value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type responseErrorReader struct {
|
||||||
|
reader io.Reader
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *responseErrorReader) Read(p []byte) (int, error) {
|
||||||
|
n, err := r.reader.Read(p)
|
||||||
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
r.err = err
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||||
|
observed := &responseErrorReader{reader: body}
|
||||||
limited := &io.LimitedReader{
|
limited := &io.LimitedReader{
|
||||||
R: body,
|
R: observed,
|
||||||
N: maxOpenAIChatResponseBytes + 1,
|
N: maxOpenAIChatResponseBytes + 1,
|
||||||
}
|
}
|
||||||
decoder := json.NewDecoder(limited)
|
decoder := json.NewDecoder(limited)
|
||||||
|
|
||||||
var response openAIChatResponse
|
var response openAIChatResponse
|
||||||
if err := decoder.Decode(&response); err != nil {
|
if err := decoder.Decode(&response); err != nil {
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
}
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
|
||||||
var trailing any
|
var trailing any
|
||||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
}
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
|
|
||||||
return response, nil
|
return response, nil
|
||||||
}
|
}
|
||||||
@@ -416,99 +424,52 @@ func openAIChatResponseTooLargeError() error {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
func openAIChatRequestPayload(req domain.GenerateRequest) (map[string]any, error) {
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
model := strings.TrimSpace(req.Target.Model)
|
||||||
if model == "" {
|
if model == "" {
|
||||||
model = strings.TrimSpace(defaultModel)
|
return nil, errors.New("model is required")
|
||||||
}
|
|
||||||
if model == "" {
|
|
||||||
return openAIChatRequest{}, errors.New("model is required")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
wireReq := openAIChatRequest{
|
|
||||||
Model: model,
|
|
||||||
}
|
|
||||||
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return openAIChatRequest{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
wireReq.SessionID = sessionID
|
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
messages := make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||||
for _, msg := range req.Prompt.Messages {
|
for _, msg := range req.Prompt.Messages {
|
||||||
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
messages = append(messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
out := map[string]any{"model": model, "messages": messages}
|
||||||
|
if sessionID != "" {
|
||||||
|
out["session_id"] = sessionID
|
||||||
|
}
|
||||||
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
||||||
wireReq.Temperature = &req.Target.Temperature
|
out["temperature"] = req.Target.Temperature
|
||||||
}
|
}
|
||||||
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
||||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
out["max_tokens"] = req.Target.MaxTokens
|
||||||
}
|
}
|
||||||
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
||||||
wireReq.TopP = &req.Target.TopP
|
out["top_p"] = req.Target.TopP
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
||||||
wireReq.ServiceTier = req.Target.ServiceTier
|
out["service_tier"] = req.Target.ServiceTier
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
||||||
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
out["reasoning_effort"] = req.Target.ReasoningEffort
|
||||||
}
|
|
||||||
if len(req.Target.ExtraParams) > 0 {
|
|
||||||
wireReq.ExtraParams = req.Target.ExtraParams
|
|
||||||
}
|
}
|
||||||
if req.StructuredOutput != nil {
|
if req.StructuredOutput != nil {
|
||||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return openAIChatRequest{}, err
|
return nil, err
|
||||||
|
}
|
||||||
|
if responseFormat != nil {
|
||||||
|
out["response_format"] = responseFormat
|
||||||
}
|
}
|
||||||
wireReq.ResponseFormat = responseFormat
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return wireReq, nil
|
for key, value := range req.Target.ExtraParams {
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
|
||||||
Model string `json:"model"`
|
|
||||||
SessionID string `json:"session_id,omitempty"`
|
|
||||||
Messages []openAIChatRequestMessage `json:"messages"`
|
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
||||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
|
||||||
ExtraParams map[string]any `json:"-"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|
||||||
out := map[string]any{
|
|
||||||
"model": req.Model,
|
|
||||||
"messages": req.Messages,
|
|
||||||
}
|
|
||||||
if req.SessionID != "" {
|
|
||||||
out["session_id"] = req.SessionID
|
|
||||||
}
|
|
||||||
if req.Temperature != nil {
|
|
||||||
out["temperature"] = *req.Temperature
|
|
||||||
}
|
|
||||||
if req.MaxTokens != nil {
|
|
||||||
out["max_tokens"] = *req.MaxTokens
|
|
||||||
}
|
|
||||||
if req.TopP != nil {
|
|
||||||
out["top_p"] = *req.TopP
|
|
||||||
}
|
|
||||||
if req.ServiceTier != "" {
|
|
||||||
out["service_tier"] = req.ServiceTier
|
|
||||||
}
|
|
||||||
if req.ReasoningEffort != "" {
|
|
||||||
out["reasoning_effort"] = req.ReasoningEffort
|
|
||||||
}
|
|
||||||
if req.ResponseFormat != nil {
|
|
||||||
out["response_format"] = req.ResponseFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
for key, value := range req.ExtraParams {
|
|
||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, errors.New("extra_params key must not be empty")
|
return nil, errors.New("extra_params key must not be empty")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -123,26 +123,19 @@ func (p *recordingProvider) lastRequest(t *testing.T) recordedProviderRequest {
|
|||||||
return p.requests[len(p.requests)-1]
|
return p.requests[len(p.requests)-1]
|
||||||
}
|
}
|
||||||
|
|
||||||
func newProviderClient(t *testing.T, provider *recordingProvider, config OpenAICompatibleConfig) *OpenAICompatibleClient {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if config.BaseURL == "" {
|
|
||||||
config.BaseURL = provider.endpoint("/v1")
|
|
||||||
}
|
|
||||||
client, err := NewOpenAICompatibleClient(config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
|
|
||||||
func ordinaryGenerateRequest() domain.GenerateRequest {
|
func ordinaryGenerateRequest() domain.GenerateRequest {
|
||||||
return domain.GenerateRequest{
|
return domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "model"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *recordingProvider) generateRequest() domain.GenerateRequest {
|
||||||
|
req := ordinaryGenerateRequest()
|
||||||
|
req.Target.Endpoint = p.endpoint("/v1")
|
||||||
|
return req
|
||||||
|
}
|
||||||
|
|
||||||
type deadlineCapturingTransport struct {
|
type deadlineCapturingTransport struct {
|
||||||
deadline time.Time
|
deadline time.Time
|
||||||
hasDeadline bool
|
hasDeadline bool
|
||||||
@@ -253,35 +246,13 @@ func assertDeadlineNear(t *testing.T, deadline, before, after time.Time, duratio
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewOpenAICompatibleClientRejectsInvalidBaseURL(t *testing.T) {
|
|
||||||
for _, endpoint := range []string{
|
|
||||||
"://invalid",
|
|
||||||
"/v1",
|
|
||||||
"https:///v1",
|
|
||||||
"ftp://provider.example/v1",
|
|
||||||
"https://user@provider.example/v1",
|
|
||||||
"https://provider.example/v1?mode=chat",
|
|
||||||
"https://provider.example/v1#chat",
|
|
||||||
} {
|
|
||||||
t.Run(endpoint, func(t *testing.T) {
|
|
||||||
_, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: endpoint})
|
|
||||||
if !errors.Is(err, ErrInvalidConfig) {
|
|
||||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
|
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
|
||||||
transport := http.DefaultTransport
|
transport := http.DefaultTransport
|
||||||
supplied := &http.Client{Transport: transport}
|
supplied := &http.Client{Transport: transport}
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != 0 {
|
if supplied.Timeout != 0 {
|
||||||
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
|
||||||
@@ -305,13 +276,10 @@ func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *t
|
|||||||
Transport: transport,
|
Transport: transport,
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: 2 * time.Second,
|
Timeout: 2 * time.Second,
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != suppliedTimeout {
|
if supplied.Timeout != suppliedTimeout {
|
||||||
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
|
||||||
@@ -335,13 +303,10 @@ func TestNewOpenAICompatibleClientTreatsSuppliedNegativeTimeoutAsUnset(t *testin
|
|||||||
}
|
}
|
||||||
configuredTimeout := 23 * time.Second
|
configuredTimeout := 23 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: configuredTimeout,
|
Timeout: configuredTimeout,
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != -time.Second {
|
if supplied.Timeout != -time.Second {
|
||||||
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
|
||||||
@@ -370,7 +335,6 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
|||||||
{name: "optional response fields", run: checkOptionalResponseFieldOmission},
|
{name: "optional response fields", run: checkOptionalResponseFieldOmission},
|
||||||
{name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping},
|
{name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping},
|
||||||
{name: "optional request field presence", run: checkRequestFieldPresence},
|
{name: "optional request field presence", run: checkRequestFieldPresence},
|
||||||
{name: "configured model fallback", run: checkConfiguredModelFallback},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -380,14 +344,14 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
|||||||
|
|
||||||
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
|
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
const content = " exact content\nwith whitespace "
|
const content = " exact content\nwith whitespace "
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||||
Role: domain.RoleDeveloper,
|
Role: domain.RoleDeveloper,
|
||||||
Content: content,
|
Content: content,
|
||||||
}}},
|
}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Generate() error = %v", err)
|
t.Fatalf("Generate() error = %v", err)
|
||||||
@@ -413,7 +377,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
|
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
|
||||||
}`)
|
}`)
|
||||||
|
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: 2 * time.Second,
|
Timeout: 2 * time.Second,
|
||||||
})
|
})
|
||||||
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key")
|
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key")
|
||||||
@@ -424,6 +388,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
{Role: "user", Content: "Say hello"},
|
{Role: "user", Content: "Say hello"},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: provider.endpoint("/v1"),
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
Temperature: 0.4,
|
Temperature: 0.4,
|
||||||
MaxTokens: 123,
|
MaxTokens: 123,
|
||||||
@@ -594,9 +559,11 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
tc.configureEnv(t)
|
tc.configureEnv(t)
|
||||||
}
|
}
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Target = tc.target
|
request.Target = tc.target
|
||||||
|
request.Target.Endpoint = provider.endpoint("/v1")
|
||||||
|
request.Target.Model = "model"
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), request)
|
_, err := client.Generate(context.Background(), request)
|
||||||
if tc.wantError != nil {
|
if tc.wantError != nil {
|
||||||
@@ -625,7 +592,7 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
|
|
||||||
func checkCacheControlledMessageMapping(t *testing.T) {
|
func checkCacheControlledMessageMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
@@ -639,7 +606,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{Role: "user", Content: "Dynamic request."},
|
{Role: "user", Content: "Dynamic request."},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -691,7 +658,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
|
|||||||
|
|
||||||
func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
@@ -703,7 +670,7 @@ func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -745,8 +712,8 @@ func checkSessionIDMapping(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Prompt.SessionID = tc.sessionID
|
request.Prompt.SessionID = tc.sessionID
|
||||||
|
|
||||||
if _, err := client.Generate(context.Background(), request); err != nil {
|
if _, err := client.Generate(context.Background(), request); err != nil {
|
||||||
@@ -766,19 +733,14 @@ func checkSessionIDMapping(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "model",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{
|
Prompt: domain.RenderedPrompt{
|
||||||
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
},
|
},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected invalid request error")
|
t.Fatal("expected invalid request error")
|
||||||
@@ -828,9 +790,9 @@ func checkContentPresence(t *testing.T) {
|
|||||||
"cache_write_tokens": 5
|
"cache_write_tokens": 5
|
||||||
}
|
}
|
||||||
}`)
|
}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
response, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
}
|
}
|
||||||
@@ -865,10 +827,11 @@ func checkCacheUsageMapping(t *testing.T) {
|
|||||||
"cache_write_tokens": 60
|
"cache_write_tokens": 60
|
||||||
}
|
}
|
||||||
}`)
|
}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -901,8 +864,8 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
|
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
resp, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
resp, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -913,17 +876,17 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
|
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
if resp, err := client.Generate(context.Background(), ordinaryGenerateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
|
if resp, err := client.Generate(context.Background(), provider.generateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
|
||||||
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
|
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
_, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -938,11 +901,12 @@ func checkOptionalResponseFieldOmission(t *testing.T) {
|
|||||||
|
|
||||||
func checkReasoningAndExtraParameterMapping(t *testing.T) {
|
func checkReasoningAndExtraParameterMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: provider.endpoint("/v1"),
|
||||||
Model: "model",
|
Model: "model",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: "high",
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
@@ -1020,8 +984,8 @@ func checkRequestFieldPresence(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.TargetPresence = tc.presence
|
request.TargetPresence = tc.presence
|
||||||
|
|
||||||
if _, err := client.Generate(context.Background(), request); err != nil {
|
if _, err := client.Generate(context.Background(), request); err != nil {
|
||||||
@@ -1073,21 +1037,17 @@ func checkOmittedGenerationTimeout(t *testing.T) {
|
|||||||
transport := &deadlineCapturingTransport{}
|
transport := &deadlineCapturingTransport{}
|
||||||
clientTimeout := 5 * time.Second
|
clientTimeout := 5 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Timeout: clientTimeout,
|
Timeout: clientTimeout,
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model", TimeoutSeconds: 0},
|
||||||
})
|
})
|
||||||
after := time.Now()
|
after := time.Now()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -1146,11 +1106,11 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model", ExtraParams: tc.extraParams},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected invalid request error")
|
t.Fatal("expected invalid request error")
|
||||||
@@ -1192,8 +1152,8 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Target.ExtraParams = tc.extraParams
|
request.Target.ExtraParams = tc.extraParams
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
@@ -1218,30 +1178,12 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkConfiguredModelFallback(t *testing.T) {
|
|
||||||
provider := newRecordingProvider(t)
|
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
var model string
|
|
||||||
if !provider.lastRequest(t).decodeField(t, "model", &model) || model != "default-model" {
|
|
||||||
t.Fatalf("model = %q, want default-model", model)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
run func(*testing.T)
|
run func(*testing.T)
|
||||||
}{
|
}{
|
||||||
{name: "request endpoint overrides configured endpoint", run: checkEndpointOverride},
|
{name: "request endpoint", run: checkSelectedEndpoint},
|
||||||
{name: "request endpoint works without configured endpoint", run: checkEmptyConfiguredEndpoint},
|
|
||||||
{name: "completion URL preserves valid base paths", run: checkCompletionURLComposition},
|
{name: "completion URL preserves valid base paths", run: checkCompletionURLComposition},
|
||||||
{name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection},
|
{name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection},
|
||||||
{name: "an endpoint is required", run: checkRequiredEndpoint},
|
{name: "an endpoint is required", run: checkRequiredEndpoint},
|
||||||
@@ -1252,35 +1194,6 @@ func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkEndpointOverride(t *testing.T) {
|
|
||||||
configuredProvider := newRecordingProvider(t)
|
|
||||||
configuredProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"default"}}]}`)
|
|
||||||
selectedProvider := newRecordingProvider(t)
|
|
||||||
selectedProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"override"}}]}`)
|
|
||||||
|
|
||||||
client := newProviderClient(t, configuredProvider, OpenAICompatibleConfig{Model: "m"})
|
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{Endpoint: selectedProvider.endpoint("/v1")},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if resp.Content != "override" {
|
|
||||||
t.Fatalf("expected override response, got %q", resp.Content)
|
|
||||||
}
|
|
||||||
if got := configuredProvider.requestCount(); got != 0 {
|
|
||||||
t.Fatalf("configured provider calls = %d, want 0", got)
|
|
||||||
}
|
|
||||||
if got := selectedProvider.requestCount(); got != 1 {
|
|
||||||
t.Fatalf("selected provider calls = %d, want 1", got)
|
|
||||||
}
|
|
||||||
if got := selectedProvider.lastRequest(t).url; got != "/v1/chat/completions" {
|
|
||||||
t.Fatalf("selected URL = %q, want /v1/chat/completions", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkCommonResponseFailures(t *testing.T) {
|
func checkCommonResponseFailures(t *testing.T) {
|
||||||
const sensitiveBody = `provider-secret-fragment request_payload_details`
|
const sensitiveBody = `provider-secret-fragment request_payload_details`
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -1310,9 +1223,9 @@ func checkCommonResponseFailures(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(tc.statusCode, tc.body)
|
provider.respond(tc.statusCode, tc.body)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "m"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
response, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if response != nil {
|
if response != nil {
|
||||||
t.Fatalf("expected no partial response, got %+v", response)
|
t.Fatalf("expected no partial response, got %+v", response)
|
||||||
}
|
}
|
||||||
@@ -1357,9 +1270,7 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
body := sizedSuccessResponseBody(tc.size)
|
body := sizedSuccessResponseBody(tc.size)
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1370,12 +1281,10 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if tc.wantErr {
|
if tc.wantErr {
|
||||||
if response != nil {
|
if response != nil {
|
||||||
@@ -1425,9 +1334,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
strings.NewReader(prefix),
|
strings.NewReader(prefix),
|
||||||
continuation,
|
continuation,
|
||||||
)}
|
)}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1438,9 +1345,6 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type outcome struct {
|
type outcome struct {
|
||||||
response *domain.GenerateResponse
|
response *domain.GenerateResponse
|
||||||
@@ -1450,6 +1354,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
go func() {
|
go func() {
|
||||||
response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{
|
response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
done <- outcome{response: response, err: generateErr}
|
done <- outcome{response: response, err: generateErr}
|
||||||
}()
|
}()
|
||||||
@@ -1497,9 +1402,7 @@ func checkSingleResponseDocument(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
body := &countingReadCloser{reader: strings.NewReader(tc.body)}
|
body := &countingReadCloser{reader: strings.NewReader(tc.body)}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1510,12 +1413,10 @@ func checkSingleResponseDocument(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if tc.wantErr {
|
if tc.wantErr {
|
||||||
if response != nil {
|
if response != nil {
|
||||||
@@ -1541,22 +1442,19 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
|
|||||||
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
||||||
generationTimeout := 2 * time.Second
|
generationTimeout := 2 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: 10 * time.Second,
|
Timeout: 10 * time.Second,
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://example.com/v1",
|
||||||
|
Model: "m",
|
||||||
TimeoutSeconds: int(generationTimeout / time.Second),
|
TimeoutSeconds: int(generationTimeout / time.Second),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -1578,25 +1476,22 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
|
|||||||
|
|
||||||
func checkCallerDeadlinePrecedence(t *testing.T) {
|
func checkCallerDeadlinePrecedence(t *testing.T) {
|
||||||
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: 10 * time.Second,
|
Timeout: 10 * time.Second,
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
callerDeadline := time.Now().Add(time.Second)
|
callerDeadline := time.Now().Add(time.Second)
|
||||||
ctx, cancel := context.WithDeadline(context.Background(), callerDeadline)
|
ctx, cancel := context.WithDeadline(context.Background(), callerDeadline)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://example.com/v1",
|
||||||
|
Model: "m",
|
||||||
TimeoutSeconds: 2,
|
TimeoutSeconds: 2,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -1618,22 +1513,18 @@ func checkCallerDeadlinePrecedence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkCallerCancellationIdentity(t *testing.T) {
|
func checkCallerCancellationIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: contextErrorTransport{},
|
Transport: contextErrorTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected cancellation error")
|
t.Fatal("expected cancellation error")
|
||||||
@@ -1647,21 +1538,17 @@ func checkCallerCancellationIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: contextErrorTransport{},
|
Transport: contextErrorTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0))
|
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0))
|
||||||
defer cancel()
|
defer cancel()
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
t.Fatalf("expected request failure and caller deadline identities, got %v", err)
|
t.Fatalf("expected request failure and caller deadline identities, got %v", err)
|
||||||
@@ -1669,20 +1556,16 @@ func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkWholeRequestTimeoutIdentity(t *testing.T) {
|
func checkWholeRequestTimeoutIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: time.Millisecond,
|
Timeout: time.Millisecond,
|
||||||
Transport: waitingContextTransport{},
|
Transport: waitingContextTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
t.Fatalf("expected request failure and client timeout identities, got %v", err)
|
t.Fatalf("expected request failure and client timeout identities, got %v", err)
|
||||||
@@ -1696,20 +1579,15 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
|
|||||||
apiKey = "sensitive-api-key"
|
apiKey = "sensitive-api-key"
|
||||||
content = "sensitive prompt content"
|
content = "sensitive prompt content"
|
||||||
)
|
)
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: endpoint,
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return nil, transportCause
|
return nil, transportCause
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}},
|
||||||
Target: domain.ExecutionTarget{APIKey: apiKey},
|
Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m", APIKey: apiKey},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) {
|
||||||
t.Fatalf("expected request failure and transport cause identities, got %v", err)
|
t.Fatalf("expected request failure and transport cause identities, got %v", err)
|
||||||
@@ -1726,13 +1604,7 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type testCase struct {
|
type testCase struct {
|
||||||
name string
|
name string
|
||||||
@@ -1765,11 +1637,9 @@ func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkEmptyConfiguredEndpoint(t *testing.T) {
|
func checkSelectedEndpoint(t *testing.T) {
|
||||||
var selectedURL string
|
var selectedURL string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
selectedURL = req.URL.String()
|
selectedURL = req.URL.String()
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -1782,13 +1652,10 @@ func checkEmptyConfiguredEndpoint(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected empty configured base URL to be allowed, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1"},
|
Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate with request endpoint: %v", err)
|
t.Fatalf("generate with request endpoint: %v", err)
|
||||||
@@ -1817,9 +1684,7 @@ func checkCompletionURLComposition(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
var selectedURL string
|
var selectedURL string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: tc.baseURL,
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
selectedURL = req.URL.String()
|
selectedURL = req.URL.String()
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -1830,12 +1695,10 @@ func checkCompletionURLComposition(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: tc.baseURL, Model: "m"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
@@ -1860,21 +1723,16 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
|
|||||||
for _, endpoint := range invalidEndpoints {
|
for _, endpoint := range invalidEndpoints {
|
||||||
t.Run(endpoint, func(t *testing.T) {
|
t.Run(endpoint, func(t *testing.T) {
|
||||||
transportCalls := 0
|
transportCalls := 0
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://configured.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
transportCalls++
|
transportCalls++
|
||||||
return nil, errors.New("transport must not be called")
|
return nil, errors.New("transport must not be called")
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Endpoint: endpoint},
|
Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
@@ -1890,17 +1748,11 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkRequiredEndpoint(t *testing.T) {
|
func checkRequiredEndpoint(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{},
|
Target: domain.ExecutionTarget{Model: "m"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected endpoint-required error")
|
t.Fatal("expected endpoint-required error")
|
||||||
@@ -1909,3 +1761,13 @@ func checkRequiredEndpoint(t *testing.T) {
|
|||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientRequiresModel(t *testing.T) {
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
_, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1"},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("missing model error = %v, want invalid request", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
108
internal/llm/openrouter_authentication.go
Normal file
108
internal/llm/openrouter_authentication.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxAuthenticationResponseBytes = 1 << 20
|
||||||
|
|
||||||
|
// AuthenticationEvidence describes the observable result of a key probe.
|
||||||
|
type AuthenticationEvidence struct {
|
||||||
|
HTTP HTTPExecutionEvidence
|
||||||
|
Accepted *bool
|
||||||
|
Overflow bool
|
||||||
|
Unrecognized bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRouterKeyEndpoint derives the default key endpoint from a base endpoint.
|
||||||
|
func OpenRouterKeyEndpoint(base string) (string, error) {
|
||||||
|
return url.JoinPath(base, "key")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProbeOpenRouterKey executes and interprets an OpenRouter key request.
|
||||||
|
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
|
||||||
|
parsed, err := url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return AuthenticationEvidence{}, err
|
||||||
|
}
|
||||||
|
parsed.User = nil
|
||||||
|
response, evidence, requestErr := c.GetWithAuthenticationEvidence(ctx, parsed.String(), timeoutSeconds, target)
|
||||||
|
result := AuthenticationEvidence{HTTP: evidence}
|
||||||
|
if response != nil && response.Body != nil {
|
||||||
|
defer response.Body.Close()
|
||||||
|
}
|
||||||
|
if !evidence.ResponseReceived && response == nil {
|
||||||
|
return result, requestErr
|
||||||
|
}
|
||||||
|
status := evidence.HTTPStatus
|
||||||
|
if status == nil && response != nil {
|
||||||
|
status = &response.StatusCode
|
||||||
|
}
|
||||||
|
if status == nil {
|
||||||
|
result.Unrecognized = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
switch *status {
|
||||||
|
case http.StatusUnauthorized:
|
||||||
|
accepted := false
|
||||||
|
result.Accepted = &accepted
|
||||||
|
case http.StatusOK:
|
||||||
|
if response == nil || response.Body == nil {
|
||||||
|
result.Unrecognized = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
contents, err := io.ReadAll(io.LimitReader(response.Body, maxAuthenticationResponseBytes+1))
|
||||||
|
if err != nil && ctx.Err() != nil {
|
||||||
|
return result, ctx.Err()
|
||||||
|
}
|
||||||
|
if len(contents) > maxAuthenticationResponseBytes {
|
||||||
|
result.Overflow = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
if recognizedOpenRouterKeyResponse(contents) {
|
||||||
|
accepted := true
|
||||||
|
result.Accepted = &accepted
|
||||||
|
} else {
|
||||||
|
result.Unrecognized = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recognizedOpenRouterKeyResponse(contents []byte) bool {
|
||||||
|
var envelope map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var data map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(envelope["data"], &data); err != nil || data == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
label := bytes.TrimSpace(data["label"])
|
||||||
|
var labelValue string
|
||||||
|
if len(label) == 0 || label[0] != '"' || json.Unmarshal(label, &labelValue) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
freeTier := bytes.TrimSpace(data["is_free_tier"])
|
||||||
|
if !bytes.Equal(freeTier, []byte("true")) && !bytes.Equal(freeTier, []byte("false")) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data["usage"]))
|
||||||
|
decoder.UseNumber()
|
||||||
|
var number any
|
||||||
|
if decoder.Decode(&number) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, ok := number.(json.Number)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
134
internal/llm/openrouter_authentication_test.go
Normal file
134
internal/llm/openrouter_authentication_test.go
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKey(t *testing.T) {
|
||||||
|
valid := `{"data":{"label":"x","is_free_tier":false,"usage":0}}`
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
body string
|
||||||
|
key string
|
||||||
|
accepted *bool
|
||||||
|
unrecognized, overflow bool
|
||||||
|
}{
|
||||||
|
{name: "recognized", status: 200, body: valid, key: "secret", accepted: boolValue(true)},
|
||||||
|
{name: "exact limit", status: 200, body: valid + strings.Repeat(" ", maxAuthenticationResponseBytes-len(valid)), accepted: boolValue(true)},
|
||||||
|
{name: "unknown", status: 200, body: `{"data":{"label":"x","is_free_tier":false,"usage":"0"}}`, unrecognized: true},
|
||||||
|
{name: "trailing value", status: 200, body: valid + ` {}`, unrecognized: true},
|
||||||
|
{name: "overflow", status: 200, body: strings.Repeat("x", maxAuthenticationResponseBytes+1), overflow: true},
|
||||||
|
{name: "unauthorized anonymous", status: 401, accepted: boolValue(false)},
|
||||||
|
{name: "other status", status: 503},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var authorization string
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
if req.URL.User != nil {
|
||||||
|
t.Fatal("request retained URL credentials")
|
||||||
|
}
|
||||||
|
authorization = req.Header.Get("Authorization")
|
||||||
|
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.HTTP.Endpoint != "https://example.invalid/key" || got.HTTP.HTTPStatus == nil || *got.HTTP.HTTPStatus != tc.status || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow {
|
||||||
|
t.Fatalf("evidence = %+v", got)
|
||||||
|
}
|
||||||
|
if (got.Accepted == nil) != (tc.accepted == nil) || got.Accepted != nil && *got.Accepted != *tc.accepted {
|
||||||
|
t.Fatalf("acceptance = %v", got.Accepted)
|
||||||
|
}
|
||||||
|
if (authorization != "") != (tc.key != "") || got.HTTP.CredentialsSupplied != (tc.key != "") {
|
||||||
|
t.Fatalf("credential evidence = %+v", got.HTTP)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolValue(value bool) *bool { return &value }
|
||||||
|
|
||||||
|
type authenticationFailingBody struct{ err error }
|
||||||
|
|
||||||
|
func (b authenticationFailingBody) Read([]byte) (int, error) { return 0, b.err }
|
||||||
|
func (b authenticationFailingBody) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
|
||||||
|
want := errors.New("interrupted")
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
|
||||||
|
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type observedAuthenticationBody struct {
|
||||||
|
reader io.Reader
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *observedAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n, err := b.reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *observedAuthenticationBody) Close() error {
|
||||||
|
b.closed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
wantRead int
|
||||||
|
}{
|
||||||
|
{"overflow", http.StatusOK, maxAuthenticationResponseBytes + 1},
|
||||||
|
{"unauthorized", http.StatusUnauthorized, 0},
|
||||||
|
{"other status", http.StatusServiceUnavailable, 0},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
body := &observedAuthenticationBody{reader: strings.NewReader(strings.Repeat("x", 2*maxAuthenticationResponseBytes))}
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
|
||||||
|
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenRouterKeyEndpoint(t *testing.T) {
|
||||||
|
got, err := OpenRouterKeyEndpoint("https://example.invalid/api/v1/")
|
||||||
|
if err != nil || got != "https://example.invalid/api/v1/key" {
|
||||||
|
t.Fatalf("endpoint = %q, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
<-req.Context().Done()
|
||||||
|
return nil, req.Context().Err()
|
||||||
|
})}})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
|
||||||
|
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,13 +12,7 @@ import (
|
|||||||
var prepareGenerationBenchmarkSink *PreparedGeneration
|
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||||
|
|
||||||
func BenchmarkPrepareGeneration(b *testing.B) {
|
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "https://api.example.test/v1",
|
|
||||||
Model: "benchmark-model",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("construct benchmark client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prompt := domain.RenderedPrompt{
|
prompt := domain.RenderedPrompt{
|
||||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||||
@@ -31,14 +25,14 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "no_extras",
|
name: "no_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "small_extras",
|
name: "small_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"priority": "low",
|
"priority": "low",
|
||||||
"attempt": 3,
|
"attempt": 3,
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
@@ -49,7 +43,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "1MiB_string_extra",
|
name: "1MiB_string_extra",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"payload": strings.Repeat("x", 1<<20),
|
"payload": strings.Repeat("x", 1<<20),
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
@@ -58,7 +52,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "nested_json_extras",
|
name: "nested_json_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"options": map[string]any{
|
"options": map[string]any{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"limits": []any{2, 4, 8},
|
"limits": []any{2, 4, 8},
|
||||||
@@ -71,7 +65,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "structured_output",
|
name: "structured_output",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
|
||||||
StructuredOutput: &domain.StructuredOutputSpec{
|
StructuredOutput: &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
|||||||
@@ -73,9 +73,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
response := responseFor(request, test.status, "")
|
response := responseFor(request, test.status, "")
|
||||||
response.Body = io.NopCloser(strings.NewReader(test.body))
|
response.Body = io.NopCloser(strings.NewReader(test.body))
|
||||||
@@ -83,9 +81,6 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
return response, nil
|
return response, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
request := ordinaryGenerateRequest()
|
request := ordinaryGenerateRequest()
|
||||||
if test.structured {
|
if test.structured {
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
@@ -136,9 +131,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
|
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
|
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -149,9 +142,6 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
|
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -174,16 +164,11 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
|
|||||||
|
|
||||||
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
|
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
|
||||||
transportErr := errors.New("provider unavailable")
|
transportErr := errors.New("provider unavailable")
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return nil, transportErr
|
return nil, transportErr
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
request := ordinaryGenerateRequest()
|
request := ordinaryGenerateRequest()
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
|||||||
@@ -26,8 +26,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
|
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
|
||||||
t.Setenv(envName, tc.initialKey)
|
t.Setenv(envName, tc.initialKey)
|
||||||
var bodies, auth, urls []string
|
var bodies, auth, urls []string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "default-model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
body, err := io.ReadAll(req.Body)
|
body, err := io.ReadAll(req.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -39,12 +38,9 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req := domain.GenerateRequest{
|
req := domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
|
||||||
Target: domain.ExecutionTarget{APIKeyEnv: envName},
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "default-model", APIKeyEnv: envName},
|
||||||
}
|
}
|
||||||
prepared, err := client.PrepareGeneration(req)
|
prepared, err := client.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -68,8 +64,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
|
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
|
||||||
var contexts []context.Context
|
var contexts []context.Context
|
||||||
var closed int
|
var closed int
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
if err := req.Context().Err(); err != nil {
|
if err := req.Context().Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -81,12 +76,9 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
|
|||||||
}}, nil
|
}}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{TimeoutSeconds: 1},
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model", TimeoutSeconds: 1},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -115,17 +107,14 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -137,17 +126,14 @@ func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -160,17 +146,14 @@ func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
|||||||
|
|
||||||
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
|
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
|
||||||
closed := false
|
closed := false
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -127,9 +127,7 @@ func TestOpenAICompatibleClientNonSuccessBodyOwnership(t *testing.T) {
|
|||||||
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: statusCode,
|
StatusCode: statusCode,
|
||||||
@@ -138,8 +136,5 @@ func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
return client
|
return client
|
||||||
}
|
}
|
||||||
|
|||||||
48
internal/llm/response_read_error_test.go
Normal file
48
internal/llm/response_read_error_test.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type responseBytesAndError struct {
|
||||||
|
data []byte
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *responseBytesAndError) Read(p []byte) (int, error) {
|
||||||
|
if len(r.data) == 0 {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
n := copy(p, r.data)
|
||||||
|
r.data = r.data[n:]
|
||||||
|
if len(r.data) == 0 {
|
||||||
|
return n, r.err
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSuccessfulResponseReadFailuresRetainCause(t *testing.T) {
|
||||||
|
cause := errors.New("private reader diagnostic")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, prefix string
|
||||||
|
together bool
|
||||||
|
}{
|
||||||
|
{"partial", `{"choices":[`, false},
|
||||||
|
{"complete then failure", successfulProviderResponse, false},
|
||||||
|
{"bytes and error together", successfulProviderResponse, true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var reader io.Reader = io.MultiReader(strings.NewReader(test.prefix), failingReader{err: cause})
|
||||||
|
if test.together {
|
||||||
|
reader = &responseBytesAndError{data: []byte(test.prefix), err: cause}
|
||||||
|
}
|
||||||
|
_, err := decodeOpenAIChatResponse(reader)
|
||||||
|
if !errors.Is(err, ErrTransport) || !errors.Is(err, cause) || errors.Is(err, ErrMalformedResponse) || strings.Contains(err.Error(), cause.Error()) {
|
||||||
|
t.Fatalf("read failure = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,4 +13,5 @@ var ErrProfileNotFound = errors.New("execution profile not found")
|
|||||||
// Repository selects execution profiles from a captured index.
|
// Repository selects execution profiles from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
||||||
|
GetProfileIdentity(ctx context.Context, id string) (string, error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,10 +24,6 @@ type goRenderer struct {
|
|||||||
templates TemplateLookup
|
templates TemplateLookup
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGoRenderer() Renderer {
|
|
||||||
return &goRenderer{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewCompiledRenderer renders definitions using templates captured with the
|
// NewCompiledRenderer renders definitions using templates captured with the
|
||||||
// configuration snapshot. Per-call clones bind request-specific input helpers.
|
// configuration snapshot. Per-call clones bind request-specific input helpers.
|
||||||
func NewCompiledRenderer(templates TemplateLookup) Renderer {
|
func NewCompiledRenderer(templates TemplateLookup) Renderer {
|
||||||
@@ -56,13 +52,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
funcs := template.FuncMap{
|
funcs := template.FuncMap{
|
||||||
"input": resolver.resolve,
|
"input": resolver.resolve,
|
||||||
}
|
}
|
||||||
var compiled CompiledTemplates
|
if r.templates == nil {
|
||||||
if r.templates != nil {
|
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
||||||
var found bool
|
}
|
||||||
compiled, found = r.templates.GetCompiledTemplates(definition.ID)
|
compiled, found := r.templates.GetCompiledTemplates(definition.ID)
|
||||||
if !found || len(compiled.Messages) != len(definition.Templates) {
|
if !found || len(compiled.Messages) != len(definition.Templates) {
|
||||||
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -70,15 +65,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}
|
}
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var err error
|
var err error
|
||||||
if r.templates != nil {
|
if definition.SessionID != "" {
|
||||||
if definition.SessionID != "" {
|
if compiled.Session == nil {
|
||||||
if compiled.Session == nil {
|
return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
|
||||||
return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
|
|
||||||
}
|
|
||||||
sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
|
|
||||||
}
|
}
|
||||||
} else {
|
sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
|
||||||
sessionID, err = renderSessionID(ctx, definition.SessionID, funcs, vars)
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -102,18 +93,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
var tmpl *template.Template
|
if compiled.Messages[i] == nil {
|
||||||
var parseErr error
|
return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure)
|
||||||
if r.templates != nil {
|
}
|
||||||
if compiled.Messages[i] == nil {
|
tmpl, parseErr := compiled.Messages[i].Clone()
|
||||||
return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure)
|
if parseErr == nil {
|
||||||
}
|
tmpl.Funcs(funcs)
|
||||||
tmpl, parseErr = compiled.Messages[i].Clone()
|
|
||||||
if parseErr == nil {
|
|
||||||
tmpl.Funcs(funcs)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
tmpl, parseErr = parseTemplate(fmt.Sprintf("msg_%d", i), tmplMsg.Content, funcs)
|
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -201,20 +186,6 @@ func (r *artifactTextResolver) resolve(name string) (string, error) {
|
|||||||
return text, nil
|
return text, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func renderSessionID(ctx context.Context, raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
tmpl, parseErr := parseTemplate("session_id", raw, funcs)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if parseErr != nil {
|
|
||||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, parseErr)
|
|
||||||
}
|
|
||||||
return executeSessionTemplate(ctx, tmpl, vars)
|
|
||||||
}
|
|
||||||
|
|
||||||
func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) {
|
func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestGoRenderer_Render(t *testing.T) {
|
func TestGoRenderer_Render(t *testing.T) {
|
||||||
renderer := NewGoRenderer()
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
inputs := map[string]*domain.Artifact{
|
inputs := map[string]*domain.Artifact{
|
||||||
@@ -33,7 +32,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -49,11 +48,11 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "user", Content: "From file: {{input \"transcript\"}}", ContentFile: "/tmp/user.tmpl"},
|
{Role: "user", Content: "From file: {{input \"transcript\"}}"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -71,7 +70,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -99,7 +98,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -129,7 +128,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -151,7 +150,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -169,7 +168,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, map[string]string{
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": "agent-session-123",
|
"session_id": "agent-session-123",
|
||||||
})
|
})
|
||||||
@@ -190,7 +189,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -208,7 +207,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -223,7 +222,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
})
|
})
|
||||||
@@ -241,7 +240,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": "session" + string([]byte{0xff}),
|
"session_id": "session" + string([]byte{0xff}),
|
||||||
})
|
})
|
||||||
@@ -258,7 +257,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -278,7 +277,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -298,7 +297,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -315,7 +314,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, map[string]*domain.Artifact{}, vars)
|
||||||
if !errors.Is(err, ErrMissingRequiredInput) {
|
if !errors.Is(err, ErrMissingRequiredInput) {
|
||||||
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
|
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -329,9 +328,9 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := CompileTemplate("message", def.Templates[0].Content)
|
||||||
if !errors.Is(err, ErrInvalidTemplate) {
|
if err == nil {
|
||||||
t.Fatalf("expected ErrInvalidTemplate, got %v", err)
|
t.Fatal("expected template compilation to fail")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -343,7 +342,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -359,7 +358,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
{Role: "", Content: "Hello"},
|
{Role: "", Content: "Hello"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
if !errors.Is(err, ErrInvalidMessageRole) {
|
||||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -371,7 +370,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
|
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
res, err := renderer.Render(ctx, def, nil, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -385,7 +384,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
|
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
|
||||||
}
|
}
|
||||||
_, err := renderer.Render(ctx, def, nil, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
|
||||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
if !errors.Is(err, ErrInvalidMessageRole) {
|
||||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -396,9 +395,9 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGoRendererCancellation(t *testing.T) {
|
func TestGoRendererCancellation(t *testing.T) {
|
||||||
t.Run("before session parsing", func(t *testing.T) {
|
t.Run("before rendering", func(t *testing.T) {
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
SessionID: "{{ malformed",
|
SessionID: "{{.session}}",
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "user", Content: "not rendered"},
|
{Role: "user", Content: "not rendered"},
|
||||||
},
|
},
|
||||||
@@ -406,18 +405,11 @@ func TestGoRendererCancellation(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
result, err := renderer.Render(ctx, definition, nil, nil)
|
||||||
if result != nil || !errors.Is(err, context.Canceled) {
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
}
|
}
|
||||||
if errors.Is(err, ErrInvalidTemplate) {
|
|
||||||
t.Fatalf("pre-canceled render parsed the malformed session: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil)
|
|
||||||
if result != nil || !errors.Is(err, ErrInvalidTemplate) {
|
|
||||||
t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("during artifact text conversion", func(t *testing.T) {
|
t.Run("during artifact text conversion", func(t *testing.T) {
|
||||||
@@ -447,14 +439,15 @@ func TestGoRendererCancellation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
counter := &checkCountingContext{Context: context.Background()}
|
counter := &checkCountingContext{Context: context.Background()}
|
||||||
if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil {
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
if _, err := renderer.Render(counter, definition, nil, nil); err != nil {
|
||||||
t.Fatalf("count render checkpoints: %v", err)
|
t.Fatalf("count render checkpoints: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The final three checks occur after template execution, after the
|
// The final three checks occur after template execution, after the
|
||||||
// message is assembled, and immediately before publication.
|
// message is assembled, and immediately before publication.
|
||||||
ctx := newCancelOnCheckContext(counter.checks - 2)
|
ctx := newCancelOnCheckContext(counter.checks - 2)
|
||||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
result, err := renderer.Render(ctx, definition, nil, nil)
|
||||||
if result != nil || !errors.Is(err, context.Canceled) {
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
}
|
}
|
||||||
@@ -472,7 +465,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
first, err := renderer.Render(context.Background(), definition, inputs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("first render: %v", err)
|
t.Fatalf("first render: %v", err)
|
||||||
}
|
}
|
||||||
@@ -488,7 +482,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
||||||
t.Fatal("completed render aliases the artifact body")
|
t.Fatal("completed render aliases the artifact body")
|
||||||
}
|
}
|
||||||
second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
second, err := renderer.Render(context.Background(), definition, inputs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("second render: %v", err)
|
t.Fatalf("second render: %v", err)
|
||||||
}
|
}
|
||||||
@@ -498,7 +492,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
nilInputs := map[string]*domain.Artifact{"document": nil}
|
nilInputs := map[string]*domain.Artifact{"document": nil}
|
||||||
result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil)
|
result, err := renderer.Render(context.Background(), definition, nilInputs, nil)
|
||||||
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
||||||
}
|
}
|
||||||
@@ -559,11 +553,56 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCompiledRendererRequiresCapturedTemplates(t *testing.T) {
|
||||||
|
message, err := CompileTemplate("message", "hello")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
lookup TemplateLookup
|
||||||
|
sessionID string
|
||||||
|
}{
|
||||||
|
{name: "missing lookup"},
|
||||||
|
{name: "missing prompt", lookup: compiledRendererTestLookup{id: "other"}},
|
||||||
|
{name: "missing message", lookup: compiledRendererTestLookup{id: "prompt"}},
|
||||||
|
{name: "nil message", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{nil}}}},
|
||||||
|
{name: "missing session", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{message}}}, sessionID: "session"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{ID: "prompt", SessionID: tc.sessionID, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hello"}}}
|
||||||
|
if _, err := NewCompiledRenderer(tc.lookup).Render(context.Background(), definition, nil, nil); !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("render error = %v, want render failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type compiledRendererTestLookup struct {
|
type compiledRendererTestLookup struct {
|
||||||
id string
|
id string
|
||||||
templates CompiledTemplates
|
templates CompiledTemplates
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func capturedTestRenderer(t testing.TB, definition *domain.PromptDefinition) Renderer {
|
||||||
|
t.Helper()
|
||||||
|
var compiled CompiledTemplates
|
||||||
|
var err error
|
||||||
|
if definition.SessionID != "" {
|
||||||
|
compiled.Session, err = CompileTemplate("session", definition.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, message := range definition.Templates {
|
||||||
|
parsed, err := CompileTemplate("message", message.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
compiled.Messages = append(compiled.Messages, parsed)
|
||||||
|
}
|
||||||
|
return NewCompiledRenderer(compiledRendererTestLookup{id: definition.ID, templates: compiled})
|
||||||
|
}
|
||||||
|
|
||||||
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
||||||
if id != lookup.id {
|
if id != lookup.id {
|
||||||
return CompiledTemplates{}, false
|
return CompiledTemplates{}, false
|
||||||
@@ -571,8 +610,7 @@ func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (Compil
|
|||||||
return lookup.templates, true
|
return lookup.templates, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
|
// BenchmarkGoRendererArtifactReferences measures rendering captured templates.
|
||||||
// template-parsing fallback rather than the captured compiled path.
|
|
||||||
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||||
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||||
inputs := map[string]*domain.Artifact{
|
inputs := map[string]*domain.Artifact{
|
||||||
@@ -604,7 +642,7 @@ func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
b.Run(tc.name, func(b *testing.B) {
|
b.Run(tc.name, func(b *testing.B) {
|
||||||
renderer := NewGoRenderer()
|
renderer := capturedTestRenderer(b, tc.definition)
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.SetBytes(int64(len(body)))
|
b.SetBytes(int64(len(body)))
|
||||||
for range b.N {
|
for range b.N {
|
||||||
|
|||||||
@@ -319,24 +319,27 @@ func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluat
|
|||||||
result := conditionEvaluation{
|
result := conditionEvaluation{
|
||||||
unsupported: left.unsupported || right.unsupported,
|
unsupported: left.unsupported || right.unsupported,
|
||||||
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
||||||
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
|
missingVariables: mergeOwnedNames(left.missingVariables, right.missingVariables),
|
||||||
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
|
contentInputs: mergeOwnedNames(left.contentInputs, right.contentInputs),
|
||||||
}
|
|
||||||
for name := range left.missingVariables {
|
|
||||||
result.missingVariables[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range right.missingVariables {
|
|
||||||
result.missingVariables[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range left.contentInputs {
|
|
||||||
result.contentInputs[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range right.contentInputs {
|
|
||||||
result.contentInputs[name] = struct{}{}
|
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Condition results own their sets; callers retain operand values and flags,
|
||||||
|
// not independent snapshots of these maps after a merge.
|
||||||
|
func mergeOwnedNames(left, right map[string]struct{}) map[string]struct{} {
|
||||||
|
if len(left) < len(right) {
|
||||||
|
left, right = right, left
|
||||||
|
}
|
||||||
|
if left == nil {
|
||||||
|
return right
|
||||||
|
}
|
||||||
|
for name := range right {
|
||||||
|
left[name] = struct{}{}
|
||||||
|
}
|
||||||
|
return left
|
||||||
|
}
|
||||||
|
|
||||||
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
||||||
if parent == requirementInactive {
|
if parent == requirementInactive {
|
||||||
return requirementInactive
|
return requirementInactive
|
||||||
@@ -448,6 +451,34 @@ func (walker *requirementWalker) walkTemplate(compiled *template.Template) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) visitBranch(branch *parse.BranchNode, reach requirementReachability, dot dotIdentity, changesDot bool) error {
|
||||||
|
condition := conditionEvaluation{}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if err := walker.visit(branch.Pipe, requirementInactive, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
condition = walker.evaluateCondition(branch.Pipe, reach, dot)
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
bodyDot := dot
|
||||||
|
if changesDot {
|
||||||
|
bodyDot = walker.withDotIdentity(branch.Pipe, dot, condition)
|
||||||
|
}
|
||||||
|
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||||
|
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||||
|
if err := walker.visit(branch.List, thenReach, bodyDot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := walker.visit(branch.ElseList, elseReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if reach != requirementInactive && !condition.known && walker.dependencies != before {
|
||||||
|
walker.recordConditionLimitation(condition)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
||||||
if err := walker.ctx.Err(); err != nil {
|
if err := walker.ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -468,55 +499,9 @@ func (walker *requirementWalker) visit(node parse.Node, reach requirementReachab
|
|||||||
case *parse.ActionNode:
|
case *parse.ActionNode:
|
||||||
return walker.visit(node.Pipe, reach, dot)
|
return walker.visit(node.Pipe, reach, dot)
|
||||||
case *parse.IfNode:
|
case *parse.IfNode:
|
||||||
if reach == requirementInactive {
|
return walker.visitBranch(&node.BranchNode, reach, dot, false)
|
||||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
|
||||||
}
|
|
||||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
|
||||||
before := walker.dependencies
|
|
||||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
|
||||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
|
||||||
if err := walker.visit(node.List, thenReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !condition.known && walker.dependencies != before {
|
|
||||||
walker.recordConditionLimitation(condition)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case *parse.WithNode:
|
case *parse.WithNode:
|
||||||
if reach == requirementInactive {
|
return walker.visitBranch(&node.BranchNode, reach, dot, true)
|
||||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
|
|
||||||
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
|
||||||
}
|
|
||||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
|
||||||
before := walker.dependencies
|
|
||||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
|
||||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
|
||||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
|
|
||||||
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !condition.known && walker.dependencies != before {
|
|
||||||
walker.recordConditionLimitation(condition)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case *parse.RangeNode:
|
case *parse.RangeNode:
|
||||||
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -314,7 +314,7 @@ func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
|
|||||||
"active": {Name: "active", Body: []byte("active")},
|
"active": {Name: "active", Body: []byte("active")},
|
||||||
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
||||||
}
|
}
|
||||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -406,6 +406,35 @@ func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *tes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConditionEvaluationMergesOwnedEvidence(t *testing.T) {
|
||||||
|
left := conditionEvaluation{missingVariables: map[string]struct{}{"a": {}, "b": {}}, contentInputs: map[string]struct{}{"document": {}}, unsupported: true}
|
||||||
|
right := conditionEvaluation{missingVariables: map[string]struct{}{"b": {}, "c": {}}, inputNameUnknown: true}
|
||||||
|
merged := mergeConditionEvaluations(left, right)
|
||||||
|
if !merged.unsupported || !merged.inputNameUnknown || len(merged.missingVariables) != 3 || len(merged.contentInputs) != 1 {
|
||||||
|
t.Fatalf("merged evidence = %+v", merged)
|
||||||
|
}
|
||||||
|
transferred := mergeConditionEvaluations(conditionEvaluation{}, conditionEvaluation{missingVariables: map[string]struct{}{"only": {}}})
|
||||||
|
if len(transferred.missingVariables) != 1 || transferred.contentInputs != nil {
|
||||||
|
t.Fatalf("transferred evidence = %+v", transferred)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsRetainsLeftHeavyMissingConditions(t *testing.T) {
|
||||||
|
compiled := compileRequirementTemplate(t, "nested", `{{if eq (eq (eq .a .b) .c) .d}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{compiled}}}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Variables) != 4 || len(result.Inputs) != 2 {
|
||||||
|
t.Fatalf("requirements = %+v", result)
|
||||||
|
}
|
||||||
|
for i, name := range []string{"a", "b", "c", "d"} {
|
||||||
|
if result.Variables[i].Name != name {
|
||||||
|
t.Fatalf("variables = %+v", result.Variables)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
||||||
description := "declared for a reason"
|
description := "declared for a reason"
|
||||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||||
@@ -497,7 +526,7 @@ func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
|
|||||||
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
||||||
}
|
}
|
||||||
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
||||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/prompt"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
|
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
|
||||||
@@ -12,5 +13,8 @@ var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
|||||||
|
|
||||||
// Repository selects prompt definitions from a captured index.
|
// Repository selects prompt definitions from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error)
|
GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
|
||||||
|
// GetPromptInspection supplies lightweight requirement-selection metadata.
|
||||||
|
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
|
||||||
|
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
||||||
}
|
}
|
||||||
|
|||||||
38
internal/usecase/operation_error.go
Normal file
38
internal/usecase/operation_error.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OperationError identifies the operation that owns a failure while retaining
|
||||||
|
// the original cause for inspection.
|
||||||
|
type OperationError struct {
|
||||||
|
Category error
|
||||||
|
Cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *OperationError) Error() string { return fmt.Sprintf("%v: %v", e.Category, e.Cause) }
|
||||||
|
func (e *OperationError) Unwrap() error { return e.Cause }
|
||||||
|
func (e *OperationError) Is(target error) bool { return target == e.Category }
|
||||||
|
|
||||||
|
func promptLookupError(cause error) error {
|
||||||
|
category := ErrPromptLoad
|
||||||
|
if errors.Is(cause, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
category = promptdef.ErrPromptDefinitionNotFound
|
||||||
|
cause = fmt.Errorf("%w: %w", ErrPromptLoad, cause)
|
||||||
|
}
|
||||||
|
return &OperationError{Category: category, Cause: cause}
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileLookupError(cause error) error {
|
||||||
|
category := ErrProfileLoad
|
||||||
|
if errors.Is(cause, profile.ErrProfileNotFound) {
|
||||||
|
category = profile.ErrProfileNotFound
|
||||||
|
cause = fmt.Errorf("%w: %w", ErrProfileLoad, cause)
|
||||||
|
}
|
||||||
|
return &OperationError{Category: category, Cause: cause}
|
||||||
|
}
|
||||||
34
internal/usecase/operation_error_test.go
Normal file
34
internal/usecase/operation_error_test.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOperationErrorRetainsOwnerAndCause(t *testing.T) {
|
||||||
|
cause := fmt.Errorf("nested: %w", promptdef.ErrPromptDefinitionNotFound)
|
||||||
|
outer := &OperationError{Category: ErrLLMGenerate, Cause: cause}
|
||||||
|
if !errors.Is(outer, ErrLLMGenerate) || !errors.Is(outer, promptdef.ErrPromptDefinitionNotFound) || errors.Unwrap(outer) != cause {
|
||||||
|
t.Fatalf("operation error = %v", outer)
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
err error
|
||||||
|
category error
|
||||||
|
}{
|
||||||
|
{promptLookupError(cause), promptdef.ErrPromptDefinitionNotFound},
|
||||||
|
{profileLookupError(profile.ErrProfileNotFound), profile.ErrProfileNotFound},
|
||||||
|
{promptLookupError(contextFailure), ErrPromptLoad},
|
||||||
|
{profileLookupError(contextFailure), ErrProfileLoad},
|
||||||
|
} {
|
||||||
|
var operation *OperationError
|
||||||
|
if !errors.As(test.err, &operation) || operation.Category != test.category {
|
||||||
|
t.Errorf("operation = %+v", operation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var contextFailure = errors.New("lookup failed")
|
||||||
@@ -172,14 +172,13 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
|
|||||||
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
||||||
collaborators.validator.prepareCalls != 0 || collaborators.validator.directValidateCalls != 0 ||
|
collaborators.validator.prepareCalls != 0 ||
|
||||||
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d validation=%d admissions=%d generation=%d",
|
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d admissions=%d generation=%d",
|
||||||
collaborators.artifacts.calls,
|
collaborators.artifacts.calls,
|
||||||
collaborators.renderer.calls,
|
collaborators.renderer.calls,
|
||||||
collaborators.validator.prepareCalls,
|
collaborators.validator.prepareCalls,
|
||||||
collaborators.validator.directValidateCalls,
|
|
||||||
len(collaborators.admitter.backendIDs),
|
len(collaborators.admitter.backendIDs),
|
||||||
collaborators.llm.calls,
|
collaborators.llm.calls,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
@@ -22,7 +21,7 @@ const (
|
|||||||
|
|
||||||
// PreparedExecution owns one frozen, single-use runner execution.
|
// PreparedExecution owns one frozen, single-use runner execution.
|
||||||
type PreparedExecution struct {
|
type PreparedExecution struct {
|
||||||
owner *Runner
|
owner *runnerIdentity
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
state preparedExecutionState
|
state preparedExecutionState
|
||||||
details *domain.PreparedRun
|
details *domain.PreparedRun
|
||||||
@@ -50,24 +49,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
executionSnapshot := operation.run
|
|
||||||
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
|
|
||||||
|
|
||||||
details, err := clonePreparedRun(executionSnapshot)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &PreparedExecution{
|
return &PreparedExecution{
|
||||||
owner: r,
|
owner: r.identity,
|
||||||
state: preparedExecutionReady,
|
state: preparedExecutionReady,
|
||||||
details: details,
|
details: operation.run,
|
||||||
payload: &preparedExecutionPayload{
|
payload: &preparedExecutionPayload{
|
||||||
prepared: executionSnapshot,
|
prepared: operation.run,
|
||||||
validation: operation.validation,
|
validation: operation.validation,
|
||||||
directKey: state.effectiveModel.APIKey,
|
directKey: state.effectiveModel.APIKey,
|
||||||
transientRetries: state.transientRetries,
|
transientRetries: state.transientRetries,
|
||||||
@@ -76,7 +63,8 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Details returns a fresh credential-redacted copy of the prepared run.
|
// Details returns immutable, credential-redacted metadata borrowed from the
|
||||||
|
// handle. Callers must copy it before exposing it to mutable external code.
|
||||||
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -85,11 +73,7 @@ func (p *PreparedExecution) Details() *domain.PreparedRun {
|
|||||||
detailsSnapshot := p.details
|
detailsSnapshot := p.details
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
details, err := clonePreparedRun(detailsSnapshot)
|
return detailsSnapshot
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return details
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Discard invalidates an unclaimed execution and drops its private payload.
|
// Discard invalidates an unclaimed execution and drops its private payload.
|
||||||
@@ -170,7 +154,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
||||||
if p == nil || owner == nil || p.owner != owner {
|
if p == nil || owner == nil || owner.identity == nil || p.owner != owner.identity {
|
||||||
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,9 +173,6 @@ func (p *preparedExecutionPayload) clear() {
|
|||||||
if p == nil {
|
if p == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if p.prepared != nil {
|
|
||||||
p.prepared.EffectiveModelParams.APIKey = ""
|
|
||||||
}
|
|
||||||
p.prepared = nil
|
p.prepared = nil
|
||||||
p.validation = nil
|
p.validation = nil
|
||||||
p.directKey = ""
|
p.directKey = ""
|
||||||
@@ -222,43 +203,3 @@ func (p noOpPreparedValidation) Validate(
|
|||||||
func (noOpPreparedValidation) SchemaDocument() any {
|
func (noOpPreparedValidation) SchemaDocument() any {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
|
||||||
if source == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
copied := *source
|
|
||||||
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
copied.EffectiveModelParams.ExtraParams = extraParams
|
|
||||||
|
|
||||||
if source.InputHashes != nil {
|
|
||||||
copied.InputHashes = make(map[string]string, len(source.InputHashes))
|
|
||||||
for name, hash := range source.InputHashes {
|
|
||||||
copied.InputHashes[name] = hash
|
|
||||||
}
|
|
||||||
}
|
|
||||||
copied.Messages = domain.CloneRenderedMessages(source.Messages)
|
|
||||||
|
|
||||||
if source.StructuredOutput != nil {
|
|
||||||
structuredOutput := *source.StructuredOutput
|
|
||||||
copied.StructuredOutput = &structuredOutput
|
|
||||||
if source.StructuredOutput.JSONSchema != nil {
|
|
||||||
jsonSchema := *source.StructuredOutput.JSONSchema
|
|
||||||
copied.StructuredOutput.JSONSchema = &jsonSchema
|
|
||||||
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
|
|
||||||
if copyErr != nil {
|
|
||||||
return nil, copyErr
|
|
||||||
}
|
|
||||||
copied.StructuredOutput.JSONSchema.Schema = schema
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &copied, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneJSONValue(source any) (any, error) {
|
|
||||||
return jsonvalue.Copy(source)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
@@ -47,25 +48,15 @@ func (p *recordingPreparedValidation) SchemaDocument() any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type recordingValidationPreparer struct {
|
type recordingValidationPreparer struct {
|
||||||
plan *recordingPreparedValidation
|
plan *recordingPreparedValidation
|
||||||
prepareErr error
|
prepareErr error
|
||||||
prepareCalls int
|
prepareCalls int
|
||||||
directValidateCalls int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type validationOnly struct{}
|
type nilPlanPreparer struct{}
|
||||||
|
|
||||||
func (validationOnly) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
|
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||||
return domain.ValidationResult{}, nil
|
return nil, nil
|
||||||
}
|
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) Validate(
|
|
||||||
context.Context,
|
|
||||||
*domain.Artifact,
|
|
||||||
domain.OutputContract,
|
|
||||||
) (domain.ValidationResult, error) {
|
|
||||||
v.directValidateCalls++
|
|
||||||
return domain.ValidationResult{}, errors.New("live validation must not be used")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) PrepareValidation(
|
func (v *recordingValidationPreparer) PrepareValidation(
|
||||||
@@ -129,12 +120,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
|||||||
}
|
}
|
||||||
defer prepared.Discard()
|
defer prepared.Discard()
|
||||||
|
|
||||||
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
|
if validator.prepareCalls != 1 {
|
||||||
t.Fatalf(
|
t.Fatalf("validation preparations = %d, want 1", validator.prepareCalls)
|
||||||
"validation calls=(prepare=%d direct=%d), want (1, 0)",
|
|
||||||
validator.prepareCalls,
|
|
||||||
validator.directValidateCalls,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
if reader.calls != 1 || renderer.calls != 1 {
|
if reader.calls != 1 || renderer.calls != 1 {
|
||||||
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
||||||
@@ -156,90 +143,37 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
|||||||
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
||||||
}
|
}
|
||||||
|
|
||||||
first.Messages[0].Content = "caller mutation"
|
|
||||||
first.InputHashes["input"] = "caller mutation"
|
|
||||||
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
|
|
||||||
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
|
|
||||||
renderer.rendered.Messages[0].Content = "source mutation"
|
renderer.rendered.Messages[0].Content = "source mutation"
|
||||||
|
|
||||||
second := prepared.Details()
|
second := prepared.Details()
|
||||||
if second.Messages[0].Content != "original message" ||
|
if second != first || second.Messages[0].Content != "original message" ||
|
||||||
second.InputHashes["input"] == "caller mutation" ||
|
|
||||||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||||
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
|
t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) {
|
func TestRunnerPreparationDetachesDeepPreparedSchema(t *testing.T) {
|
||||||
operations := []struct {
|
var nested any = "original"
|
||||||
name string
|
for range 150 {
|
||||||
run func(*Runner, domain.RunRequest) error
|
nested = map[string]any{"child": nested}
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "Prepare",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.Prepare(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Run",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.Run(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PrepareExecution",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.PrepareExecution(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
schema := map[string]any{"annotation": nested}
|
||||||
for _, operation := range operations {
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
t.Run(operation.name, func(t *testing.T) {
|
def.Validation.SchemaPath = "schema.json"
|
||||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{schemaDocument: schema}}
|
||||||
def.Validation.SchemaPath = "schema.json"
|
runner := NewRunner(&fakePromptRepo{def: def},
|
||||||
llmClient := &fakeLLM{forbid: true}
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
validator := &recordingValidationPreparer{
|
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, validator, nil)
|
||||||
plan: &recordingPreparedValidation{schemaDocument: excessivelyDeepPreparedJSONValue()},
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
}
|
if err != nil {
|
||||||
runner := NewRunner(
|
t.Fatalf("prepare execution: %v", err)
|
||||||
&fakePromptRepo{def: def},
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
||||||
|
|
||||||
defaultArtifactReader(),
|
|
||||||
defaultRenderer(),
|
|
||||||
llmClient,
|
|
||||||
validator,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
err := operation.run(runner, domain.RunRequest{
|
|
||||||
PromptID: "p",
|
|
||||||
ProfileID: "exec",
|
|
||||||
Inputs: singleInputRef(),
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrValidation) {
|
|
||||||
t.Fatalf("expected ErrValidation, got %v", err)
|
|
||||||
}
|
|
||||||
if llmClient.calls != 0 {
|
|
||||||
t.Fatalf("invalid prepared schema reached generation: %d calls", llmClient.calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
schema["annotation"].(map[string]any)["child"] = "source mutation"
|
||||||
|
first := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||||
func excessivelyDeepPreparedJSONValue() any {
|
if _, ok := first["annotation"].(map[string]any)["child"].(map[string]any); !ok {
|
||||||
const clearlyUnsafeContainerDepth = 1_000
|
t.Fatal("prepared schema aliases the validator document")
|
||||||
var value any = true
|
|
||||||
for level := 0; level < clearlyUnsafeContainerDepth; level++ {
|
|
||||||
value = map[string]any{"child": value}
|
|
||||||
}
|
}
|
||||||
return value
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
||||||
@@ -421,8 +355,8 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
|||||||
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
||||||
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
||||||
}
|
}
|
||||||
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
if repairer.calls != 1 {
|
||||||
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
t.Fatalf("repair calls = %d, want 1", repairer.calls)
|
||||||
}
|
}
|
||||||
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
|
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
|
||||||
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
|
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
|
||||||
@@ -498,7 +432,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
repairer: &fakeRepairer{err: repairInvalidRequest},
|
repairer: &fakeRepairer{err: repairInvalidRequest},
|
||||||
wantError: ErrInvalidRequest,
|
wantError: ErrLLMGenerate,
|
||||||
wantSource: repairInvalidRequest,
|
wantSource: repairInvalidRequest,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -600,6 +534,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
owner := newRunner()
|
owner := newRunner()
|
||||||
|
if other := newRunner(); owner.identity == nil || owner.identity == other.identity {
|
||||||
|
t.Fatal("constructed runners must have independent identity tokens")
|
||||||
|
}
|
||||||
prepared, err := owner.PrepareExecution(context.Background(), request)
|
prepared, err := owner.PrepareExecution(context.Background(), request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare execution: %v", err)
|
t.Fatalf("prepare execution: %v", err)
|
||||||
@@ -616,6 +553,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
if prepared.Details() == nil {
|
if prepared.Details() == nil {
|
||||||
t.Fatal("details unavailable after execution")
|
t.Fatal("details unavailable after execution")
|
||||||
}
|
}
|
||||||
|
if prepared.owner != owner.identity || prepared.payload != nil {
|
||||||
|
t.Fatal("claimed handle retained payload or lost owner identity")
|
||||||
|
}
|
||||||
|
|
||||||
discarded, err := owner.PrepareExecution(context.Background(), request)
|
discarded, err := owner.PrepareExecution(context.Background(), request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -623,6 +563,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
discarded.Discard()
|
discarded.Discard()
|
||||||
discarded.Discard()
|
discarded.Discard()
|
||||||
|
if discarded.owner != owner.identity || discarded.payload != nil {
|
||||||
|
t.Fatal("discarded handle retained payload or lost owner identity")
|
||||||
|
}
|
||||||
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
||||||
}
|
}
|
||||||
@@ -687,7 +630,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
wantLabel := "profile source"
|
wantLabel := "profile source"
|
||||||
if test.requestOverride {
|
if test.requestOverride {
|
||||||
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras}
|
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: jsonvalue.CloneTrusted(requestExtras).(map[string]any)}
|
||||||
wantLabel = "request source"
|
wantLabel = "request source"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -711,11 +654,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||||
t.Fatalf("prepared details retained a source alias: %#v", details)
|
t.Fatalf("prepared details retained a source alias: %#v", details)
|
||||||
}
|
}
|
||||||
details.Messages[0].Content = "mutated details"
|
|
||||||
if again := prepared.Details(); again == nil || len(again.Messages) != 1 || again.Messages[0].Content != "captured rendering" {
|
|
||||||
t.Fatalf("details mutation escaped into the retained snapshot: %+v", again)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
||||||
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
||||||
@@ -737,22 +675,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
|
|
||||||
profile := defaultExecutionProfile()
|
|
||||||
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
|
|
||||||
runner := NewRunner(
|
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
|
||||||
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
|
|
||||||
)
|
|
||||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
||||||
PromptID: "p", ProfileID: "exec",
|
|
||||||
})
|
|
||||||
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
|
|
||||||
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||||
@@ -782,7 +704,7 @@ func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
func TestRunnerPrepareExecutionRejectsNilValidationPlan(t *testing.T) {
|
||||||
reader := defaultArtifactReader()
|
reader := defaultArtifactReader()
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
@@ -791,7 +713,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
|||||||
reader,
|
reader,
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
validationOnly{},
|
nilPlanPreparer{},
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
38
internal/usecase/profile_fixture_test.go
Normal file
38
internal/usecase/profile_fixture_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
type inspectionProfileRepository struct {
|
||||||
|
profile *domain.ExecutionProfile
|
||||||
|
err error
|
||||||
|
calls int
|
||||||
|
id string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
r.calls++
|
||||||
|
r.id = id
|
||||||
|
if r.err != nil {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
if r.profile == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
profile := *r.profile
|
||||||
|
if profile.ResolvedTarget != nil {
|
||||||
|
target := *profile.ResolvedTarget
|
||||||
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
|
profile.ResolvedTarget = &target
|
||||||
|
}
|
||||||
|
return &profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionProfileRepository) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
_, err := r.GetProfile(ctx, id)
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type resolvedProfileSelection struct {
|
type resolvedProfileSelection struct {
|
||||||
@@ -15,10 +14,6 @@ type resolvedProfileSelection struct {
|
|||||||
profile *domain.ExecutionProfile
|
profile *domain.ExecutionProfile
|
||||||
}
|
}
|
||||||
|
|
||||||
type profileIdentityRepository interface {
|
|
||||||
GetProfileIdentity(context.Context, string) (string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
||||||
if strings.TrimSpace(profileID) == "" {
|
if strings.TrimSpace(profileID) == "" {
|
||||||
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||||
@@ -39,7 +34,7 @@ func (r *Runner) resolveProfileSelection(
|
|||||||
}
|
}
|
||||||
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, profileLookupError(err)
|
||||||
}
|
}
|
||||||
if selectedProfile == nil {
|
if selectedProfile == nil {
|
||||||
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
||||||
@@ -61,18 +56,11 @@ func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if indexed, ok := r.profiles.(profileIdentityRepository); ok {
|
identity, err := r.profiles.GetProfileIdentity(ctx, lookupID)
|
||||||
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
return identity, nil
|
|
||||||
}
|
|
||||||
selection, err := r.resolveProfileSelection(ctx, lookupID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", profileLookupError(err)
|
||||||
}
|
}
|
||||||
return selection.id, nil
|
return identity, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||||
@@ -97,36 +85,3 @@ func normalizeResolvedExecutionSettings(target domain.ExecutionTarget) (domain.E
|
|||||||
}
|
}
|
||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectProfile resolves one explicit profile without prompt or execution work.
|
|
||||||
func (r *Runner) InspectProfile(
|
|
||||||
ctx context.Context,
|
|
||||||
profileID string,
|
|
||||||
) (*domain.ProfileInspection, error) {
|
|
||||||
if strings.TrimSpace(profileID) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
selection, err := r.resolveProfileSelection(ctx, profileID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
target, _ := resolveExecutionTarget(selection.profile, nil)
|
|
||||||
target.APIKey = ""
|
|
||||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: profile extra parameters: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
target.ExtraParams = params
|
|
||||||
|
|
||||||
return &domain.ProfileInspection{
|
|
||||||
ProfileID: selection.id,
|
|
||||||
EffectiveModelParams: target,
|
|
||||||
APIKeyRequired: target.APIKeyRequired,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
package usecase
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/profile"
|
|
||||||
)
|
|
||||||
|
|
||||||
type inspectionProfileRepository struct {
|
|
||||||
profile *domain.ExecutionProfile
|
|
||||||
err error
|
|
||||||
calls int
|
|
||||||
id string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
r.calls++
|
|
||||||
r.id = id
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, r.err
|
|
||||||
}
|
|
||||||
return r.profile, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {
|
|
||||||
configured := domain.ExecutionTarget{
|
|
||||||
BackendID: "backend", Endpoint: "https://backend.example/v1", Model: "model",
|
|
||||||
Temperature: 0, APIKeyEnv: "OPTIONAL_KEY", ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
}
|
|
||||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
|
||||||
ID: " literal ", ResolvedTarget: &configured,
|
|
||||||
TargetPresence: domain.ExecutionTargetPresence{Temperature: true},
|
|
||||||
}}
|
|
||||||
runner := &Runner{profiles: profiles}
|
|
||||||
first, err := runner.InspectProfile(context.Background(), " literal ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect exact ID: %v", err)
|
|
||||||
}
|
|
||||||
if profiles.calls != 1 || profiles.id != " literal " || first.ProfileID != profiles.id || !reflect.DeepEqual(first.EffectiveModelParams, configured) {
|
|
||||||
t.Fatalf("inspection=%+v lookup=(%d,%q), want exact snapshot", first, profiles.calls, profiles.id)
|
|
||||||
}
|
|
||||||
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
second, err := runner.InspectProfile(context.Background(), " literal ")
|
|
||||||
if err != nil || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" {
|
|
||||||
t.Fatalf("inspection mutated snapshot: (%+v, %v)", second, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
|
|
||||||
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
|
|
||||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
|
||||||
ID: "endpoint-only", ResolvedTarget: &domain.ExecutionTarget{
|
|
||||||
Endpoint: "https://profile.example/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
|
|
||||||
},
|
|
||||||
}}
|
|
||||||
inspection, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "endpoint-only")
|
|
||||||
if err != nil || inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" || inspection.APIKeyRequired {
|
|
||||||
t.Fatalf("unexpected endpoint-only inspection: (%+v, %v)", inspection, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectProfileClassifiesLookupFailures(t *testing.T) {
|
|
||||||
profiles := &inspectionProfileRepository{}
|
|
||||||
runner := &Runner{profiles: profiles}
|
|
||||||
if _, err := runner.InspectProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
|
|
||||||
t.Fatalf("blank inspection=(%v, calls=%d)", err, profiles.calls)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
if _, err := runner.InspectProfile(ctx, "profile"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
|
|
||||||
t.Fatalf("canceled inspection=(%v, calls=%d)", err, profiles.calls)
|
|
||||||
}
|
|
||||||
profiles.err = profile.ErrProfileNotFound
|
|
||||||
if _, err := runner.InspectProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
|
|
||||||
t.Fatalf("missing profile error=%v", err)
|
|
||||||
}
|
|
||||||
for _, invalid := range []*Runner{
|
|
||||||
{},
|
|
||||||
{profiles: &inspectionProfileRepository{}},
|
|
||||||
{profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}}},
|
|
||||||
} {
|
|
||||||
if _, err := invalid.InspectProfile(context.Background(), "profile"); !errors.Is(err, ErrProfileLoad) {
|
|
||||||
t.Fatalf("invalid dependency error=%v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -13,10 +13,6 @@ type resolvedPromptDefinition struct {
|
|||||||
hash string
|
hash string
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptInspectionRepository interface {
|
|
||||||
GetPromptInspection(context.Context, string) (*domain.PromptInspection, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
|
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
|
||||||
if strings.TrimSpace(promptID) == "" {
|
if strings.TrimSpace(promptID) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
@@ -25,29 +21,14 @@ func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (
|
|||||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
|
|
||||||
var definition *domain.PromptDefinition
|
definition, hash, err := r.promptDefs.GetPromptSnapshot(ctx, promptID)
|
||||||
var hash string
|
|
||||||
var err error
|
|
||||||
if indexed, ok := r.promptDefs.(interface {
|
|
||||||
GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error)
|
|
||||||
}); ok {
|
|
||||||
definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID)
|
|
||||||
} else {
|
|
||||||
definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID)
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, promptLookupError(err)
|
||||||
}
|
}
|
||||||
if definition == nil {
|
if definition == nil {
|
||||||
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hash == "" {
|
|
||||||
hash, err = hashPromptDefinition(definition)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,41 +39,12 @@ func (r *Runner) resolvePromptInspection(ctx context.Context, promptID string) (
|
|||||||
if r == nil || r.promptDefs == nil {
|
if r == nil || r.promptDefs == nil {
|
||||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
inspection, err := r.promptDefs.GetPromptInspection(ctx, promptID)
|
||||||
inspection, err := indexed.GetPromptInspection(ctx, promptID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
|
||||||
}
|
|
||||||
if inspection == nil {
|
|
||||||
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
|
||||||
}
|
|
||||||
return inspection, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
selection, err := r.resolvePromptDefinition(ctx, promptID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, promptLookupError(err)
|
||||||
}
|
}
|
||||||
definition := selection.definition
|
if inspection == nil {
|
||||||
return &domain.PromptInspection{
|
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
||||||
PromptID: definition.ID,
|
}
|
||||||
PromptHash: selection.hash,
|
return inspection, nil
|
||||||
DefaultProfileID: definition.DefaultProfile,
|
|
||||||
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
|
||||||
OutputContract: definition.Validation,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt without execution work.
|
|
||||||
func (r *Runner) InspectPrompt(ctx context.Context, promptID string) (*domain.PromptInspection, error) {
|
|
||||||
if strings.TrimSpace(promptID) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
return r.resolvePromptInspection(ctx, promptID)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,208 +0,0 @@
|
|||||||
package usecase
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/promptdef"
|
|
||||||
)
|
|
||||||
|
|
||||||
type inspectionPromptRepository struct {
|
|
||||||
definition *domain.PromptDefinition
|
|
||||||
err error
|
|
||||||
calls int
|
|
||||||
id string
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptMetadataRepository struct {
|
|
||||||
inspection domain.PromptInspection
|
|
||||||
inspectionErr error
|
|
||||||
lookupID string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
|
||||||
_ context.Context,
|
|
||||||
id string,
|
|
||||||
) (*domain.PromptDefinition, error) {
|
|
||||||
r.calls++
|
|
||||||
r.id = id
|
|
||||||
if r.err != nil {
|
|
||||||
return nil, r.err
|
|
||||||
}
|
|
||||||
return r.definition, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string) (*domain.PromptDefinition, error) {
|
|
||||||
return nil, errors.New("full prompt definition should not be requested")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptInspection(
|
|
||||||
_ context.Context,
|
|
||||||
id string,
|
|
||||||
) (*domain.PromptInspection, error) {
|
|
||||||
r.lookupID = id
|
|
||||||
inspection := r.inspection
|
|
||||||
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
|
|
||||||
return &inspection, r.inspectionErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
|
|
||||||
definition := &domain.PromptDefinition{
|
|
||||||
ID: "normalized.prompt",
|
|
||||||
DefaultProfile: "not-resolved",
|
|
||||||
Inputs: []domain.PromptInput{
|
|
||||||
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
|
|
||||||
{Name: "audience", ContentType: "text/plain", Description: "Intended reader."},
|
|
||||||
},
|
|
||||||
Validation: domain.OutputContract{
|
|
||||||
Format: domain.FormatJSON,
|
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
|
||||||
SchemaPath: "schemas/result.json",
|
|
||||||
RepairAttempts: 2,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
repository := &inspectionPromptRepository{definition: definition}
|
|
||||||
runner := &Runner{promptDefs: repository}
|
|
||||||
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
|
||||||
}
|
|
||||||
wantHash, err := hashPromptDefinition(definition)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("hash prompt definition: %v", err)
|
|
||||||
}
|
|
||||||
if repository.calls != 1 || repository.id != " prompt-id " {
|
|
||||||
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
|
||||||
}
|
|
||||||
if inspection.PromptID != definition.ID ||
|
|
||||||
inspection.PromptHash != wantHash ||
|
|
||||||
inspection.DefaultProfileID != definition.DefaultProfile ||
|
|
||||||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
|
||||||
inspection.OutputContract != definition.Validation {
|
|
||||||
t.Fatalf("inspection=%#v, want definition metadata", inspection)
|
|
||||||
}
|
|
||||||
|
|
||||||
inspection.Inputs[0].Name = "changed"
|
|
||||||
second, err := runner.InspectPrompt(context.Background(), " prompt-id ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect prompt again: %v", err)
|
|
||||||
}
|
|
||||||
if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" {
|
|
||||||
t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
|
||||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
|
||||||
PromptID: "captured.prompt", PromptHash: "cached-prompt-hash",
|
|
||||||
DefaultProfileID: "partial-profile",
|
|
||||||
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
|
|
||||||
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
|
|
||||||
}}
|
|
||||||
runner := &Runner{promptDefs: repository}
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ")
|
|
||||||
if err != nil || repository.lookupID != " requested-id " ||
|
|
||||||
inspection.PromptID != "captured.prompt" ||
|
|
||||||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
|
|
||||||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
|
|
||||||
t.Fatalf("metadata inspection=(%+v, %v), lookup=%q", inspection, err, repository.lookupID)
|
|
||||||
}
|
|
||||||
inspection.Inputs[0].Name = "caller mutation"
|
|
||||||
inspection.OutputContract.SchemaPath = "caller mutation"
|
|
||||||
second, err := runner.InspectPrompt(context.Background(), "requested-id")
|
|
||||||
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
|
|
||||||
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
|
||||||
repository := &requirementInspectionPromptRepository{
|
|
||||||
definition: &domain.PromptDefinition{
|
|
||||||
ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
|
||||||
if err != nil || inspection.PromptID != "captured.prompt" ||
|
|
||||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
|
||||||
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
|
|
||||||
t.Run("blank ID", func(t *testing.T) {
|
|
||||||
repository := &inspectionPromptRepository{}
|
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ")
|
|
||||||
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
|
|
||||||
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("canceled context", func(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
repository := &inspectionPromptRepository{}
|
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt")
|
|
||||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
|
|
||||||
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing prompt", func(t *testing.T) {
|
|
||||||
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
|
|
||||||
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing")
|
|
||||||
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
|
||||||
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("defensive prompt dependencies", func(t *testing.T) {
|
|
||||||
var nilRunner *Runner
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
runner *Runner
|
|
||||||
}{
|
|
||||||
{name: "nil runner", runner: nilRunner},
|
|
||||||
{name: "nil repository", runner: &Runner{}},
|
|
||||||
{name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}},
|
|
||||||
}
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
_, err := tc.runner.InspectPrompt(context.Background(), "prompt")
|
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
|
||||||
t.Fatalf("inspection error=%v, want prompt load", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
|
|
||||||
definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)
|
|
||||||
repository := &fakePromptRepo{def: definition}
|
|
||||||
runner := NewRunner(
|
|
||||||
repository,
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
||||||
|
|
||||||
&fakeArtifactReader{},
|
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
|
|
||||||
nil,
|
|
||||||
nil,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
inspection, err := runner.InspectPrompt(context.Background(), definition.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
|
||||||
}
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, ProfileID: "exec"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("prepare prompt: %v", err)
|
|
||||||
}
|
|
||||||
if inspection.PromptHash != prepared.PromptHash {
|
|
||||||
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
48
internal/usecase/repair_prefix.go
Normal file
48
internal/usecase/repair_prefix.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// boundedDiagnosticJSON returns a JSON string prefix and whether it represents
|
||||||
|
// the complete normalized input. It retains at most the useful prefix.
|
||||||
|
func boundedDiagnosticJSON(value string, maxBytes int) ([]byte, bool) {
|
||||||
|
if maxBytes < 2 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
limit := maxBytes - 2
|
||||||
|
var prefix strings.Builder
|
||||||
|
used := 0
|
||||||
|
more := false
|
||||||
|
for i := 0; i < len(value); {
|
||||||
|
r, size := utf8.DecodeRuneInString(value[i:])
|
||||||
|
piece := value[i : i+size]
|
||||||
|
if r == utf8.RuneError && size == 1 {
|
||||||
|
piece = "\uFFFD"
|
||||||
|
i++
|
||||||
|
for i < len(value) {
|
||||||
|
r, size = utf8.DecodeRuneInString(value[i:])
|
||||||
|
if r != utf8.RuneError || size != 1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
i += size
|
||||||
|
}
|
||||||
|
if used+len(piece) > limit {
|
||||||
|
more = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
prefix.WriteString(piece)
|
||||||
|
used += len(piece)
|
||||||
|
}
|
||||||
|
normalized := prefix.String()
|
||||||
|
encoded, _ := json.Marshal(normalized)
|
||||||
|
if len(encoded) > maxBytes {
|
||||||
|
return truncateDiagnosticJSONValue(normalized, maxBytes), false
|
||||||
|
}
|
||||||
|
return encoded, !more
|
||||||
|
}
|
||||||
33
internal/usecase/repair_prefix_test.go
Normal file
33
internal/usecase/repair_prefix_test.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBoundedDiagnosticJSON(t *testing.T) {
|
||||||
|
for _, input := range []string{"", "plain", "<&>\n\"", "雪☃", "a\xff\xfe\xffb", "\xff\xfe" + strings.Repeat("z", 100)} {
|
||||||
|
for budget := 0; budget <= 30; budget++ {
|
||||||
|
encoded, complete := boundedDiagnosticJSON(input, budget)
|
||||||
|
if budget < 2 {
|
||||||
|
if encoded != nil || complete {
|
||||||
|
t.Fatalf("budget %d: %q %v", budget, encoded, complete)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(encoded) > budget || !utf8.Valid(encoded) {
|
||||||
|
t.Fatalf("budget %d: %q", budget, encoded)
|
||||||
|
}
|
||||||
|
var decoded string
|
||||||
|
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
normalized := strings.ToValidUTF8(input, "\uFFFD")
|
||||||
|
if !strings.HasPrefix(normalized, decoded) || complete != (decoded == normalized) {
|
||||||
|
t.Fatalf("input %q budget %d: %q complete=%v normalized=%q", input, budget, decoded, complete, normalized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -117,27 +117,38 @@ func repairGuidance(mode domain.ValidationMode) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func formatRepairDiagnostics(errors []string) string {
|
func formatRepairDiagnostics(errors []string) string {
|
||||||
diagnostics := make([]string, len(errors))
|
complete := []byte{'['}
|
||||||
for index, diagnostic := range errors {
|
fits := true
|
||||||
diagnostics[index] = strings.ToValidUTF8(diagnostic, "\uFFFD")
|
for _, diagnostic := range errors {
|
||||||
|
separator := 0
|
||||||
|
if len(complete) > 1 {
|
||||||
|
separator = 1
|
||||||
|
}
|
||||||
|
entry, whole := boundedDiagnosticJSON(diagnostic, maxRepairDiagnosticBytes-len(complete)-separator-1)
|
||||||
|
if !whole {
|
||||||
|
fits = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if separator != 0 {
|
||||||
|
complete = append(complete, ',')
|
||||||
|
}
|
||||||
|
complete = append(complete, entry...)
|
||||||
}
|
}
|
||||||
|
if fits {
|
||||||
complete, _ := json.Marshal(diagnostics)
|
return string(append(complete, ']'))
|
||||||
if len(complete) <= maxRepairDiagnosticBytes {
|
|
||||||
return string(complete)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
||||||
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
|
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
|
||||||
encoded = append(encoded, '[')
|
encoded = append(encoded, '[')
|
||||||
for _, diagnostic := range diagnostics {
|
for _, diagnostic := range errors {
|
||||||
entry, _ := json.Marshal(diagnostic)
|
|
||||||
separator := 0
|
separator := 0
|
||||||
if len(encoded) > 1 {
|
if len(encoded) > 1 {
|
||||||
separator = 1
|
separator = 1
|
||||||
}
|
}
|
||||||
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
|
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
|
||||||
if len(entry) <= available {
|
entry, whole := boundedDiagnosticJSON(diagnostic, available)
|
||||||
|
if whole {
|
||||||
if separator != 0 {
|
if separator != 0 {
|
||||||
encoded = append(encoded, ',')
|
encoded = append(encoded, ',')
|
||||||
}
|
}
|
||||||
@@ -151,7 +162,7 @@ func formatRepairDiagnostics(errors []string) string {
|
|||||||
if separator != 0 {
|
if separator != 0 {
|
||||||
encoded = append(encoded, ',')
|
encoded = append(encoded, ',')
|
||||||
}
|
}
|
||||||
encoded = append(encoded, truncateDiagnosticJSONValue(diagnostic, available)...)
|
encoded = append(encoded, entry...)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if len(encoded) > 1 {
|
if len(encoded) > 1 {
|
||||||
|
|||||||
@@ -298,6 +298,23 @@ func TestFormatRepairDiagnosticsBoundsAndPreservesData(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFormatRepairDiagnosticsEmptyAndEscapedOversize(t *testing.T) {
|
||||||
|
for _, values := range [][]string{nil, {}} {
|
||||||
|
if got := formatRepairDiagnostics(values); got != "[]" {
|
||||||
|
t.Fatalf("empty diagnostics = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value := strings.Repeat("<&>\x00\xff\xfe", maxRepairDiagnosticBytes)
|
||||||
|
encoded := formatRepairDiagnostics([]string{value})
|
||||||
|
var got []string
|
||||||
|
if err := json.Unmarshal([]byte(encoded), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(encoded) > maxRepairDiagnosticBytes || len(got) != 2 || got[1] != omittedRepairDiagnostics || !strings.HasPrefix(strings.ToValidUTF8(value, "\uFFFD"), got[0]) {
|
||||||
|
t.Fatalf("oversize diagnostics length=%d, values=%d", len(encoded), len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func manyRepairDiagnostics() []string {
|
func manyRepairDiagnostics() []string {
|
||||||
diagnostics := make([]string, 1_000)
|
diagnostics := make([]string, 1_000)
|
||||||
for index := range diagnostics {
|
for index := range diagnostics {
|
||||||
|
|||||||
@@ -18,10 +18,6 @@ type RequirementInspectionRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
type requirementSnapshotRepository interface {
|
|
||||||
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
|
||||||
}
|
|
||||||
|
|
||||||
// InspectRequirements discovers prompt inputs and caller variables without
|
// InspectRequirements discovers prompt inputs and caller variables without
|
||||||
// loading artifacts or performing execution preparation.
|
// loading artifacts or performing execution preparation.
|
||||||
func (r *Runner) InspectRequirements(
|
func (r *Runner) InspectRequirements(
|
||||||
@@ -36,7 +32,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
selection, err := r.resolvePromptInspection(ctx, req.PromptID)
|
selection, err := r.resolvePromptInspection(ctx, req.PromptID)
|
||||||
@@ -44,7 +40,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, &OperationError{Category: ErrProfileLoad, Cause: err}
|
||||||
}
|
}
|
||||||
selectedProfileID := req.ProfileID
|
selectedProfileID := req.ProfileID
|
||||||
if strings.TrimSpace(selectedProfileID) == "" {
|
if strings.TrimSpace(selectedProfileID) == "" {
|
||||||
@@ -58,21 +54,17 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
repository, ok := r.promptDefs.(requirementSnapshotRepository)
|
snapshot, ok := r.promptDefs.GetRequirementSnapshot(selection.PromptID)
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
|
||||||
}
|
|
||||||
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID)
|
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: selection.PromptID}
|
definition := &domain.PromptDefinition{ID: selection.PromptID}
|
||||||
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
inspection.ProfileID = profileIdentity
|
inspection.ProfileID = profileIdentity
|
||||||
return &inspection, nil
|
return &inspection, nil
|
||||||
|
|||||||
@@ -45,13 +45,6 @@ func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
|
|||||||
return repository.identity, repository.identityErr
|
return repository.identity, repository.identityErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
|
||||||
context.Context,
|
|
||||||
string,
|
|
||||||
) (*domain.PromptDefinition, error) {
|
|
||||||
return repository.definition, repository.lookupErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
@@ -71,6 +64,11 @@ func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
|
|||||||
return repository.definition, "captured-hash", repository.lookupErr
|
return repository.definition, "captured-hash", repository.lookupErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (repository *requirementInspectionPromptRepository) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
|
definition, hash, err := repository.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
||||||
id string,
|
id string,
|
||||||
) (prompt.RequirementSnapshot, bool) {
|
) (prompt.RequirementSnapshot, bool) {
|
||||||
@@ -163,7 +161,7 @@ func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) {
|
func TestRunnerInspectRequirementsUsesProfileIdentity(t *testing.T) {
|
||||||
message, err := prompt.CompileTemplate("message", `{{.name}}`)
|
message, err := prompt.CompileTemplate("message", `{{.name}}`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -177,7 +175,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
|
|||||||
}
|
}
|
||||||
requestedID := " literal-profile "
|
requestedID := " literal-profile "
|
||||||
|
|
||||||
t.Run("optional capability", func(t *testing.T) {
|
t.Run("identity lookup", func(t *testing.T) {
|
||||||
profiles := &inspectionProfileIdentityRepository{
|
profiles := &inspectionProfileIdentityRepository{
|
||||||
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||||
@@ -196,21 +194,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("full-profile fallback", func(t *testing.T) {
|
t.Run("identity failure", func(t *testing.T) {
|
||||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
|
||||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
|
||||||
}}
|
|
||||||
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
|
||||||
context.Background(), RequirementInspectionRequest{
|
|
||||||
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil || inspection.ProfileID != requestedID || profiles.calls != 1 || profiles.id != requestedID {
|
|
||||||
t.Fatalf("fallback inspection=(%+v, %v), full lookup=(%d,%q)", inspection, err, profiles.calls, profiles.id)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("capability failure", func(t *testing.T) {
|
|
||||||
lookupErr := errors.New("profile identity lookup failed")
|
lookupErr := errors.New("profile identity lookup failed")
|
||||||
profiles := &inspectionProfileIdentityRepository{
|
profiles := &inspectionProfileIdentityRepository{
|
||||||
inspectionProfileRepository: &inspectionProfileRepository{},
|
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||||
@@ -329,14 +313,6 @@ func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *test
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("repository lacks snapshot capability", func(t *testing.T) {
|
|
||||||
_, err := (&Runner{promptDefs: &inspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
|
||||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}},
|
|
||||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
|
||||||
t.Fatalf("missing capability error=%v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("nil runner error=%v", err)
|
t.Fatalf("nil runner error=%v", err)
|
||||||
}
|
}
|
||||||
@@ -348,11 +324,6 @@ func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *test
|
|||||||
); !errors.Is(err, ErrProfileLoad) {
|
); !errors.Is(err, ErrProfileLoad) {
|
||||||
t.Fatalf("missing profile error=%v", err)
|
t.Fatalf("missing profile error=%v", err)
|
||||||
}
|
}
|
||||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
|
||||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}},
|
|
||||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrProfileLoad) {
|
|
||||||
t.Fatalf("unresolved profile error=%v", err)
|
|
||||||
}
|
|
||||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
|
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
|
||||||
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
|
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
|
||||||
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
|||||||
@@ -9,20 +9,6 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestInspectProfileAllowsIncompleteResolvedTarget(t *testing.T) {
|
|
||||||
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
|
||||||
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
|
||||||
}}
|
|
||||||
runner := &Runner{profiles: staticExecutionProfileRepo{profile: profile}}
|
|
||||||
inspection, err := runner.InspectProfile(context.Background(), "partial")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("partial profile inspection failed: %v", err)
|
|
||||||
}
|
|
||||||
if inspection.EffectiveModelParams.Endpoint != "" || inspection.EffectiveModelParams.Model != "" {
|
|
||||||
t.Fatalf("inspection invented a runnable target: %+v", inspection)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
|
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
|
||||||
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
||||||
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
||||||
@@ -58,6 +44,19 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvedTargetPreservesProviderMapSelection(t *testing.T) {
|
||||||
|
profile := &domain.ExecutionProfile{ResolvedTarget: &domain.ExecutionTarget{ExtraParams: map[string]any{"configured": true}}}
|
||||||
|
selected, _ := resolveExecutionTarget(profile, &domain.ExecutionTargetOverride{ExtraParams: map[string]any{}})
|
||||||
|
if selected.ExtraParams["configured"] != true {
|
||||||
|
t.Fatalf("empty override did not inherit provider settings: %#v", selected.ExtraParams)
|
||||||
|
}
|
||||||
|
profile.ResolvedTarget.ExtraParams = map[string]any{}
|
||||||
|
selected, _ = resolveExecutionTarget(profile, nil)
|
||||||
|
if selected.ExtraParams != nil {
|
||||||
|
t.Fatalf("empty provider settings changed projection: %#v", selected.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvedTargetCredentialAvailability(t *testing.T) {
|
func TestResolvedTargetCredentialAvailability(t *testing.T) {
|
||||||
profile := &domain.ExecutionProfile{ID: "direct", ResolvedTarget: &domain.ExecutionTarget{
|
profile := &domain.ExecutionProfile{ID: "direct", ResolvedTarget: &domain.ExecutionTarget{
|
||||||
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
@@ -42,17 +43,22 @@ var (
|
|||||||
|
|
||||||
// Runner executes the Promptkit core use case.
|
// Runner executes the Promptkit core use case.
|
||||||
type Runner struct {
|
type Runner struct {
|
||||||
|
identity *runnerIdentity
|
||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
artifacts artifact.Reader
|
artifacts artifact.Reader
|
||||||
renderer prompt.Renderer
|
renderer prompt.Renderer
|
||||||
llm llm.Client
|
llm llm.Client
|
||||||
validator validate.Validator
|
validator validate.ValidationPreparer
|
||||||
repairer OutputRepairer
|
repairer OutputRepairer
|
||||||
admitter RunAdmitter
|
admitter RunAdmitter
|
||||||
defaultTransientRetries uint64
|
defaultTransientRetries uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A non-zero-sized token gives each constructed runner a distinct identity
|
||||||
|
// without retaining its collaborators in prepared handles.
|
||||||
|
type runnerIdentity struct{ marker byte }
|
||||||
|
|
||||||
// RunAdmitter reserves capacity for one resolved backend run.
|
// RunAdmitter reserves capacity for one resolved backend run.
|
||||||
type RunAdmitter interface {
|
type RunAdmitter interface {
|
||||||
Admit(context.Context, string) (func(), error)
|
Admit(context.Context, string) (func(), error)
|
||||||
@@ -81,7 +87,7 @@ func NewRunner(
|
|||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
validator validate.Validator,
|
validator validate.ValidationPreparer,
|
||||||
admitter RunAdmitter,
|
admitter RunAdmitter,
|
||||||
) *Runner {
|
) *Runner {
|
||||||
return NewRunnerWithRepairer(
|
return NewRunnerWithRepairer(
|
||||||
@@ -102,7 +108,7 @@ func NewRunnerWithRepairer(
|
|||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
validator validate.Validator,
|
validator validate.ValidationPreparer,
|
||||||
repairer OutputRepairer,
|
repairer OutputRepairer,
|
||||||
admitter RunAdmitter,
|
admitter RunAdmitter,
|
||||||
defaultTransientRetries ...uint64,
|
defaultTransientRetries ...uint64,
|
||||||
@@ -112,6 +118,7 @@ func NewRunnerWithRepairer(
|
|||||||
retries = defaultTransientRetries[0]
|
retries = defaultTransientRetries[0]
|
||||||
}
|
}
|
||||||
return &Runner{
|
return &Runner{
|
||||||
|
identity: &runnerIdentity{},
|
||||||
promptDefs: promptDefs,
|
promptDefs: promptDefs,
|
||||||
profiles: profiles,
|
profiles: profiles,
|
||||||
artifacts: artifacts,
|
artifacts: artifacts,
|
||||||
@@ -204,7 +211,7 @@ func (r *Runner) executePreparedRun(
|
|||||||
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||||
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
|
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.shouldAttemptRepair(prepared.OutputContract, validationResult) {
|
if r.shouldAttemptRepair(prepared.OutputContract, validationResult) {
|
||||||
@@ -238,7 +245,7 @@ func (r *Runner) executePreparedRun(
|
|||||||
|
|
||||||
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
|
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -269,10 +276,10 @@ func (r *Runner) executePreparedRun(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func wrapGenerationError(err error) error {
|
func wrapGenerationError(err error) error {
|
||||||
if errors.Is(err, llm.ErrInvalidRequest) {
|
if _, ok := err.(*generation.PreparationError); ok {
|
||||||
return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return &OperationError{Category: ErrInvalidRequest, Cause: err}
|
||||||
}
|
}
|
||||||
return fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
return &OperationError{Category: ErrLLMGenerate, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
||||||
@@ -402,13 +409,9 @@ func (r *Runner) prepareValidation(
|
|||||||
if r.validator == nil {
|
if r.validator == nil {
|
||||||
return noOpPreparedValidation{contract: contract}, nil
|
return noOpPreparedValidation{contract: contract}, nil
|
||||||
}
|
}
|
||||||
preparer, ok := r.validator.(validate.ValidationPreparer)
|
plan, err := r.validator.PrepareValidation(ctx, contract)
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
|
|
||||||
}
|
|
||||||
plan, err := preparer.PrepareValidation(ctx, contract)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||||
}
|
}
|
||||||
if plan == nil {
|
if plan == nil {
|
||||||
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||||
@@ -432,11 +435,7 @@ func (r *Runner) structuredOutputFromValidationPlan(
|
|||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
||||||
}
|
}
|
||||||
schemaDocument, err := jsonvalue.Copy(schemaDocument)
|
return structuredOutputSpec(def, jsonvalue.CloneTrusted(schemaDocument)), nil
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: invalid prepared json_schema schema document: %v", ErrValidation, err)
|
|
||||||
}
|
|
||||||
return structuredOutputSpec(def, schemaDocument), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) completePreparationWithStructuredOutput(
|
func (r *Runner) completePreparationWithStructuredOutput(
|
||||||
@@ -450,7 +449,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
|
|||||||
for name, ref := range req.Inputs {
|
for name, ref := range req.Inputs {
|
||||||
art, readErr := r.artifacts.Read(ctx, ref)
|
art, readErr := r.artifacts.Read(ctx, ref)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
return nil, fmt.Errorf("%w: input %q: %w", ErrArtifactLoad, name, readErr)
|
return nil, &OperationError{Category: ErrArtifactLoad, Cause: fmt.Errorf("input %q: %w", name, readErr)}
|
||||||
}
|
}
|
||||||
if art.Name == "" {
|
if art.Name == "" {
|
||||||
art.Name = name
|
art.Name = name
|
||||||
@@ -467,7 +466,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
|
|||||||
}
|
}
|
||||||
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
|
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
|
return nil, &OperationError{Category: ErrPromptRender, Cause: err}
|
||||||
}
|
}
|
||||||
effectivePrompt := *renderedPrompt
|
effectivePrompt := *renderedPrompt
|
||||||
if state.directSessionID != "" {
|
if state.directSessionID != "" {
|
||||||
@@ -593,14 +592,16 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, presence domain.E
|
|||||||
out.APIKeyEnv = override.APIKeyEnv
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
}
|
}
|
||||||
if len(override.ExtraParams) > 0 {
|
if len(override.ExtraParams) > 0 {
|
||||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
out.ExtraParams = override.ExtraParams
|
||||||
}
|
}
|
||||||
return out, presence
|
return out, presence
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
|
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
|
||||||
out := *profileValue.ResolvedTarget
|
out := *profileValue.ResolvedTarget
|
||||||
out.ExtraParams = copyExtraParams(out.ExtraParams)
|
if len(out.ExtraParams) == 0 {
|
||||||
|
out.ExtraParams = nil
|
||||||
|
}
|
||||||
presence := profileValue.TargetPresence
|
presence := profileValue.TargetPresence
|
||||||
if override != nil {
|
if override != nil {
|
||||||
return mergeExecutionTargetOverride(out, presence, *override)
|
return mergeExecutionTargetOverride(out, presence, *override)
|
||||||
@@ -625,22 +626,8 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyExtraParams(src map[string]any) map[string]any {
|
|
||||||
if len(src) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cp := make(map[string]any, len(src))
|
|
||||||
for k, v := range src {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
return cp
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) {
|
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) {
|
||||||
contract := def.Validation
|
contract := def.Validation
|
||||||
if contract.Format == "" {
|
|
||||||
contract.Format = def.OutputFormat
|
|
||||||
}
|
|
||||||
if override != nil {
|
if override != nil {
|
||||||
contract = *override
|
contract = *override
|
||||||
if contract.Format == "" {
|
if contract.Format == "" {
|
||||||
@@ -707,7 +694,3 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
|
|||||||
Hash: hex.EncodeToString(hash[:]),
|
Hash: hex.EncodeToString(hash[:]),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
|
|
||||||
return promptdef.HashDefinition(def)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,13 +13,17 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/prompt"
|
"go.maximumdirect.net/promptkit/internal/prompt"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,11 +43,61 @@ type staticPromptRepo struct {
|
|||||||
def *domain.PromptDefinition
|
def *domain.PromptDefinition
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r staticPromptRepo) GetPromptDefinition(
|
type capturedTemplateFixture struct {
|
||||||
|
id string
|
||||||
|
templates prompt.CompiledTemplates
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fixture capturedTemplateFixture) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
||||||
|
return fixture.templates, id == fixture.id
|
||||||
|
}
|
||||||
|
|
||||||
|
func testCompiledRenderer(t *testing.T, definition *domain.PromptDefinition) prompt.Renderer {
|
||||||
|
t.Helper()
|
||||||
|
var templates prompt.CompiledTemplates
|
||||||
|
if definition.SessionID != "" {
|
||||||
|
var err error
|
||||||
|
templates.Session, err = prompt.CompileTemplate("session", definition.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
templates.Messages = make([]*template.Template, 0, len(definition.Templates))
|
||||||
|
for _, message := range definition.Templates {
|
||||||
|
compiled, err := prompt.CompileTemplate("message", message.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
templates.Messages = append(templates.Messages, compiled)
|
||||||
|
}
|
||||||
|
return prompt.NewCompiledRenderer(capturedTemplateFixture{id: definition.ID, templates: templates})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r staticPromptRepo) GetPromptSnapshot(
|
||||||
context.Context,
|
context.Context,
|
||||||
string,
|
string,
|
||||||
) (*domain.PromptDefinition, error) {
|
) (*domain.PromptDefinition, string, error) {
|
||||||
return r.def, nil
|
hash, err := promptdef.HashDefinition(r.def)
|
||||||
|
return r.def, hash, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r staticPromptRepo) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
|
definition, hash, err := r.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (staticPromptRepo) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixturePromptInspection(definition *domain.PromptDefinition, hash string) *domain.PromptInspection {
|
||||||
|
if definition == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &domain.PromptInspection{
|
||||||
|
PromptID: definition.ID, PromptHash: hash, DefaultProfileID: definition.DefaultProfile,
|
||||||
|
Inputs: append([]domain.PromptInput(nil), definition.Inputs...), OutputContract: definition.Validation,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type staticExecutionProfileRepo struct {
|
type staticExecutionProfileRepo struct {
|
||||||
@@ -58,6 +112,10 @@ func (r staticExecutionProfileRepo) GetProfile(
|
|||||||
return &value, nil
|
return &value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (staticExecutionProfileRepo) GetProfileIdentity(_ context.Context, id string) (string, error) {
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
type staticArtifactReader struct{}
|
type staticArtifactReader struct{}
|
||||||
|
|
||||||
func (staticArtifactReader) Read(
|
func (staticArtifactReader) Read(
|
||||||
@@ -87,17 +145,40 @@ func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*
|
|||||||
}
|
}
|
||||||
if p, ok := f.profiles[id]; ok {
|
if p, ok := f.profiles[id]; ok {
|
||||||
cp := *p
|
cp := *p
|
||||||
|
if cp.ResolvedTarget != nil {
|
||||||
|
target := *cp.ResolvedTarget
|
||||||
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
|
cp.ResolvedTarget = &target
|
||||||
|
}
|
||||||
return &cp, nil
|
return &cp, nil
|
||||||
}
|
}
|
||||||
return nil, errors.New("profile not found")
|
return nil, errors.New("profile not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
|
func (f *fakeExecutionProfileRepo) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
_, err := f.GetProfile(ctx, id)
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakePromptRepo) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error) {
|
||||||
f.lastID = id
|
f.lastID = id
|
||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return nil, f.err
|
return nil, "", f.err
|
||||||
}
|
}
|
||||||
return f.def, nil
|
if f.def == nil {
|
||||||
|
return nil, "", nil
|
||||||
|
}
|
||||||
|
hash, err := promptdef.HashDefinition(f.def)
|
||||||
|
return f.def, hash, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakePromptRepo) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
|
definition, hash, err := f.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*fakePromptRepo) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeArtifactReader struct {
|
type fakeArtifactReader struct {
|
||||||
@@ -165,14 +246,6 @@ type fakeValidator struct {
|
|||||||
validateCalls int
|
validateCalls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
|
||||||
f.validateCalls++
|
|
||||||
if f.err != nil {
|
|
||||||
return domain.ValidationResult{}, f.err
|
|
||||||
}
|
|
||||||
return f.result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
||||||
var schemaDocument any
|
var schemaDocument any
|
||||||
if contract.ValidationMode == domain.ValidationJSONSchema {
|
if contract.ValidationMode == domain.ValidationJSONSchema {
|
||||||
@@ -186,17 +259,20 @@ func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.Out
|
|||||||
schemaDocument = map[string]any{"type": "object"}
|
schemaDocument = map[string]any{"type": "object"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &fakePreparedValidator{validator: f, contract: contract, schemaDocument: schemaDocument}, nil
|
return &fakePreparedValidator{validator: f, schemaDocument: schemaDocument}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakePreparedValidator struct {
|
type fakePreparedValidator struct {
|
||||||
validator *fakeValidator
|
validator *fakeValidator
|
||||||
contract domain.OutputContract
|
|
||||||
schemaDocument any
|
schemaDocument any
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
||||||
return p.validator.Validate(ctx, artifact, p.contract)
|
p.validator.validateCalls++
|
||||||
|
if p.validator.err != nil {
|
||||||
|
return domain.ValidationResult{}, p.validator.err
|
||||||
|
}
|
||||||
|
return p.validator.result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *fakePreparedValidator) SchemaDocument() any {
|
func (p *fakePreparedValidator) SchemaDocument() any {
|
||||||
@@ -405,7 +481,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -452,7 +528,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -482,7 +558,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -842,7 +918,7 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -869,7 +945,7 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -2074,7 +2150,7 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
func TestRunnerRunInjectedInvalidRequestRemainsGenerationFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
@@ -2089,11 +2165,8 @@ func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
|||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrLLMGenerate) || !errors.Is(err, llm.ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected generation error retaining cause, got %v", err)
|
||||||
}
|
|
||||||
if errors.Is(err, ErrLLMGenerate) {
|
|
||||||
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2401,7 +2474,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
|
|||||||
baseClient := &controlledRepairLLM{
|
baseClient := &controlledRepairLLM{
|
||||||
events: make(chan controlledGeneration, 4),
|
events: make(chan controlledGeneration, 4),
|
||||||
}
|
}
|
||||||
scheduledClient := capacity.NewClient(manager, baseClient)
|
scheduledClient := generation.NewInjected(manager, baseClient)
|
||||||
admitter := &recordingRunAdmitter{next: manager}
|
admitter := &recordingRunAdmitter{next: manager}
|
||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
@@ -2507,7 +2580,6 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
|
|||||||
DefaultProfile: "exec",
|
DefaultProfile: "exec",
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
||||||
OutputFormat: format,
|
|
||||||
Validation: domain.OutputContract{
|
Validation: domain.OutputContract{
|
||||||
ValidationMode: mode,
|
ValidationMode: mode,
|
||||||
RepairAttempts: attempts,
|
RepairAttempts: attempts,
|
||||||
@@ -2589,10 +2661,6 @@ func float64Ptr(v float64) *float64 {
|
|||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
func stringPtr(v string) *string {
|
|
||||||
return &v
|
|
||||||
}
|
|
||||||
|
|
||||||
func intPtr(v int) *int {
|
func intPtr(v int) *int {
|
||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
package usecase
|
package usecase
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
@@ -13,10 +10,12 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
type authenticationProbe struct {
|
type smokeAuthenticationClient interface {
|
||||||
client smokeHTTPGetter
|
ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type authenticationProbe struct{ client smokeAuthenticationClient }
|
||||||
|
|
||||||
type authenticationProbeOutcome struct {
|
type authenticationProbeOutcome struct {
|
||||||
supported bool
|
supported bool
|
||||||
applicable *bool
|
applicable *bool
|
||||||
@@ -26,16 +25,33 @@ type authenticationProbeOutcome struct {
|
|||||||
failure domain.ExecutionFailureKind
|
failure domain.ExecutionFailureKind
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p authenticationProbe) run(
|
func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, target domain.SmokeTestTarget) (authenticationProbeOutcome, error) {
|
||||||
ctx context.Context,
|
|
||||||
backend domain.Backend,
|
|
||||||
target domain.SmokeTestTarget,
|
|
||||||
) (authenticationProbeOutcome, error) {
|
|
||||||
outcome, err := p.support(backend)
|
outcome, err := p.support(backend)
|
||||||
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
|
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
|
||||||
return outcome, err
|
return outcome, err
|
||||||
}
|
}
|
||||||
return p.runOpenRouterKey(ctx, target)
|
result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
|
||||||
|
outcome.evidence = result.HTTP
|
||||||
|
outcome.credentialsAccepted = result.Accepted
|
||||||
|
if result.Overflow {
|
||||||
|
outcome.reason = domain.SmokeTestReasonAuthenticationResponseLimitExceeded
|
||||||
|
outcome.failure = domain.ExecutionFailureOther
|
||||||
|
} else if result.Unrecognized {
|
||||||
|
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
||||||
|
outcome.failure = domain.ExecutionFailureMalformedResponse
|
||||||
|
} else if requestErr != nil {
|
||||||
|
if !result.HTTP.DispatchInvoked {
|
||||||
|
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
|
||||||
|
}
|
||||||
|
outcome.failure = httpProbeFailure(ctx, requestErr)
|
||||||
|
} else if result.HTTP.ResponseReceived {
|
||||||
|
if result.HTTP.HTTPStatus != nil && *result.HTTP.HTTPStatus != http.StatusOK && *result.HTTP.HTTPStatus != http.StatusUnauthorized {
|
||||||
|
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
outcome.failure = httpProbeFailure(ctx, nil)
|
||||||
|
}
|
||||||
|
return outcome, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
|
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
|
||||||
@@ -48,11 +64,7 @@ func (p authenticationProbe) support(backend domain.Backend) (authenticationProb
|
|||||||
}
|
}
|
||||||
switch *authentication.Kind {
|
switch *authentication.Kind {
|
||||||
case "none":
|
case "none":
|
||||||
return authenticationProbeOutcome{
|
return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil
|
||||||
supported: true,
|
|
||||||
applicable: boolPointer(false),
|
|
||||||
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
|
|
||||||
}, nil
|
|
||||||
case "openrouter_key":
|
case "openrouter_key":
|
||||||
if p.client == nil {
|
if p.client == nil {
|
||||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
||||||
@@ -62,129 +74,3 @@ func (p authenticationProbe) support(backend domain.Backend) (authenticationProb
|
|||||||
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
|
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p authenticationProbe) runOpenRouterKey(
|
|
||||||
ctx context.Context,
|
|
||||||
target domain.SmokeTestTarget,
|
|
||||||
) (authenticationProbeOutcome, error) {
|
|
||||||
endpoint, err := endpointWithoutUserInfo(target.Endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return authenticationProbeOutcome{}, fmt.Errorf("prepare authentication endpoint: %w", err)
|
|
||||||
}
|
|
||||||
response, evidence, requestErr := p.client.GetWithAuthenticationEvidence(
|
|
||||||
ctx,
|
|
||||||
endpoint,
|
|
||||||
target.EffectiveSettings.TimeoutSeconds,
|
|
||||||
target.EffectiveSettings,
|
|
||||||
)
|
|
||||||
if response != nil && response.Body != nil {
|
|
||||||
defer response.Body.Close()
|
|
||||||
}
|
|
||||||
outcome := authenticationProbeOutcome{
|
|
||||||
supported: true,
|
|
||||||
applicable: boolPointer(true),
|
|
||||||
evidence: evidence,
|
|
||||||
}
|
|
||||||
if evidence.ResponseReceived || response != nil {
|
|
||||||
status := evidence.HTTPStatus
|
|
||||||
if status == nil && response != nil {
|
|
||||||
status = &response.StatusCode
|
|
||||||
}
|
|
||||||
if status == nil {
|
|
||||||
outcome.failure = domain.ExecutionFailureMalformedResponse
|
|
||||||
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
switch *status {
|
|
||||||
case http.StatusUnauthorized:
|
|
||||||
outcome.credentialsAccepted = boolPointer(false)
|
|
||||||
case http.StatusOK:
|
|
||||||
if response == nil || response.Body == nil {
|
|
||||||
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
|
||||||
outcome.failure = domain.ExecutionFailureMalformedResponse
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
recognized, bodyErr := recognizedOpenRouterKeyResponse(response.Body)
|
|
||||||
if bodyErr != nil {
|
|
||||||
outcome.failure = httpProbeFailure(ctx, bodyErr)
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
if recognized {
|
|
||||||
outcome.credentialsAccepted = boolPointer(true)
|
|
||||||
} else {
|
|
||||||
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
|
||||||
outcome.failure = domain.ExecutionFailureMalformedResponse
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
|
|
||||||
}
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if evidence.DispatchInvoked {
|
|
||||||
if requestErr != nil {
|
|
||||||
outcome.failure = httpProbeFailure(ctx, requestErr)
|
|
||||||
} else {
|
|
||||||
outcome.failure = httpProbeFailure(ctx, nil)
|
|
||||||
}
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
if requestErr != nil {
|
|
||||||
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
|
|
||||||
}
|
|
||||||
outcome.failure = generation.ClassifyFailure(llm.ErrTransport)
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func recognizedOpenRouterKeyResponse(body io.Reader) (bool, error) {
|
|
||||||
contents, err := io.ReadAll(body)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
var envelope map[string]json.RawMessage
|
|
||||||
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
dataJSON, ok := envelope["data"]
|
|
||||||
if !ok || !isJSONObject(dataJSON) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
var data map[string]json.RawMessage
|
|
||||||
if err := json.Unmarshal(dataJSON, &data); err != nil || data == nil {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
label, labelOK := data["label"]
|
|
||||||
freeTier, freeTierOK := data["is_free_tier"]
|
|
||||||
usage, usageOK := data["usage"]
|
|
||||||
if !labelOK || !isJSONString(label) || !freeTierOK || !isJSONBoolean(freeTier) || !usageOK {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
var number any
|
|
||||||
usageDecoder := json.NewDecoder(bytes.NewReader(usage))
|
|
||||||
usageDecoder.UseNumber()
|
|
||||||
if usageDecoder.Decode(&number) != nil {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
_, ok = number.(json.Number)
|
|
||||||
return ok, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isJSONObject(value json.RawMessage) bool {
|
|
||||||
value = bytes.TrimSpace(value)
|
|
||||||
return len(value) > 0 && value[0] == '{'
|
|
||||||
}
|
|
||||||
|
|
||||||
func isJSONString(value json.RawMessage) bool {
|
|
||||||
value = bytes.TrimSpace(value)
|
|
||||||
if len(value) == 0 || value[0] != '"' {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
var parsed string
|
|
||||||
return json.Unmarshal(value, &parsed) == nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isJSONBoolean(value json.RawMessage) bool {
|
|
||||||
value = bytes.TrimSpace(value)
|
|
||||||
return bytes.Equal(value, []byte("true")) || bytes.Equal(value, []byte("false"))
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -11,6 +12,58 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type measuredAuthenticationBody struct {
|
||||||
|
reader io.Reader
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *measuredAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n, err := b.reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
func (b *measuredAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestAuthenticationResponseRecognitionBound(t *testing.T) {
|
||||||
|
const maxAuthenticationResponseBytes = 1 << 20
|
||||||
|
valid := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
declared int64
|
||||||
|
status int
|
||||||
|
overflow bool
|
||||||
|
}{
|
||||||
|
{"exact limit", maxAuthenticationResponseBytes, -1, 200, false},
|
||||||
|
{"overflow unknown length", maxAuthenticationResponseBytes + 10, -1, 200, true},
|
||||||
|
{"overflow understated length", maxAuthenticationResponseBytes + 10, 1, 200, true},
|
||||||
|
{"unauthorized does not read", maxAuthenticationResponseBytes + 10, -1, 401, false},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
body := &measuredAuthenticationBody{reader: strings.NewReader(valid + strings.Repeat(" ", test.size-len(valid)))}
|
||||||
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: test.status, Header: make(http.Header), Body: body, ContentLength: test.declared}, nil
|
||||||
|
})
|
||||||
|
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"})
|
||||||
|
if err != nil || !body.closed {
|
||||||
|
t.Fatalf("outcome = %+v, err = %v, closed = %v", outcome, err, body.closed)
|
||||||
|
}
|
||||||
|
if test.overflow {
|
||||||
|
if outcome.reason != domain.SmokeTestReasonAuthenticationResponseLimitExceeded || outcome.failure != domain.ExecutionFailureOther || outcome.credentialsAccepted != nil || body.read != maxAuthenticationResponseBytes+1 {
|
||||||
|
t.Fatalf("overflow = %+v, read = %d", outcome, body.read)
|
||||||
|
}
|
||||||
|
} else if test.status == 401 {
|
||||||
|
if body.read != 0 || outcome.credentialsAccepted == nil || *outcome.credentialsAccepted {
|
||||||
|
t.Fatalf("unauthorized = %+v, read = %d", outcome, body.read)
|
||||||
|
}
|
||||||
|
} else if outcome.credentialsAccepted == nil || !*outcome.credentialsAccepted {
|
||||||
|
t.Fatalf("exact limit = %+v", outcome)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
|
func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
|
||||||
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
t.Fatal("authentication applicability unexpectedly dispatched")
|
t.Fatal("authentication applicability unexpectedly dispatched")
|
||||||
@@ -159,21 +212,26 @@ func TestAuthenticationProbeInterpretsOpenRouterResponses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) {
|
func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) {
|
||||||
for _, body := range []string{
|
for _, tc := range []struct {
|
||||||
`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`,
|
body string
|
||||||
`{"data":[]}`,
|
accepted bool
|
||||||
`{"data":{"label":"member","is_free_tier":false,"usage":null}}`,
|
}{
|
||||||
`{"data":{"label":"member","is_free_tier":0,"usage":1}}`,
|
{`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`, false},
|
||||||
`{"data":{"is_free_tier":false,"usage":1}}`,
|
{`{"data":[]}`, false},
|
||||||
`{"data":{"label":null,"is_free_tier":false,"usage":1}}`,
|
{`{"data":{"label":"member","is_free_tier":false,"usage":null}}`, false},
|
||||||
|
{`{"data":{"label":"member","is_free_tier":0,"usage":1}}`, false},
|
||||||
|
{`{"data":{"is_free_tier":false,"usage":1}}`, false},
|
||||||
|
{`{"data":{"label":null,"is_free_tier":false,"usage":1}}`, false},
|
||||||
|
{`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`, true},
|
||||||
} {
|
} {
|
||||||
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(body)); err != nil || recognized {
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
t.Errorf("recognized invalid account response %s", body)
|
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: ioNopCloser(tc.body)}, nil
|
||||||
|
})
|
||||||
|
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"})
|
||||||
|
if err != nil || (outcome.credentialsAccepted != nil && *outcome.credentialsAccepted) != tc.accepted {
|
||||||
|
t.Errorf("account response %s: outcome = %+v, error = %v", tc.body, outcome, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`)); err != nil || !recognized {
|
|
||||||
t.Fatal("valid minimal identifying response was rejected")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) {
|
func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) {
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ type smokeInferenceCase struct {
|
|||||||
kind domain.SmokeTestKind
|
kind domain.SmokeTestKind
|
||||||
request domain.GenerateRequest
|
request domain.GenerateRequest
|
||||||
expectedContent string
|
expectedContent string
|
||||||
jsonValidator validate.Validator
|
jsonValidation validate.PreparedValidation
|
||||||
schemaValidation validate.PreparedValidation
|
schemaValidation validate.PreparedValidation
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -52,11 +52,13 @@ func prepareSmokeInferenceCase(
|
|||||||
}, nil
|
}, nil
|
||||||
case domain.SmokeTestStructuredOutput:
|
case domain.SmokeTestStructuredOutput:
|
||||||
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
||||||
preparer, ok := validator.(validate.ValidationPreparer)
|
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||||
if !ok {
|
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
|
||||||
return nil, fmt.Errorf("%w: private validator does not support prepared validation", ErrValidation)
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
plan, err := preparer.PrepareValidation(ctx, domain.OutputContract{
|
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||||
Format: domain.FormatJSON,
|
Format: domain.FormatJSON,
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
SchemaPath: smokeStructuredSchemaPath,
|
SchemaPath: smokeStructuredSchemaPath,
|
||||||
@@ -68,7 +70,7 @@ func prepareSmokeInferenceCase(
|
|||||||
return &smokeInferenceCase{
|
return &smokeInferenceCase{
|
||||||
kind: kind,
|
kind: kind,
|
||||||
expectedContent: smokeStructuredExpectedContent,
|
expectedContent: smokeStructuredExpectedContent,
|
||||||
jsonValidator: validate.NewStandardValidator(""),
|
jsonValidation: jsonPlan,
|
||||||
schemaValidation: plan,
|
schemaValidation: plan,
|
||||||
request: newGenerationRequest(
|
request: newGenerationRequest(
|
||||||
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
|
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
|
||||||
@@ -101,10 +103,7 @@ func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.Gene
|
|||||||
return facts, nil
|
return facts, nil
|
||||||
case domain.SmokeTestStructuredOutput:
|
case domain.SmokeTestStructuredOutput:
|
||||||
artifact := domain.Artifact{Body: []byte(response.Content)}
|
artifact := domain.Artifact{Body: []byte(response.Content)}
|
||||||
jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{
|
jsonResult, err := c.jsonValidation.Validate(ctx, &artifact)
|
||||||
Format: domain.FormatJSON,
|
|
||||||
ValidationMode: domain.ValidationJSON,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
|
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user