Compare commits
92 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4f3212bfb6 | |||
| 4faf43cc29 | |||
| c9663ec63c | |||
| d8c98ac575 | |||
| 5ee602cc4b | |||
| 96c619f7d4 | |||
| b3b5ec3820 | |||
| 5dd1418953 | |||
| f5c900cb0a | |||
| c5f3de13f3 | |||
| 88725e79ab | |||
| 6c015a17fa | |||
| 9463c9463a | |||
| 1fbd175e53 | |||
| 4b4ce43796 | |||
| d78474a3bf | |||
| acaf2ccd8b | |||
| 97e5739bfb | |||
| 90f796aef7 | |||
| 0dec3dcec0 | |||
| 448e64dc40 | |||
| de46935a0e | |||
| c69703b2b6 | |||
| f660b6e0fc | |||
| 5f56c65427 | |||
| cfa3a85308 | |||
| 1ff6d778af | |||
| 3f93763c9c | |||
| 4aeb58f913 | |||
| ac1e150168 | |||
| ff57dbf8b3 | |||
| beb18cc9d6 | |||
| 7203e5d6a1 | |||
| 356563a6e5 | |||
| 6673284312 | |||
| 6f42e1e007 | |||
| cee9bbc7b2 | |||
| 7db5572b62 | |||
| a91bc1608b | |||
| e7a3665443 | |||
| 59e8010623 | |||
| ada5377f32 | |||
| df208e132b | |||
| 8c0fb15727 | |||
| 2e2fba4993 | |||
| 68668cee56 | |||
| f745b14b71 | |||
| dbba1485a7 | |||
| 7c435032b6 | |||
| e834e7d806 | |||
| a4901cbb0e | |||
| 5dcfc2e953 | |||
| a02561d40f | |||
| 6112ee8fb4 | |||
| 0aceff8c08 | |||
| cddb2b389c | |||
| e056cb1607 | |||
| 1e122f0473 | |||
| 2f0f5fa607 | |||
| 7d500ee62f | |||
| 7be677c80b | |||
| 1c075bd510 | |||
| b8dc8a49a9 | |||
| cd373ee46b | |||
| e881bbdcf4 | |||
| 106c808442 | |||
| 4ec35f5ca6 | |||
| 6a1a32ae49 | |||
| e85ea2db35 | |||
| 3ef2a9e18e | |||
| a85a6788b4 | |||
| 21fea46fb3 | |||
| 62bf82a56b | |||
| bd3dfa992c | |||
| 425f53a62e | |||
| fcc94eb9e9 | |||
| 95730a3c11 | |||
| c542b81666 | |||
| e8e112186e | |||
| aa08d46ef6 | |||
| 668d09997a | |||
| 69203dcb12 | |||
| 3b760310a6 | |||
| b39460a507 | |||
| 5177db0611 | |||
| 31f1e6a89d | |||
| f7085d76f6 | |||
| cf0bfd4d76 | |||
| 6b1323d1fe | |||
| fb87fabc2e | |||
| 5adc1c1e64 | |||
| 0d27252999 |
43
artifact_ownership_contract_test.go
Normal file
43
artifact_ownership_contract_test.go
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type independentArtifactReader struct{}
|
||||||
|
|
||||||
|
func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
return &promptkit.Artifact{Body: []byte(ref.Body), Hash: ref.Body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
|
||||||
|
input := configkit.Input{Source: configkit.Mapping("ownership", map[string]any{
|
||||||
|
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
|
||||||
|
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithArtifactReader(independentArtifactReader{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var group sync.WaitGroup
|
||||||
|
for i := range 24 {
|
||||||
|
group.Go(func() {
|
||||||
|
value := fmt.Sprintf("body-%d", i)
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"document": promptkit.Inline(value)}})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("prepare: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != value || prepared.InputHashes["document"] != value {
|
||||||
|
t.Errorf("prepared = %+v", prepared)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
group.Wait()
|
||||||
|
}
|
||||||
@@ -78,7 +78,8 @@ type AuthenticationProbeConfig struct {
|
|||||||
// It encodes fields without validating them; NewEngine validates the final
|
// It encodes fields without validating them; NewEngine validates the final
|
||||||
// effective configuration after all selected source layers are combined.
|
// effective configuration after all selected source layers are combined.
|
||||||
// Mapping captures a copy of supported input values at creation; unsupported
|
// Mapping captures a copy of supported input values at creation; unsupported
|
||||||
// values fail when the source is loaded.
|
// values fail when the source is loaded. ExtraParams must use ConfigKit Mapping
|
||||||
|
// input forms; callers must explicitly convert numbers or containers as needed.
|
||||||
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||||
definitions := make([]any, 0, len(backends))
|
definitions := make([]any, 0, len(backends))
|
||||||
for _, config := range backends {
|
for _, config := range backends {
|
||||||
@@ -96,7 +97,7 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
|||||||
fields["queue_capacity"] = *config.QueueCapacity
|
fields["queue_capacity"] = *config.QueueCapacity
|
||||||
}
|
}
|
||||||
if config.ExtraParams != nil {
|
if config.ExtraParams != nil {
|
||||||
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
|
fields["extra_params"] = config.ExtraParams
|
||||||
}
|
}
|
||||||
if config.SmokeTests != nil {
|
if config.SmokeTests != nil {
|
||||||
smokeTests := make(map[string]any)
|
smokeTests := make(map[string]any)
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ package promptkit_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"reflect"
|
||||||
"strings"
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
@@ -58,11 +58,10 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
|
|||||||
if target := inspect("openrouter/profile"); target.BackendID != "openrouter/local" || target.Model != "collection-model" {
|
if target := inspect("openrouter/profile"); target.BackendID != "openrouter/local" || target.Model != "collection-model" {
|
||||||
t.Fatalf("collection = %+v", target)
|
t.Fatalf("collection = %+v", target)
|
||||||
}
|
}
|
||||||
for _, tier := range []string{"low", "medium", "high", "xhigh", "max"} {
|
target := inspect("auto-low")
|
||||||
target := inspect("auto-" + tier)
|
wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
|
||||||
if target.BackendID != "other" || target.Model != "other-model" || !strings.Contains(fmt.Sprint(target.ExtraParams), "cost_tier:"+tier) || !strings.Contains(fmt.Sprint(target.ExtraParams), "id:auto-router") {
|
if target.BackendID != "other" || target.Model != "other-model" || !reflect.DeepEqual(target.ExtraParams, wantPlugins) {
|
||||||
t.Errorf("tier %s = %+v", tier, target)
|
t.Errorf("inherited tier = %+v", target)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,11 +69,11 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
|
|||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name, mode, id string
|
name, mode, id string
|
||||||
wantErr bool
|
wantErr bool
|
||||||
count int
|
hasAuto bool
|
||||||
}{
|
}{
|
||||||
{"duplicate append", "append", "auto", true, 0},
|
{"duplicate append", "append", "auto", true, false},
|
||||||
{"distinct append", "append", "custom", false, 18},
|
{"distinct append", "append", "custom", false, true},
|
||||||
{"replace", "replace", "custom", false, 1},
|
{"replace", "replace", "custom", false, false},
|
||||||
} {
|
} {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
|
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
|
||||||
@@ -92,7 +91,7 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles, err := engine.ListProfiles(context.Background())
|
profiles, err := engine.ListProfiles(context.Background())
|
||||||
if err != nil || len(profiles) != test.count {
|
if err != nil || !slices.Contains(profiles, "custom") || slices.Contains(profiles, "auto") != test.hasAuto {
|
||||||
t.Fatalf("profiles = %v, %v", profiles, err)
|
t.Fatalf("profiles = %v, %v", profiles, err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -51,8 +51,7 @@ func TestBundledProfilePreparationAndExecution(t *testing.T) {
|
|||||||
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
func TestBundledProfileHTTPPayloads(t *testing.T) {
|
||||||
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
for _, test := range []struct{ id, model, tier string }{
|
for _, test := range []struct{ id, model, tier string }{
|
||||||
{"auto-low", "openrouter/auto", "low"}, {"auto-medium", "openrouter/auto", "medium"}, {"auto-high", "openrouter/auto", "high"}, {"auto-xhigh", "openrouter/auto", "xhigh"}, {"auto-max", "openrouter/auto", "max"},
|
{"auto-low", "openrouter/auto", "low"}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
||||||
{"auto", "openrouter/auto", ""}, {"free", "openrouter/free", ""}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
|
|
||||||
} {
|
} {
|
||||||
t.Run(test.id, func(t *testing.T) {
|
t.Run(test.id, func(t *testing.T) {
|
||||||
var body map[string]any
|
var body map[string]any
|
||||||
|
|||||||
@@ -51,23 +51,16 @@ func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
want := 0
|
if (len(profiles) > 0) != test.enabled || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
||||||
if test.enabled {
|
|
||||||
want = 17
|
|
||||||
}
|
|
||||||
if len(profiles) != want || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
|
|
||||||
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
|
||||||
}
|
}
|
||||||
if test.enabled {
|
if test.enabled {
|
||||||
wantProfiles := []string{"auto", "auto-high", "auto-low", "auto-max", "auto-medium", "auto-xhigh", "claude-fable-latest", "claude-haiku-latest", "claude-opus-latest", "claude-sonnet-latest", "free", "gemini-flash-latest", "gemini-pro-latest", "gpt-astra-latest", "gpt-luna-latest", "gpt-sol-latest", "gpt-terra-latest"}
|
if !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
||||||
if !reflect.DeepEqual(profiles, wantProfiles) || !reflect.DeepEqual(backends, []string{"openrouter"}) {
|
|
||||||
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
t.Fatalf("bundle inventory: %v %v", profiles, backends)
|
||||||
}
|
}
|
||||||
for id, model := range map[string]string{"auto": "openrouter/auto", "auto-max": "openrouter/auto", "gpt-luna-latest": "~openai/gpt-luna-latest", "free": "openrouter/free"} {
|
profile, err := engine.InspectProfile(context.Background(), "auto")
|
||||||
profile, err := engine.InspectProfile(context.Background(), id)
|
if err != nil || profile.EffectiveModelParams.BackendID != "openrouter" {
|
||||||
if err != nil || profile.EffectiveModelParams.Model != model || profile.EffectiveModelParams.BackendID != "openrouter" {
|
t.Fatalf("bundle profile = %+v, %v", profile, err)
|
||||||
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package promptkit_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"reflect"
|
"reflect"
|
||||||
@@ -13,6 +14,45 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type cancelingAuthenticationBody struct {
|
||||||
|
remaining int
|
||||||
|
cancel context.CancelFunc
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *cancelingAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n := min(len(p), b.remaining)
|
||||||
|
for i := range p[:n] {
|
||||||
|
p[i] = 'x'
|
||||||
|
}
|
||||||
|
b.remaining -= n
|
||||||
|
if b.remaining == 0 {
|
||||||
|
b.cancel()
|
||||||
|
return n, context.Canceled
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
func (b *cancelingAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestAuthenticationReadCancellationWinsOverOverflow(t *testing.T) {
|
||||||
|
for _, size := range []int{1024, (1 << 20) + 1} {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
body := &cancelingAuthenticationBody{remaining: size, cancel: cancel}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter"})
|
||||||
|
cancel()
|
||||||
|
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != promptkit.ExecutionFailureCanceled || report.Facts.CredentialsAccepted != nil || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || !body.closed {
|
||||||
|
t.Fatalf("size %d: report=%+v error=%v closed=%v", size, report, err, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
func TestBundledSmokeHTTPDefaults(t *testing.T) {
|
||||||
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
|
||||||
var urls []string
|
var urls []string
|
||||||
@@ -125,3 +165,17 @@ func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBundledSmokeAuthenticationResponseOverflow(t *testing.T) {
|
||||||
|
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(strings.Repeat("x", (1<<20)+1)))}, nil
|
||||||
|
})
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter", APIKey: "private-key-marker"})
|
||||||
|
if err != nil || report == nil || !report.Completed || report.Reason != promptkit.SmokeTestReasonAuthenticationResponseLimitExceeded || report.Failure != promptkit.ExecutionFailureOther || report.Facts.CredentialsAccepted != nil {
|
||||||
|
t.Fatalf("overflow = %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
83
convert.go
83
convert.go
@@ -2,7 +2,6 @@ package promptkit
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
@@ -395,88 +394,10 @@ func copyAnyMap(src map[string]any) map[string]any {
|
|||||||
if src == nil {
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := make(map[string]any, len(src))
|
return jsonvalue.CloneTrusted(src).(map[string]any)
|
||||||
for k, v := range src {
|
|
||||||
out[k] = copyAny(v)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyAny(value any) any {
|
func copyAny(value any) any { return jsonvalue.CloneTrusted(value) }
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
switch v := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
return copyAnyMap(v)
|
|
||||||
case []any:
|
|
||||||
out := make([]any, len(v))
|
|
||||||
for i, item := range v {
|
|
||||||
out[i] = copyAny(item)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case []string:
|
|
||||||
return copyStringSlice(v)
|
|
||||||
case []byte:
|
|
||||||
return copyBytes(v)
|
|
||||||
default:
|
|
||||||
return copyReflectValue(reflect.ValueOf(value)).Interface()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyReflectValue(value reflect.Value) reflect.Value {
|
|
||||||
if !value.IsValid() {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
switch value.Kind() {
|
|
||||||
case reflect.Interface:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
copied := copyReflectValue(value.Elem())
|
|
||||||
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
|
||||||
return copied
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
out.Set(copied)
|
|
||||||
return out
|
|
||||||
case reflect.Pointer:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.New(value.Type().Elem())
|
|
||||||
out.Elem().Set(copyReflectValue(value.Elem()))
|
|
||||||
return out
|
|
||||||
case reflect.Map:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
|
||||||
iter := value.MapRange()
|
|
||||||
for iter.Next() {
|
|
||||||
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Slice:
|
|
||||||
if value.IsNil() {
|
|
||||||
return reflect.Zero(value.Type())
|
|
||||||
}
|
|
||||||
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case reflect.Array:
|
|
||||||
out := reflect.New(value.Type()).Elem()
|
|
||||||
for i := 0; i < value.Len(); i++ {
|
|
||||||
out.Index(i).Set(copyReflectValue(value.Index(i)))
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
default:
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyStringSlice(src []string) []string {
|
func copyStringSlice(src []string) []string {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
|
|||||||
@@ -139,8 +139,10 @@ and diffs expose resolved values, recorded normalization and provenance.
|
|||||||
can apply further overrides, so the views are not promised to be identical.
|
can apply further overrides, so the views are not promised to be identical.
|
||||||
|
|
||||||
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
|
||||||
`prompt_version` JSON members. Prompt hashes are opaque equality values and may
|
`prompt_version` JSON members. Prompt definition hashes are opaque equality
|
||||||
change when the removed field no longer participates in hashing.
|
values and may also change because redundant captured fields were removed.
|
||||||
|
Invalidate persisted caches or comparisons that span this change; hashes from
|
||||||
|
inspection and preparation of the same captured definition still agree.
|
||||||
|
|
||||||
## Identity Mapping
|
## Identity Mapping
|
||||||
|
|
||||||
|
|||||||
@@ -178,7 +178,12 @@ input := configkit.Input{Source: document}
|
|||||||
```
|
```
|
||||||
|
|
||||||
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
`ProfileSource` and `BackendSource` are smaller typed conveniences when null
|
||||||
is not needed. Backend and profile definitions are supplied by the caller;
|
is not needed. Their provider maps use ConfigKit Mapping input forms: built-in
|
||||||
|
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
|
||||||
|
typed containers explicitly to an appropriate supported value, choosing numeric
|
||||||
|
precision for your application. Mapping captures supported values immediately;
|
||||||
|
unsupported or cyclic values are reported when loaded. Runtime override values
|
||||||
|
follow their separate rules. Backend and profile definitions are supplied by the caller;
|
||||||
there is no registration-only backend namespace.
|
there is no registration-only backend namespace.
|
||||||
|
|
||||||
When configuration or message files change, build and validate a new engine,
|
When configuration or message files change, build and validate a new engine,
|
||||||
@@ -714,18 +719,19 @@ if err != nil {
|
|||||||
}
|
}
|
||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
target := inspection.EffectiveModelParams
|
||||||
|
if inspection.APIKeyRequired {
|
||||||
|
// Arrange a usable direct or configured environment credential.
|
||||||
|
}
|
||||||
if target.APIKeyEnv != "" {
|
if target.APIKeyEnv != "" {
|
||||||
// This is a configured optional environment lookup source.
|
// This is the configured environment lookup source, not its value.
|
||||||
} else if inspection.APIKeyRequired {
|
|
||||||
// Arrange a direct credential before later execution.
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Use this configuration-time boundary when only the profile and its target need
|
Use this configuration-time boundary when only the profile and its target need
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
or rendering work; use prepared execution when that work must remain tied to a
|
||||||
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
later execution. A reported `APIKeyEnv` identifies a configured source, while
|
||||||
`APIKeyRequired` is the explicit local requirement. The
|
`APIKeyRequired` is the explicit requirement; both may be present. The
|
||||||
[credential format reference](../formats.md#credentials) and the method's
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
contracts.
|
contracts.
|
||||||
@@ -879,6 +885,12 @@ secret-handling policies.
|
|||||||
|
|
||||||
## Extension Interfaces
|
## Extension Interfaces
|
||||||
|
|
||||||
|
JSON Schema validation and native structured output use a self-contained root
|
||||||
|
document. Internal references are supported; references to other files or
|
||||||
|
network resources fail during preparation. `WithSchemaFile` selects one root
|
||||||
|
file, while `WithSchemaFS` and `SchemaDir` select a root source. Prepared
|
||||||
|
executions retain their captured schema even if the source later changes.
|
||||||
|
|
||||||
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||||
behavior does not fit the application. Their GoDoc defines concurrent use,
|
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||||
context handling, ownership of copied values, nil responses, and preservation
|
context handling, ownership of copied values, nil responses, and preservation
|
||||||
@@ -886,6 +898,11 @@ of collaborator errors. Implementations must honor cancellation, safely manage
|
|||||||
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||||
the application policy that motivated the injection.
|
the application policy that motivated the injection.
|
||||||
|
|
||||||
|
A successful `ArtifactReader.Read` transfers the returned artifact and mutable
|
||||||
|
body storage to Promptkit. Readers that previously reused a returned buffer
|
||||||
|
must instead return independent storage and stop mutating it after return;
|
||||||
|
private scratch buffers may still be reused when the returned bytes are copied.
|
||||||
|
|
||||||
## Handle Errors
|
## Handle Errors
|
||||||
|
|
||||||
Use `errors.Is` with the
|
Use `errors.Is` with the
|
||||||
|
|||||||
@@ -58,6 +58,14 @@ exist.
|
|||||||
|
|
||||||
## Maintainer Validation
|
## Maintainer Validation
|
||||||
|
|
||||||
|
Use the newest Go installation already available locally for ordinary
|
||||||
|
development checks. Put its `bin` directory first on `PATH` and set
|
||||||
|
`GOTOOLCHAIN=local` to keep that toolchain selected. For repository-isolated
|
||||||
|
checks, set `GOWORK=off`; consumers may use their own workspaces. The selected
|
||||||
|
validation patch does not by itself change the module's consumer minimum. See
|
||||||
|
the [release toolchain policy](policy/release.md#consumer-minimum-and-release-toolchain)
|
||||||
|
and [release procedure](release.md#prepare-the-candidate) for release selection.
|
||||||
|
|
||||||
For ordinary work, select validation proportionate to the change. Do not repeat
|
For ordinary work, select validation proportionate to the change. Do not repeat
|
||||||
successful checks for unchanged content merely to create a commit. During
|
successful checks for unchanged content merely to create a commit. During
|
||||||
development, run relevant Go tests and vet, format changed Go files, and execute
|
development, run relevant Go tests and vet, format changed Go files, and execute
|
||||||
|
|||||||
@@ -418,11 +418,17 @@ generic inheritance and layer mechanics, consult
|
|||||||
## Schemas
|
## Schemas
|
||||||
|
|
||||||
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
Schemas are JSON documents selected by a prompt or request `schema_path`.
|
||||||
Directory and `fs.FS` schema paths resolve within the configured root;
|
Directory and `fs.FS` schema paths resolve within the configured root. Each
|
||||||
referenced nested schemas resolve relative to the owning schema document.
|
schema must be self-contained: in-document references, IDs, and anchors are
|
||||||
|
supported, but references requiring another file or network resource fail.
|
||||||
|
References must also work without the source filename as a base; use fragment
|
||||||
|
references or explicit schema IDs rather than relying on that filename.
|
||||||
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
`WithSchemaFile` exposes one schema under its base name. An unreadable,
|
||||||
invalid, or unresolvable schema causes an operational validation error.
|
invalid, or unresolvable schema causes an operational validation error.
|
||||||
Valid JSON that fails the schema instead yields a failed validation result.
|
Valid JSON that fails the schema instead yields a failed validation result.
|
||||||
|
Schema documents are detached without applying the provider extra-parameter
|
||||||
|
copy limits; this does not change the schema decoder, compiler, or
|
||||||
|
self-contained-resource rules.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
|
|||||||
@@ -16,14 +16,13 @@ Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
|||||||
Before the client is called, the engine resolves framework, backend, profile,
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
and request values into one execution target. Endpoint configuration is trimmed
|
and request values into one execution target. Endpoint configuration is trimmed
|
||||||
and must be an absolute HTTP or HTTPS URL with a host and without user
|
and must be an absolute HTTP or HTTPS URL with a host and without user
|
||||||
information, a query, or a fragment. A non-empty endpoint from the target
|
information, a query, or a fragment. The target's endpoint is validated again
|
||||||
overrides the client's configured base URL. The final selected endpoint is
|
before transport.
|
||||||
validated again before transport.
|
|
||||||
|
|
||||||
The completion URL is composed through parsed URL path operations. Nested base
|
The completion URL is composed through parsed URL path operations. Nested base
|
||||||
paths are retained, repeated trailing slashes are normalized, and the result
|
paths are retained, repeated trailing slashes are normalized, and the result
|
||||||
has exactly one appended `/chat/completions` suffix. Generation fails before
|
has exactly one appended `/chat/completions` suffix. Generation fails before
|
||||||
sending when neither source supplies a valid endpoint.
|
sending when the target does not supply a valid endpoint.
|
||||||
|
|
||||||
The target's backend ID is routing metadata for prepared values, results, and
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
injected clients. The built-in client does not derive the URL from that ID and
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
@@ -70,7 +69,9 @@ and numeric `usage`; unrelated fields and account limits are ignored. This is
|
|||||||
only a minimal identifying subset of OpenRouter's documented
|
only a minimal identifying subset of OpenRouter's documented
|
||||||
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
|
||||||
not validation of the full account document. Other statuses and unrecognized
|
not validation of the full account document. Other statuses and unrecognized
|
||||||
200 responses do not establish acceptance.
|
200 responses do not establish acceptance. A 200 response exceeding the bounded
|
||||||
|
recognition allowance reports `authentication_response_limit_exceeded`, leaving
|
||||||
|
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
|
||||||
|
|
||||||
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||||
RoundTripper boundary and any response associated with it. It includes the
|
RoundTripper boundary and any response associated with it. It includes the
|
||||||
@@ -87,8 +88,7 @@ transport for HTTP-only diagnostics.
|
|||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
target's model takes precedence over the client's configured model, and one
|
target's model must be available.
|
||||||
must be available.
|
|
||||||
|
|
||||||
Each ordinary message contains its `role` and string `content`. A
|
Each ordinary message contains its `role` and string `content`. A
|
||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
@@ -115,9 +115,10 @@ The client conditionally includes:
|
|||||||
strict flag, and schema document. Plain JSON validation does not add an
|
strict flag, and schema document. Plain JSON validation does not add an
|
||||||
object-only response constraint.
|
object-only response constraint.
|
||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
Configured profile maps recursively overlay backend defaults, while a nonempty
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
runtime override replaces the complete configured map. The [format reference](../formats.md#defaults-and-overrides)
|
||||||
merged into the top-level body. Empty keys, values that cannot be encoded, and
|
owns these resolution rules. The resulting effective map is merged into the
|
||||||
|
top-level body. Empty keys, values that cannot be encoded, and
|
||||||
collisions with these reserved fields are rejected before any provider call:
|
collisions with these reserved fields are rejected before any provider call:
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
|
|||||||
@@ -68,9 +68,8 @@ Preparation and validation never hold an active permit.
|
|||||||
|
|
||||||
The manager's `Acquire` operation owns that active-permit acquisition and
|
The manager's `Acquire` operation owns that active-permit acquisition and
|
||||||
returns an idempotent release function. The coordinator uses it around
|
returns an idempotent release function. The coordinator uses it around
|
||||||
individual attempts. The earlier one-shot wrapper still delegates to the same
|
individual attempts. There is one queue and one active limit per limited backend
|
||||||
operation for focused internal callers and tests, but engine assembly no
|
pool.
|
||||||
longer uses it. There is one queue and one active limit per limited backend pool.
|
|
||||||
|
|
||||||
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
When all active permits are occupied, calls join a mutex-protected FIFO waiter
|
||||||
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
list. Releasing a permit transfers it directly to the oldest remaining waiter
|
||||||
@@ -121,7 +120,7 @@ validation, bounded admission, idempotent release, context handling, and
|
|||||||
unlimited admission. The
|
unlimited admission. The
|
||||||
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
[client tests](../../internal/capacity/client_test.go) own peak enforcement,
|
||||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
unlimited calls, passthrough behavior, and panic release.
|
unlimited calls, and panic release using test-only one-shot calls.
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
admission, lease lifetime, failure release, and shared initial/repair
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
|
|||||||
@@ -26,10 +26,9 @@ client uses its endpoint, credential metadata, generation fields, and extra
|
|||||||
parameters. `BackendID` remains routing metadata
|
parameters. `BackendID` remains routing metadata
|
||||||
for the generation boundary and is not mapped into the provider payload.
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
Construction trims and validates a nonempty configured base URL and clones any
|
Construction clones any supplied `http.Client` so Promptkit can apply its
|
||||||
supplied `http.Client` so Promptkit can apply its timeout default without
|
timeout default without mutating the caller's client. Generation preparation
|
||||||
mutating the caller's client. An empty configured base remains valid because a
|
then:
|
||||||
resolved request target may supply the endpoint. Generation preparation then:
|
|
||||||
|
|
||||||
1. validates shared execution-setting invariants and the final selected base
|
1. validates shared execution-setting invariants and the final selected base
|
||||||
endpoint;
|
endpoint;
|
||||||
@@ -42,6 +41,10 @@ resolved request target may supply the endpoint. Generation preparation then:
|
|||||||
the complete-request failure remains a request failure;
|
the complete-request failure remains a request failure;
|
||||||
6. resolves authentication and freezes the URL, body, and optional credential.
|
6. resolves authentication and freezes the URL, body, and optional credential.
|
||||||
|
|
||||||
|
Provider values are detached with trusted copying before encoding. Custom JSON
|
||||||
|
encoding methods and per-value encoding-failure diagnostics operate on those
|
||||||
|
private copies rather than retained prepared or diagnostic metadata.
|
||||||
|
|
||||||
Each execution of that prepared request creates a fresh HTTP request, body
|
Each execution of that prepared request creates a fresh HTTP request, body
|
||||||
reader, and attempt timeout context. It closes the response body and returns
|
reader, and attempt timeout context. It closes the response body and returns
|
||||||
the HTTP status, decoded response or error, reported token values and presence,
|
the HTTP status, decoded response or error, reported token values and presence,
|
||||||
@@ -77,7 +80,8 @@ not retain headers, credentials, raw request or response bodies, or a redirect
|
|||||||
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
trace. These are observations at Promptkit's RoundTripper boundary, not proof
|
||||||
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
that bytes reached a peer or that a custom transport did no hidden dispatch.
|
||||||
Reachability and named authentication probes use the same configured built-in
|
Reachability and named authentication probes use the same configured built-in
|
||||||
client but discard their response bodies after local recognition. The
|
client. The OpenRouter adapter owns endpoint derivation, bounded account-response
|
||||||
|
recognition, and body closure; the smoke coordinator interprets its evidence. The
|
||||||
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
|
||||||
owns their observable HTTP semantics.
|
owns their observable HTTP semantics.
|
||||||
|
|
||||||
@@ -105,14 +109,13 @@ public ownership and redaction semantics belong to the
|
|||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid generation
|
||||||
configuration, invalid generation requests, request execution failures,
|
requests, request execution failures, non-success provider statuses, and
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
malformed successful responses. Provider response bodies are never exposed in
|
||||||
response bodies are never exposed in raw form through non-success errors.
|
raw form through non-success errors.
|
||||||
|
|
||||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
A missing or invalid final selected endpoint is an invalid generation request
|
||||||
invalid final selected endpoint is an invalid generation request and is
|
and is rejected before transport.
|
||||||
rejected before transport.
|
|
||||||
|
|
||||||
Authentication resolves a trimmed direct key before a trimmed configured
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
environment value. Optional missing, empty, or whitespace-only sources do not
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ lifecycle instrumentation spans the facade and runner boundaries.
|
|||||||
Prepared execution starts only after its private payload is atomically claimed;
|
Prepared execution starts only after its private payload is atomically claimed;
|
||||||
the facade receives that invocation's recorder privately and emits the terminal
|
the facade receives that invocation's recorder privately and emits the terminal
|
||||||
event after public error mapping. Failed claims and preparation emit nothing.
|
event after public error mapping. Failed claims and preparation emit nothing.
|
||||||
|
The public projection retains private operation-owner metadata separately from
|
||||||
|
inspectable child causes. Terminal classification uses that owner, preserving
|
||||||
|
context precedence and generation-specific transport/provider details rather
|
||||||
|
than mistaking a collaborator's nested lookup error for the parent's operation.
|
||||||
|
|
||||||
The facade masks inherited recorder and prepared-claim state when entering
|
The facade masks inherited recorder and prepared-claim state when entering
|
||||||
preparation or execution. Delegated calls from an injected collaborator retain
|
preparation or execution. Delegated calls from an injected collaborator retain
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ contributor workflow and validation.
|
|||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
|
|||||||
@@ -42,10 +42,11 @@ Ordinary preparation selects the full prompt definition and construction-time
|
|||||||
hash. It retains its early request-ID check before direct-session
|
hash. It retains its early request-ID check before direct-session
|
||||||
normalization. Referenced message content and static template syntax were
|
normalization. Referenced message content and static template syntax were
|
||||||
captured when the engine was built.
|
captured when the engine was built.
|
||||||
|
Rendering uses those captured parses, cloning them per call to bind request
|
||||||
|
input helpers; it does not parse templates during execution.
|
||||||
|
|
||||||
`InspectPrompt` selects captured metadata through an optional repository
|
`InspectPrompt` selects captured metadata directly from the repository. It
|
||||||
capability, falling back to the full definition and existing hash resolution
|
returns owned metadata without
|
||||||
for internal repositories without it. It returns owned metadata without
|
|
||||||
rendering templates or touching profile, artifact, schema, renderer, validator,
|
rendering templates or touching profile, artifact, schema, renderer, validator,
|
||||||
admission, or model collaborators. The root
|
admission, or model collaborators. The root
|
||||||
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
|
||||||
@@ -57,8 +58,7 @@ snapshot by the selected prompt ID. The analyzer receives only
|
|||||||
those identity fields rather than an execution definition. It analyzes the
|
those identity fields rather than an execution definition. It analyzes the
|
||||||
snapshot with the supplied caller variables and normalized direct-session
|
snapshot with the supplied caller variables and normalized direct-session
|
||||||
bypass; it does not resolve target readiness or enter the preparation
|
bypass; it does not resolve target readiness or enter the preparation
|
||||||
pipeline. Missing snapshot capability is a prompt-load failure rather than a
|
pipeline. An unavailable captured snapshot is a prompt-load failure. The root
|
||||||
source fallback. The root
|
|
||||||
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
|
||||||
operation's exact contract.
|
operation's exact contract.
|
||||||
|
|
||||||
@@ -142,11 +142,10 @@ one execution-precedence and error-ordering implementation while allowing a
|
|||||||
full backend pool to reject work before expensive schema, artifact, and
|
full backend pool to reject work before expensive schema, artifact, and
|
||||||
rendering operations. `Prepare` discards the plan after returning its public
|
rendering operations. `Prepare` discards the plan after returning its public
|
||||||
metadata. `Run` retains the plan through initial and repaired-output validation
|
metadata. `Run` retains the plan through initial and repaired-output validation
|
||||||
and discards it when the operation ends. Prepared execution transfers the
|
and discards it when the operation ends. Prepared execution retains one
|
||||||
completed preparation's owned message, input-hash, and copied-schema values
|
immutable, redacted metadata snapshot shared by execution and internal details.
|
||||||
into its private payload, while deep-copying mutable provider extras and
|
The facade detaches mutable public details. Claim completion or discard clears
|
||||||
creating an independent details snapshot. Claim completion or discard clears
|
execution-only payload references without modifying the retained metadata.
|
||||||
the execution payload without removing the details snapshot.
|
|
||||||
|
|
||||||
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||||
path as `Prepare` and discards the resulting internal prepared value without
|
path as `Prepare` and discards the resulting internal prepared value without
|
||||||
@@ -176,7 +175,7 @@ encoding of that session, every role and content value, and cache-control
|
|||||||
presence and values; its hexadecimal value is opaque.
|
presence and values; its hexadecimal value is opaque.
|
||||||
|
|
||||||
The configuration snapshot is read-only after engine construction. Concurrent
|
The configuration snapshot is read-only after engine construction. Concurrent
|
||||||
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
|
`Prepare` and `Run` calls use detached captured targets and keep invocation
|
||||||
state local.
|
state local.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
|
|||||||
@@ -85,6 +85,8 @@ returning and retains no source capability for public inspection.
|
|||||||
|
|
||||||
Prompt materialization shares a lazy metadata snapshot across root and collection
|
Prompt materialization shares a lazy metadata snapshot across root and collection
|
||||||
resources during construction, avoiding a full metadata copy per collection.
|
resources during construction, avoiding a full metadata copy per collection.
|
||||||
|
Input metadata, message content and templates, and output contracts are captured
|
||||||
|
separately before the definition, inspection, and hash are assembled.
|
||||||
|
|
||||||
Normalization and defaults are recorded at individual resolved field paths,
|
Normalization and defaults are recorded at individual resolved field paths,
|
||||||
including message roles and cache controls. List indexes identify fields for
|
including message roles and cache controls. List indexes identify fields for
|
||||||
@@ -146,8 +148,8 @@ change rendering behavior.
|
|||||||
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
ConfigKit resolves profile inheritance before the handoff. Within a collection,
|
||||||
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
|
||||||
The configuration adapter validates final effective resource references and
|
The configuration adapter validates final effective resource references and
|
||||||
field values, then builds target
|
field values, then captures backend definitions and profile target snapshots
|
||||||
snapshots with backend defaults and presence/suppression state. A profile can
|
with backend defaults and presence/suppression state. A profile can
|
||||||
remain partial until runtime overrides complete it. `InspectProfile` selects
|
remain partial until runtime overrides complete it. `InspectProfile` selects
|
||||||
that captured target without prompt, artifact, schema, credential lookup, or
|
that captured target without prompt, artifact, schema, credential lookup, or
|
||||||
provider work. Preparation applies runtime overrides and then checks target
|
provider work. Preparation applies runtime overrides and then checks target
|
||||||
@@ -216,12 +218,14 @@ instead of first making a response-sized string copy.
|
|||||||
|
|
||||||
Every preparation operation creates one operation-local validation plan. None,
|
Every preparation operation creates one operation-local validation plan. None,
|
||||||
basic, and JSON modes retain the effective output contract without source
|
basic, and JSON modes retain the effective output contract without source
|
||||||
access. JSON Schema mode loads the root document once, resolves and compiles
|
access. JSON Schema mode loads the root document once, compiles its declared
|
||||||
each transitive reference, and retains the compiled validator. Schema compiler
|
schema locations, and retains the compiled root validator. External resources
|
||||||
resources use canonical escaped file or private-scheme URLs; loaders decode
|
are rejected without file or network loading; in-document references remain
|
||||||
their paths once and enforce the configured source boundary. The
|
supported. Root resources use canonical escaped file or private-scheme URLs. The
|
||||||
provider-facing structured-output metadata uses the root document captured by
|
provider-facing structured-output metadata uses the root document captured by
|
||||||
the same plan.
|
the same plan. Its detached schema document is not revalidated against the
|
||||||
|
runtime provider-value copier's traversal limits. Smoke inference uses a
|
||||||
|
separate JSON-only prepared plan before checking the prepared schema plan.
|
||||||
|
|
||||||
Schema preparation and execution remain synchronous. Promptkit checks
|
Schema preparation and execution remain synchronous. Promptkit checks
|
||||||
cancellation before and after source resolution, JSON decoding, compilation,
|
cancellation before and after source resolution, JSON decoding, compilation,
|
||||||
|
|||||||
80
docs/releases/v0.19.0.md
Normal file
80
docs/releases/v0.19.0.md
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
# Promptkit v0.19.0
|
||||||
|
|
||||||
|
This release completes two rounds of audit remediation, correcting error and
|
||||||
|
diagnostic reporting and simplifying configuration, validation, and value
|
||||||
|
ownership. It includes intentional compatibility changes for custom artifact
|
||||||
|
readers, programmatic configuration sources, and schemas. The minimum supported
|
||||||
|
Go version remains 1.27.0; ConfigKit remains at v1.1.0.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
Consumers upgrading from v0.18.0 should review these changes:
|
||||||
|
|
||||||
|
- A successful `ArtifactReader.Read` transfers ownership of the returned
|
||||||
|
artifact and its mutable body storage to Promptkit. Custom readers must stop
|
||||||
|
mutating returned values through retained aliases. Readers that reuse scratch
|
||||||
|
buffers must return independent storage. Promptkit retains its defensive
|
||||||
|
body copy, but that copy does not make concurrent buffer reuse safe. See the
|
||||||
|
[extension guidance](../consumers/pkg-promptkit.md#extension-interfaces).
|
||||||
|
- Provider maps passed through `ProfileSource` and `BackendSource` now use
|
||||||
|
ConfigKit's native Mapping input forms directly: built-in scalars, `[]any`,
|
||||||
|
and `map[string]any` recursively. Convert `json.Number` and typed containers
|
||||||
|
explicitly to supported values, choosing numeric precision appropriate to
|
||||||
|
the application. The former recursive compatibility conversion is removed.
|
||||||
|
Runtime execution overrides retain their separate accepted-value behavior.
|
||||||
|
- Schemas used for local validation and native structured output must be
|
||||||
|
self-contained. Move externally referenced definitions into the root document
|
||||||
|
and use internal references or explicit schema IDs. References requiring
|
||||||
|
another file or network resource, or relying on the source filename as a base,
|
||||||
|
fail during preparation. Promptkit does not bundle external schemas. See the
|
||||||
|
[schema reference](../formats.md#schemas).
|
||||||
|
|
||||||
|
Prepared schemas no longer inherit the depth and node limits used when copying
|
||||||
|
runtime provider values. Schemas previously rejected solely by those incidental
|
||||||
|
limits may now be accepted; decoder/compiler checks and the self-contained
|
||||||
|
schema requirement still apply.
|
||||||
|
|
||||||
|
No configuration-format migration is required from v0.18.0. After addressing
|
||||||
|
any affected integrations:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get go.maximumdirect.net/promptkit@v0.19.0
|
||||||
|
```
|
||||||
|
|
||||||
|
## Errors And Diagnostics
|
||||||
|
|
||||||
|
Public errors and terminal observations now preserve the category of the
|
||||||
|
operation that failed even when an injected collaborator returns an error from
|
||||||
|
another Promptkit operation. Underlying causes remain available through
|
||||||
|
`errors.Is` and `errors.As`, without repeatedly duplicating nested cause trees.
|
||||||
|
|
||||||
|
Built-in generation preserves response-body read failures separately from
|
||||||
|
malformed response content, and failures during request preparation are
|
||||||
|
distinguished from dispatched attempts.
|
||||||
|
|
||||||
|
OpenRouter authentication diagnostics bound response recognition to 1 MiB.
|
||||||
|
An oversized HTTP 200 response reports
|
||||||
|
`SmokeTestReasonAuthenticationResponseLimitExceeded`, leaving credential
|
||||||
|
acceptance unknown. Cancellation during the read remains an interruption.
|
||||||
|
Consumers that enumerate diagnostic reasons should handle the new reason;
|
||||||
|
see the [diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||||
|
|
||||||
|
## Internal Simplification And Ownership
|
||||||
|
|
||||||
|
Prompt lookup and rendering now consistently use captured configuration, and
|
||||||
|
smoke syntax and schema checks share the prepared output-validation lifecycle.
|
||||||
|
Unused routing defaults, alternate private workflows, and duplicate wire
|
||||||
|
representations have been removed.
|
||||||
|
|
||||||
|
Provider-setting resolution, inspection, prepared details, and smoke reports
|
||||||
|
avoid redundant validation and copying of already accepted values. Prepared
|
||||||
|
execution shares one private redacted metadata snapshot while preserving
|
||||||
|
independent caller-visible values. Provider values are detached before custom
|
||||||
|
JSON encoders run, protecting retained metadata from encoder-side mutation.
|
||||||
|
|
||||||
|
Requirement analysis, namespace-filtered listings, and repair-feedback
|
||||||
|
construction also avoid unnecessary traversal and temporary allocation. These
|
||||||
|
changes preserve existing request isolation and execution behavior without
|
||||||
|
introducing new performance guarantees.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/docs/oss/attribution.md)
|
||||||
@@ -38,8 +38,17 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
No unselected ideas are currently recorded here. Deliberately postponed ideas
|
### Schema Bundling
|
||||||
remain in the [deferred catalog](deferred.md).
|
|
||||||
|
Allow consumers to author reusable schemas across multiple files and package
|
||||||
|
them into a single self-contained schema for local validation and provider
|
||||||
|
submission. This could restore multi-file authoring convenience while retaining
|
||||||
|
the [self-contained schema requirement](../formats.md#schemas).
|
||||||
|
|
||||||
|
Bundling would need to preserve reference semantics, identifiers, anchors, and
|
||||||
|
recursion. A complete bundle would not guarantee that a provider supports every
|
||||||
|
schema feature. This is a possible future enhancement; implementation is not
|
||||||
|
currently approved.
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
@@ -213,7 +213,8 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
|||||||
//
|
//
|
||||||
// Prompt schema_path values refer to the file's base name. path must name an
|
// Prompt schema_path values refer to the file's base name. path must name an
|
||||||
// existing non-directory file when NewEngine applies the option. This option
|
// existing non-directory file when NewEngine applies the option. This option
|
||||||
// replaces Config.SchemaDir and earlier schema-source options.
|
// replaces Config.SchemaDir and earlier schema-source options. The selected
|
||||||
|
// schema must be self-contained; external resources are not loaded.
|
||||||
func WithSchemaFile(path string) Option {
|
func WithSchemaFile(path string) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
return options.internal.SetSchemaFile(path)
|
return options.internal.SetSchemaFile(path)
|
||||||
@@ -311,8 +312,8 @@ func (e *Engine) InspectPrompt(
|
|||||||
// override. BackendID is empty for an endpoint-only profile.
|
// override. BackendID is empty for an endpoint-only profile.
|
||||||
//
|
//
|
||||||
// APIKeyEnv in the returned target is an environment-variable name, never its
|
// APIKeyEnv in the returned target is an environment-variable name, never its
|
||||||
// value. APIKeyRequired instead reports a direct credential requirement and is
|
// value. APIKeyRequired reports that a credential is required; an explicit
|
||||||
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
|
// profile environment source can also be present. InspectProfile neither derives
|
||||||
// an ID from a prompt default_profile nor checks credential availability, so an
|
// an ID from a prompt default_profile nor checks credential availability, so an
|
||||||
// absent or blank named environment variable is not an error.
|
// absent or blank named environment variable is not an error.
|
||||||
//
|
//
|
||||||
|
|||||||
34
errors.go
34
errors.go
@@ -13,10 +13,44 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/usecase"
|
"go.maximumdirect.net/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// publicOperationError keeps classification separate from inspectable causes.
|
||||||
|
// Cache wording so formatting nested operations does not repeatedly rebuild it.
|
||||||
|
type publicOperationError struct {
|
||||||
|
owner error
|
||||||
|
cause error
|
||||||
|
message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *publicOperationError) Error() string { return e.message }
|
||||||
|
func (e *publicOperationError) Unwrap() error { return e.cause }
|
||||||
|
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
|
||||||
|
|
||||||
func mapPublicError(err error) error {
|
func mapPublicError(err error) error {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
var operation *usecase.OperationError
|
||||||
|
if errors.As(err, &operation) && operation != nil {
|
||||||
|
owner := publicErrorFor(operation.Category)
|
||||||
|
if owner != nil {
|
||||||
|
// Project the underlying specialized error separately, retaining both
|
||||||
|
// the outer operation and the original cause for Is/As callers.
|
||||||
|
mapped := mapPublicError(operation.Cause)
|
||||||
|
if errors.Is(operation.Cause, llm.ErrTransport) {
|
||||||
|
mapped = errors.Join(ErrTransport, mapped)
|
||||||
|
}
|
||||||
|
if errors.Is(operation.Cause, llm.ErrMalformedResponse) {
|
||||||
|
mapped = errors.Join(ErrMalformedResponse, mapped)
|
||||||
|
}
|
||||||
|
// The projection already retains ordinary causes; adding the original
|
||||||
|
// again would duplicate its entire subtree at each enclosing operation.
|
||||||
|
return &publicOperationError{owner: owner, cause: mapped, message: fmt.Sprintf("%v: %v", owner, mapped)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mapUnownedPublicError(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapUnownedPublicError(err error) error {
|
||||||
var internalCapacityError *usecase.CapacityError
|
var internalCapacityError *usecase.CapacityError
|
||||||
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
|
||||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
|
|||||||
@@ -80,3 +80,12 @@ func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
|
|||||||
t.Fatalf("request encoding category = %v", err)
|
t.Fatalf("request encoding category = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorRetainsOwnerWithSpecializedCapacity(t *testing.T) {
|
||||||
|
internal := &usecase.CapacityError{BackendID: "limited"}
|
||||||
|
err := mapPublicError(&usecase.OperationError{Category: usecase.ErrLLMGenerate, Cause: internal})
|
||||||
|
var capacity *CapacityError
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) || !errors.As(err, &capacity) || capacity.BackendID != "limited" || !errors.Is(err, ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("mapped owned capacity = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,8 +3,6 @@ package artifact
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"fmt"
|
|
||||||
"mime"
|
"mime"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -153,9 +151,12 @@ func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef,
|
|||||||
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
|
||||||
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
|
||||||
}
|
}
|
||||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(want)})
|
||||||
if artifact.Hash != wantHash {
|
if err != nil {
|
||||||
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
b.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,9 +3,7 @@ package artifact
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -284,9 +282,12 @@ func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
|
|||||||
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
|
||||||
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
|
||||||
}
|
}
|
||||||
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
|
inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(tc.body)})
|
||||||
if artifact.Hash != wantHash {
|
if err != nil {
|
||||||
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inline.Hash == "" || artifact.Hash != inline.Hash {
|
||||||
|
t.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,17 +27,13 @@ type Registry struct {
|
|||||||
// NewEffectiveRegistry stores already-validated effective backend definitions
|
// NewEffectiveRegistry stores already-validated effective backend definitions
|
||||||
// under their exact configuration keys. Configuration validation and
|
// under their exact configuration keys. Configuration validation and
|
||||||
// normalization precede this ownership boundary.
|
// normalization precede this ownership boundary.
|
||||||
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) {
|
func NewEffectiveRegistry(definitions map[string]domain.Backend) *Registry {
|
||||||
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
|
||||||
for id, definition := range definitions {
|
for id, definition := range definitions {
|
||||||
definition.ID = id
|
definition.ID = id
|
||||||
cloned, err := domain.CloneBackend(definition)
|
registry.backends[id] = domain.CloneBackend(definition)
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("backend %q: %w", id, err)
|
|
||||||
}
|
|
||||||
registry.backends[id] = cloned
|
|
||||||
}
|
}
|
||||||
return registry, nil
|
return registry
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetBackend returns a defensive copy of the backend registered with id.
|
// GetBackend returns a defensive copy of the backend registered with id.
|
||||||
@@ -49,11 +45,7 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
}
|
}
|
||||||
cloned, err := domain.CloneBackend(definition)
|
return domain.CloneBackend(definition), nil
|
||||||
if err != nil {
|
|
||||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
|
||||||
}
|
|
||||||
return cloned, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CapacityPolicies returns a copy of the normalized policies for limited
|
// CapacityPolicies returns a copy of the normalized policies for limited
|
||||||
|
|||||||
@@ -22,16 +22,13 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
|||||||
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
registry := backend.NewEffectiveRegistry(map[string]domain.Backend{
|
||||||
" local.v1 ": {
|
" local.v1 ": {
|
||||||
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
Endpoint: "https://backend.example/v1", ExtraParams: params,
|
||||||
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
|
ConcurrencyLimit: 2, QueueCapacity: 3,
|
||||||
SmokeTests: smokeTests,
|
SmokeTests: smokeTests,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
params["nested"].(map[string]any)["enabled"] = false
|
params["nested"].(map[string]any)["enabled"] = false
|
||||||
defaultProfile = "mutated-profile"
|
defaultProfile = "mutated-profile"
|
||||||
reachabilityEndpoint = "https://mutated.example/ready"
|
reachabilityEndpoint = "https://mutated.example/ready"
|
||||||
@@ -79,10 +76,7 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
func TestEffectiveRegistryMissingBackend(t *testing.T) {
|
||||||
registry, err := backend.NewEffectiveRegistry(nil)
|
registry := backend.NewEffectiveRegistry(nil)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
t.Fatalf("missing backend error = %v", err)
|
t.Fatalf("missing backend error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,40 +0,0 @@
|
|||||||
package capacity
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
|
||||||
)
|
|
||||||
|
|
||||||
type client struct {
|
|
||||||
manager *Manager
|
|
||||||
next llm.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient wraps next with configured active-generation limits. A nil manager
|
|
||||||
// leaves next unchanged.
|
|
||||||
func NewClient(manager *Manager, next llm.Client) llm.Client {
|
|
||||||
if manager == nil {
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
return &client{
|
|
||||||
manager: manager,
|
|
||||||
next: next,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *client) Generate(
|
|
||||||
ctx context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
if c.manager.getPool(req.Target.BackendID) == nil {
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return c.next.Generate(ctx, req)
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,6 @@ package capacity
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"reflect"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -24,6 +23,18 @@ type clientFunc func(
|
|||||||
domain.GenerateRequest,
|
domain.GenerateRequest,
|
||||||
) (*domain.GenerateResponse, error)
|
) (*domain.GenerateResponse, error)
|
||||||
|
|
||||||
|
// newTestClient exercises one acquisition around a controlled call.
|
||||||
|
func newTestClient(manager *Manager, next llm.Client) llm.Client {
|
||||||
|
return clientFunc(func(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
release, err := manager.Acquire(ctx, req.Target.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
return next.Generate(ctx, req)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (f clientFunc) Generate(
|
func (f clientFunc) Generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -167,7 +178,7 @@ func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
|
|||||||
"second": secondRelease,
|
"second": secondRelease,
|
||||||
"third": thirdRelease,
|
"third": thirdRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
first := generateAsync(client, context.Background(), "limited", "first")
|
first := generateAsync(client, context.Background(), "limited", "first")
|
||||||
if got := receiveStarted(t, next.started); got != "first" {
|
if got := receiveStarted(t, next.started); got != "first" {
|
||||||
@@ -212,7 +223,7 @@ func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
|
|||||||
releases[string(rune('a'+i))] = gate
|
releases[string(rune('a'+i))] = gate
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
results := make([]<-chan generateResult, 0, len(releases))
|
results := make([]<-chan generateResult, 0, len(releases))
|
||||||
for id := range releases {
|
for id := range releases {
|
||||||
@@ -257,7 +268,7 @@ func TestClientRemovesCanceledWaiters(t *testing.T) {
|
|||||||
"three": make(chan struct{}),
|
"three": make(chan struct{}),
|
||||||
}
|
}
|
||||||
next := newBlockingClient(releases)
|
next := newBlockingClient(releases)
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
if got := receiveStarted(t, next.started); got != "holder" {
|
if got := receiveStarted(t, next.started); got != "holder" {
|
||||||
@@ -321,7 +332,7 @@ func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
holder := generateAsync(client, context.Background(), "limited", "holder")
|
holder := generateAsync(client, context.Background(), "limited", "holder")
|
||||||
waitForActiveCount(t, manager, "limited", 1)
|
waitForActiveCount(t, manager, "limited", 1)
|
||||||
@@ -394,7 +405,7 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
"alpha": alphaRelease,
|
"alpha": alphaRelease,
|
||||||
"beta": betaRelease,
|
"beta": betaRelease,
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
|
|
||||||
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
alpha := generateAsync(client, context.Background(), "alpha", "alpha")
|
||||||
beta := generateAsync(client, context.Background(), "beta", "beta")
|
beta := generateAsync(client, context.Background(), "beta", "beta")
|
||||||
@@ -423,63 +434,6 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
|
|||||||
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if got := NewClient(nil, next); got != next {
|
|
||||||
t.Fatal("nil manager did not return the wrapped client unchanged")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
|
|
||||||
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
|
|
||||||
"limited": {ConcurrencyLimit: 1},
|
|
||||||
})
|
|
||||||
request := domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{
|
|
||||||
SessionID: "session",
|
|
||||||
Messages: []domain.RenderedMessage{
|
|
||||||
{Role: "user", Content: "content"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Target: domain.ExecutionTarget{
|
|
||||||
BackendID: "limited",
|
|
||||||
Model: "model",
|
|
||||||
ExtraParams: map[string]any{"key": "value"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
response := &domain.GenerateResponse{
|
|
||||||
Content: "output",
|
|
||||||
Usage: domain.TokenUsage{TotalTokens: 7},
|
|
||||||
}
|
|
||||||
collaboratorErr := errors.New("collaborator failure")
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
response *domain.GenerateResponse
|
|
||||||
err error
|
|
||||||
}{
|
|
||||||
{name: "successful response", response: response},
|
|
||||||
{name: "nil response"},
|
|
||||||
{name: "collaborator error", response: response, err: collaboratorErr},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
var captured domain.GenerateRequest
|
|
||||||
next := clientFunc(func(
|
|
||||||
_ context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
) (*domain.GenerateResponse, error) {
|
|
||||||
captured = req
|
|
||||||
return tc.response, tc.err
|
|
||||||
})
|
|
||||||
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
|
|
||||||
if !reflect.DeepEqual(captured, request) {
|
|
||||||
t.Fatalf("request changed: %#v", captured)
|
|
||||||
}
|
|
||||||
if gotResponse != tc.response || gotErr != tc.err {
|
|
||||||
t.Fatalf("response=(%p, %v), want (%p, %v)",
|
|
||||||
gotResponse, gotErr, tc.response, tc.err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
||||||
@@ -496,7 +450,7 @@ func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return &domain.GenerateResponse{Content: "recovered"}, nil
|
return &domain.GenerateResponse{Content: "recovered"}, nil
|
||||||
})
|
})
|
||||||
client := NewClient(manager, next)
|
client := newTestClient(manager, next)
|
||||||
request := domain.GenerateRequest{
|
request := domain.GenerateRequest{
|
||||||
Target: domain.ExecutionTarget{BackendID: "limited"},
|
Target: domain.ExecutionTarget{BackendID: "limited"},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ package configuration
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/configkit"
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBundledOpenRouterUnderlay(t *testing.T) {
|
func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||||
@@ -15,9 +17,12 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backend, err := state.GetBackend("openrouter")
|
backend, err := state.GetBackend("openrouter")
|
||||||
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||||
t.Fatalf("backend = %+v, %v", backend, err)
|
t.Fatalf("backend = %+v, %v", backend, err)
|
||||||
}
|
}
|
||||||
|
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
|
||||||
|
t.Fatalf("bundle inventories: backends=%v prompts=%v collections=%v", state.backendIDs, state.promptIDs, state.collections)
|
||||||
|
}
|
||||||
models := map[string]string{
|
models := map[string]string{
|
||||||
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
|
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
|
||||||
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
|
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
|
||||||
@@ -31,14 +36,16 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for id, model := range models {
|
for id, model := range models {
|
||||||
profile, err := state.GetProfile(context.Background(), id)
|
profile, err := state.GetProfile(context.Background(), id)
|
||||||
if err != nil || profile.ResolvedTarget.Model != model || profile.ResolvedTarget.BackendID != "openrouter" {
|
if err != nil {
|
||||||
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
||||||
}
|
}
|
||||||
|
want := defaults.ExecutionTargetDefault()
|
||||||
|
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
|
||||||
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
||||||
plugins := fmt.Sprint(profile.ResolvedTarget.ExtraParams)
|
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
|
||||||
if !strings.Contains(plugins, "cost_tier:"+tier) || !strings.Contains(plugins, "id:auto-router") {
|
}
|
||||||
t.Fatalf("profile %s plugins = %+v", id, profile.ResolvedTarget.ExtraParams)
|
if !reflect.DeepEqual(*profile.ResolvedTarget, want) {
|
||||||
}
|
t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") {
|
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") {
|
||||||
|
|||||||
@@ -2,36 +2,66 @@ package configuration
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// IdentityFilter selects all identities, one namespace, or unqualified IDs.
|
||||||
|
type IdentityFilter struct {
|
||||||
|
All bool
|
||||||
|
Namespace string
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectIdentities[T any](ctx context.Context, entries []T, filter IdentityFilter, id func(T) string) ([]T, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if filter.All {
|
||||||
|
result := append([]T(nil), entries...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if filter.Namespace != "" {
|
||||||
|
prefix := filter.Namespace + "/"
|
||||||
|
start := sort.Search(len(entries), func(i int) bool { return id(entries[i]) >= prefix })
|
||||||
|
end := start + sort.Search(len(entries)-start, func(i int) bool { return !strings.HasPrefix(id(entries[start+i]), prefix) })
|
||||||
|
result := append([]T(nil), entries[start:end]...)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
var result []T
|
||||||
|
for _, entry := range entries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !strings.Contains(id(entry), "/") {
|
||||||
|
result = append(result, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
// CollectionIdentity is the captured metadata for one final collection entry.
|
// CollectionIdentity is the captured metadata for one final collection entry.
|
||||||
type CollectionIdentity struct {
|
type CollectionIdentity struct {
|
||||||
Namespace string
|
Namespace string
|
||||||
ReleaseVersion *string
|
ReleaseVersion *string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
|
func (state *loaded) ListProfiles(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.profileIDs, filter, func(id string) string { return id })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
profiles := append([]string(nil), state.profileIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return profiles, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) {
|
func (state *loaded) ListBackends(ctx context.Context, filter IdentityFilter) ([]string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.backendIDs, filter, func(id string) string { return id })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
backends := append([]string(nil), state.backendIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return backends, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
@@ -72,13 +102,6 @@ func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity,
|
|||||||
return collections, nil
|
return collections, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
|
func (state *loaded) ListPrompts(ctx context.Context, filter IdentityFilter) ([]domain.PromptIdentity, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
return selectIdentities(ctx, state.promptIDs, filter, func(entry domain.PromptIdentity) string { return entry.ID })
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return prompts, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -32,7 +32,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -41,7 +41,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
|
|||||||
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
|
||||||
}
|
}
|
||||||
|
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -65,36 +65,36 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles = append(profiles, "caller profile")
|
profiles = append(profiles, "caller profile")
|
||||||
profiles[0] = "changed profile"
|
profiles[0] = "changed profile"
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backends[0] = "changed backend"
|
backends[0] = "changed backend"
|
||||||
|
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
prompts[0].ID = "changed prompt"
|
prompts[0].ID = "changed prompt"
|
||||||
|
|
||||||
profiles, err = state.ListProfiles(context.Background())
|
profiles, err = state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
if !reflect.DeepEqual(profiles, []string{"profile"}) {
|
||||||
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
|
||||||
}
|
}
|
||||||
backends, err = state.ListBackends(context.Background())
|
backends, err = state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
|
||||||
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
|
||||||
}
|
}
|
||||||
prompts, err = state.ListPrompts(context.Background())
|
prompts, err = state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -104,20 +104,66 @@ func TestLoadedListsAreIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadedListsSelectNamespaces(t *testing.T) {
|
||||||
|
ids := []string{"a", "a-b/one", "a/one", "a/two", "b", "z/one"}
|
||||||
|
state := &loaded{profileIDs: append([]string(nil), ids...), backendIDs: append([]string(nil), ids...)}
|
||||||
|
for _, id := range ids {
|
||||||
|
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
|
||||||
|
}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
filter IdentityFilter
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"all", IdentityFilter{All: true}, ids},
|
||||||
|
{"unqualified", IdentityFilter{}, []string{"a", "b"}},
|
||||||
|
{"namespace", IdentityFilter{Namespace: "a"}, []string{"a/one", "a/two"}},
|
||||||
|
{"prefix sharing", IdentityFilter{Namespace: "a-b"}, []string{"a-b/one"}},
|
||||||
|
{"missing", IdentityFilter{Namespace: "absent"}, nil},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
profiles, err := state.ListProfiles(context.Background(), tc.filter)
|
||||||
|
if err != nil || !reflect.DeepEqual(profiles, tc.want) {
|
||||||
|
t.Fatalf("profiles = %v, %v; want %v", profiles, err, tc.want)
|
||||||
|
}
|
||||||
|
backends, err := state.ListBackends(context.Background(), tc.filter)
|
||||||
|
if err != nil || !reflect.DeepEqual(backends, tc.want) {
|
||||||
|
t.Fatalf("backends = %v, %v; want %v", backends, err, tc.want)
|
||||||
|
}
|
||||||
|
prompts, err := state.ListPrompts(context.Background(), tc.filter)
|
||||||
|
if err != nil || len(prompts) != len(tc.want) {
|
||||||
|
t.Fatalf("prompts = %v, %v; want %v", prompts, err, tc.want)
|
||||||
|
}
|
||||||
|
for i := range prompts {
|
||||||
|
if prompts[i].ID != tc.want[i] {
|
||||||
|
t.Fatalf("prompt ID = %q; want %q", prompts[i].ID, tc.want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(profiles) > 0 {
|
||||||
|
profiles[0], backends[0], prompts[0].ID = "changed", "changed", "changed"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(state.profileIDs, ids) || !reflect.DeepEqual(state.backendIDs, ids) || state.promptIDs[0].ID != ids[0] {
|
||||||
|
t.Fatal("caller mutation changed captured identities")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadedListsEmptyInventory(t *testing.T) {
|
func TestLoadedListsEmptyInventory(t *testing.T) {
|
||||||
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profiles, err := state.ListProfiles(context.Background())
|
profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(profiles) != 0 {
|
if err != nil || len(profiles) != 0 {
|
||||||
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
|
||||||
}
|
}
|
||||||
backends, err := state.ListBackends(context.Background())
|
backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(backends) != 0 {
|
if err != nil || len(backends) != 0 {
|
||||||
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
t.Fatalf("empty backend listing = %#v, %v", backends, err)
|
||||||
}
|
}
|
||||||
prompts, err := state.ListPrompts(context.Background())
|
prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
|
||||||
if err != nil || len(prompts) != 0 {
|
if err != nil || len(prompts) != 0 {
|
||||||
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
|
||||||
}
|
}
|
||||||
@@ -134,17 +180,19 @@ func TestLoadedListsHonorCanceledContext(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
profiles, err := state.ListProfiles(ctx)
|
for _, filter := range []IdentityFilter{{All: true}, {}, {Namespace: "example"}} {
|
||||||
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
profiles, err := state.ListProfiles(ctx, filter)
|
||||||
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
|
||||||
}
|
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
|
||||||
backends, err := state.ListBackends(ctx)
|
}
|
||||||
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
backends, err := state.ListBackends(ctx, filter)
|
||||||
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
if !errors.Is(err, context.Canceled) || len(backends) != 0 {
|
||||||
}
|
t.Fatalf("canceled backend listing = %#v, %v", backends, err)
|
||||||
prompts, err := state.ListPrompts(ctx)
|
}
|
||||||
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
prompts, err := state.ListPrompts(ctx, filter)
|
||||||
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
|
||||||
|
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -119,7 +119,6 @@ const (
|
|||||||
|
|
||||||
type loaded struct {
|
type loaded struct {
|
||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
backends map[string]map[string]fieldState
|
|
||||||
profiles map[string]map[string]fieldState
|
profiles map[string]map[string]fieldState
|
||||||
promptMetadata map[string][]promptInputPresence
|
promptMetadata map[string][]promptInputPresence
|
||||||
prompts map[string]*promptEntry
|
prompts map[string]*promptEntry
|
||||||
@@ -212,7 +211,6 @@ func indexedProfiles(config *document) map[string]profileConfig {
|
|||||||
|
|
||||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
|
|
||||||
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
|
||||||
|
|
||||||
// Load captures one immutable effective configuration for internal consumers.
|
// Load captures one immutable effective configuration for internal consumers.
|
||||||
@@ -237,7 +235,6 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
Sensitive: secretProviderPath,
|
Sensitive: secretProviderPath,
|
||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
captureFields(resources.backends, func(v backendConfig) string { return v.ID }, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
|
|
||||||
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
|
||||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
||||||
}
|
}
|
||||||
@@ -257,7 +254,6 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
// load performs one ConfigKit lifecycle over the selected source tiers.
|
// load performs one ConfigKit lifecycle over the selected source tiers.
|
||||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
state := &loaded{
|
state := &loaded{
|
||||||
backends: make(map[string]map[string]fieldState),
|
|
||||||
profiles: make(map[string]map[string]fieldState),
|
profiles: make(map[string]map[string]fieldState),
|
||||||
promptMetadata: make(map[string][]promptInputPresence),
|
promptMetadata: make(map[string][]promptInputPresence),
|
||||||
}
|
}
|
||||||
@@ -419,101 +415,110 @@ func validateEffective(config *document, validation *configkit.Validation) {
|
|||||||
}
|
}
|
||||||
for _, resources := range sets {
|
for _, resources := range sets {
|
||||||
for index, backend := range resources.backends {
|
for index, backend := range resources.backends {
|
||||||
base := resources.bases.backends.Index(index)
|
validateBackendRecord(backend, resources.bases.backends.Index(index), validation)
|
||||||
validateBackendEndpoints(backend, base, validation)
|
|
||||||
if !domain.IsValidLocalResourceKey(backend.ID) {
|
|
||||||
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
|
||||||
}
|
|
||||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
|
||||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
|
||||||
}
|
|
||||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
|
||||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
|
||||||
}
|
|
||||||
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
|
||||||
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
|
||||||
}
|
|
||||||
if backend.QueueCapacity != nil {
|
|
||||||
if *backend.QueueCapacity < 0 {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
|
||||||
}
|
|
||||||
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
|
||||||
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
|
||||||
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
|
||||||
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
|
||||||
smokePath := base.Key("smoke_tests")
|
|
||||||
if smokeTests.DefaultProfile != nil {
|
|
||||||
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
|
||||||
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
|
||||||
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
|
||||||
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if authentication := smokeTests.Authentication; authentication != nil {
|
|
||||||
kindPath := smokePath.Key("authentication").Key("kind")
|
|
||||||
if authentication.Kind == nil {
|
|
||||||
validation.Error(kindPath, "authentication kind is required")
|
|
||||||
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
|
||||||
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for index, profile := range resources.profiles {
|
for index, profile := range resources.profiles {
|
||||||
base := resources.bases.profiles.Index(index)
|
validateProfileRecord(profile, resources.bases.profiles.Index(index), backendIDs, validation)
|
||||||
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
|
||||||
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
|
|
||||||
validation.Error(base.Key("endpoint"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !domain.IsValidLocalResourceKey(profile.ID) {
|
|
||||||
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
|
||||||
}
|
|
||||||
if profile.Backend != nil && *profile.Backend != "" {
|
|
||||||
if !domain.IsValidResourceID(*profile.Backend) {
|
|
||||||
validation.Error(base.Key("backend"), "backend reference is malformed")
|
|
||||||
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
|
||||||
validation.Error(base.Key("backend"), "backend is not defined")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
|
||||||
validation.Error(base.Key("model"), "model must not be blank")
|
|
||||||
}
|
|
||||||
if profile.Temperature != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
|
||||||
validation.Error(base.Key("temperature"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.TopP != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
|
||||||
validation.Error(base.Key("top_p"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.MaxTokens != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
|
||||||
validation.Error(base.Key("max_tokens"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile.TimeoutSeconds != nil {
|
|
||||||
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
|
||||||
validation.Error(base.Key("timeout_seconds"), err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
|
||||||
}
|
}
|
||||||
for index, prompt := range resources.prompts {
|
for index, prompt := range resources.prompts {
|
||||||
base := resources.bases.prompts.Index(index)
|
validatePromptRecord(prompt, resources.bases.prompts.Index(index), validation)
|
||||||
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
}
|
||||||
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
}
|
||||||
}
|
}
|
||||||
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
|
||||||
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
func validateBackendRecord(backend backendConfig, base configkit.Path, validation *configkit.Validation) {
|
||||||
|
validateBackendEndpoints(backend, base, validation)
|
||||||
|
if !domain.IsValidLocalResourceKey(backend.ID) {
|
||||||
|
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||||
|
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||||
|
}
|
||||||
|
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||||
|
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||||
|
}
|
||||||
|
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
|
||||||
|
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
|
||||||
|
}
|
||||||
|
if backend.QueueCapacity != nil {
|
||||||
|
if *backend.QueueCapacity < 0 {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
|
||||||
|
}
|
||||||
|
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
|
||||||
|
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
|
||||||
|
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
|
||||||
|
if smokeTests := backend.SmokeTests; smokeTests != nil {
|
||||||
|
smokePath := base.Key("smoke_tests")
|
||||||
|
if smokeTests.DefaultProfile != nil {
|
||||||
|
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
|
||||||
|
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
|
||||||
|
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if authentication := smokeTests.Authentication; authentication != nil {
|
||||||
|
kindPath := smokePath.Key("authentication").Key("kind")
|
||||||
|
if authentication.Kind == nil {
|
||||||
|
validation.Error(kindPath, "authentication kind is required")
|
||||||
|
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||||
|
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateProfileRecord(profile profileConfig, base configkit.Path, backendIDs map[string]struct{}, validation *configkit.Validation) {
|
||||||
|
if profile.Endpoint != nil && *profile.Endpoint != "" {
|
||||||
|
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
|
||||||
|
validation.Error(base.Key("endpoint"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !domain.IsValidLocalResourceKey(profile.ID) {
|
||||||
|
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if profile.Backend != nil && *profile.Backend != "" {
|
||||||
|
if !domain.IsValidResourceID(*profile.Backend) {
|
||||||
|
validation.Error(base.Key("backend"), "backend reference is malformed")
|
||||||
|
} else if _, exists := backendIDs[*profile.Backend]; !exists {
|
||||||
|
validation.Error(base.Key("backend"), "backend is not defined")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
|
||||||
|
validation.Error(base.Key("model"), "model must not be blank")
|
||||||
|
}
|
||||||
|
if profile.Temperature != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
|
||||||
|
validation.Error(base.Key("temperature"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.TopP != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
|
||||||
|
validation.Error(base.Key("top_p"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.MaxTokens != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
|
||||||
|
validation.Error(base.Key("max_tokens"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.TimeoutSeconds != nil {
|
||||||
|
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
|
||||||
|
validation.Error(base.Key("timeout_seconds"), err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePromptRecord(prompt promptConfig, base configkit.Path, validation *configkit.Validation) {
|
||||||
|
if !domain.IsValidLocalResourceKey(prompt.ID) {
|
||||||
|
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
|
||||||
|
}
|
||||||
|
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
|
||||||
|
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -496,7 +496,7 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
|
|||||||
t.Fatalf("recursive overlay failed: %+v", provider)
|
t.Fatalf("recursive overlay failed: %+v", provider)
|
||||||
}
|
}
|
||||||
backend := loaded.result.Config.Backends[0]
|
backend := loaded.result.Config.Backends[0]
|
||||||
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue {
|
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 {
|
||||||
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
|
||||||
}
|
}
|
||||||
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
|
||||||
@@ -582,9 +582,6 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
|
|||||||
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
|
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
|
||||||
t.Fatalf("resolved backend was not normalized: %+v", composed)
|
t.Fatalf("resolved backend was not normalized: %+v", composed)
|
||||||
}
|
}
|
||||||
if loaded.backends["local"]["queue_capacity"] != fieldNull {
|
|
||||||
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
|
|
||||||
}
|
|
||||||
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
|
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
|
||||||
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||||
t.Fatalf("endpoint origin was not preserved: %+v", field)
|
t.Fatalf("endpoint origin was not preserved: %+v", field)
|
||||||
@@ -763,7 +760,7 @@ func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backend, err := loaded.GetBackend("local")
|
backend, err := loaded.GetBackend("local")
|
||||||
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 || loaded.backends["local"]["queue_capacity"] != fieldAbsent {
|
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 {
|
||||||
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,104 +72,21 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
definition.SessionID = *raw.SessionID
|
definition.SessionID = *raw.SessionID
|
||||||
}
|
}
|
||||||
seenInputs := make(map[string]bool, len(raw.Inputs))
|
inputs, inputMetadata, err := state.capturePromptInputs(id, raw.Inputs, base)
|
||||||
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
|
if err != nil {
|
||||||
entry := &promptEntry{
|
return nil, err
|
||||||
messages: make([]*template.Template, 0, len(raw.Messages)),
|
|
||||||
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
|
|
||||||
}
|
}
|
||||||
for index, input := range raw.Inputs {
|
messages, compiledMessages, err := state.capturePromptMessages(raw.Messages, base, lookupOrigin)
|
||||||
path := base.Key("inputs").Index(index)
|
if err != nil {
|
||||||
if input.Name == "" {
|
return nil, err
|
||||||
return nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
|
||||||
}
|
|
||||||
if seenInputs[input.Name] {
|
|
||||||
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
|
||||||
}
|
|
||||||
seenInputs[input.Name] = true
|
|
||||||
contentType := ""
|
|
||||||
if input.ContentType != nil {
|
|
||||||
contentType = *input.ContentType
|
|
||||||
}
|
|
||||||
description := ""
|
|
||||||
if input.Description != nil {
|
|
||||||
description = *input.Description
|
|
||||||
}
|
|
||||||
definition.Inputs = append(definition.Inputs, domain.PromptInput{
|
|
||||||
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
|
||||||
})
|
|
||||||
presence := state.promptMetadata[id][index]
|
|
||||||
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
|
||||||
if presence.contentType == fieldValue {
|
|
||||||
metadata.ContentType = stringPointer(contentType)
|
|
||||||
}
|
|
||||||
if presence.description == fieldValue {
|
|
||||||
metadata.Description = stringPointer(description)
|
|
||||||
}
|
|
||||||
entry.requirementInputs = append(entry.requirementInputs, metadata)
|
|
||||||
}
|
}
|
||||||
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
contract, err := capturePromptOutput(raw.Output, base)
|
||||||
for index, message := range raw.Messages {
|
if err != nil {
|
||||||
path := base.Key("messages").Index(index)
|
return nil, err
|
||||||
if message.Role == nil {
|
|
||||||
return nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
|
||||||
}
|
|
||||||
if _, err := domain.NormalizeMessageRole(*message.Role); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
|
||||||
}
|
|
||||||
if (message.Content != nil) == (message.ContentFile != nil) {
|
|
||||||
return nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
|
||||||
}
|
|
||||||
content := ""
|
|
||||||
if message.Content != nil {
|
|
||||||
content = *message.Content
|
|
||||||
} else {
|
|
||||||
var err error
|
|
||||||
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var control *domain.CacheControl
|
|
||||||
if message.CacheControl != nil {
|
|
||||||
if message.CacheControl.Type == nil {
|
|
||||||
return nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
|
||||||
}
|
|
||||||
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
|
||||||
if message.CacheControl.TTL != nil {
|
|
||||||
control.TTL = *message.CacheControl.TTL
|
|
||||||
}
|
|
||||||
if _, err := domain.NormalizeCacheControl(control); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
|
||||||
}
|
|
||||||
entry.messages = append(entry.messages, compiled)
|
|
||||||
definition.Templates = append(definition.Templates, domain.PromptMessageTemplate{
|
|
||||||
Role: *message.Role, Content: content, CacheControl: control,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
if raw.Output == nil {
|
definition.Inputs = inputs
|
||||||
return nil, fmt.Errorf("%s: output is required", base.Key("output"))
|
definition.Templates = messages
|
||||||
}
|
entry := &promptEntry{requirementInputs: inputMetadata, messages: compiledMessages}
|
||||||
if raw.Output.Format == nil || raw.Output.ValidationMode == nil {
|
|
||||||
return nil, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
|
||||||
}
|
|
||||||
contract := domain.OutputContract{
|
|
||||||
Format: domain.OutputFormat(*raw.Output.Format),
|
|
||||||
ValidationMode: domain.ValidationMode(*raw.Output.ValidationMode),
|
|
||||||
RepairAttempts: raw.Output.RepairAttempts,
|
|
||||||
}
|
|
||||||
if raw.Output.SchemaPath != nil {
|
|
||||||
contract.SchemaPath = *raw.Output.SchemaPath
|
|
||||||
}
|
|
||||||
if err := domain.ValidateOutputContract(contract); err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
|
||||||
}
|
|
||||||
definition.OutputFormat = contract.Format
|
|
||||||
definition.Validation = contract
|
definition.Validation = contract
|
||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
var err error
|
var err error
|
||||||
@@ -192,6 +109,113 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
return entry, nil
|
return entry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (state *loaded) capturePromptInputs(id string, rawInputs []promptInputConfig, base configkit.Path) ([]domain.PromptInput, []prompt.InputMetadata, error) {
|
||||||
|
seenInputs := make(map[string]bool, len(rawInputs))
|
||||||
|
inputs := make([]domain.PromptInput, 0, len(rawInputs))
|
||||||
|
metadata := make([]prompt.InputMetadata, 0, len(rawInputs))
|
||||||
|
for index, input := range rawInputs {
|
||||||
|
path := base.Key("inputs").Index(index)
|
||||||
|
if input.Name == "" {
|
||||||
|
return nil, nil, fmt.Errorf("%s: input name is required", path.Key("name"))
|
||||||
|
}
|
||||||
|
if seenInputs[input.Name] {
|
||||||
|
return nil, nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
|
||||||
|
}
|
||||||
|
seenInputs[input.Name] = true
|
||||||
|
contentType := ""
|
||||||
|
if input.ContentType != nil {
|
||||||
|
contentType = *input.ContentType
|
||||||
|
}
|
||||||
|
description := ""
|
||||||
|
if input.Description != nil {
|
||||||
|
description = *input.Description
|
||||||
|
}
|
||||||
|
inputs = append(inputs, domain.PromptInput{
|
||||||
|
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
|
||||||
|
})
|
||||||
|
presence := state.promptMetadata[id][index]
|
||||||
|
item := prompt.InputMetadata{Name: input.Name, Required: input.Required}
|
||||||
|
if presence.contentType == fieldValue {
|
||||||
|
item.ContentType = stringPointer(contentType)
|
||||||
|
}
|
||||||
|
if presence.description == fieldValue {
|
||||||
|
item.Description = stringPointer(description)
|
||||||
|
}
|
||||||
|
metadata = append(metadata, item)
|
||||||
|
}
|
||||||
|
return inputs, metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) capturePromptMessages(rawMessages []promptMessageConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) ([]domain.PromptMessageTemplate, []*template.Template, error) {
|
||||||
|
templates := make([]domain.PromptMessageTemplate, 0, len(rawMessages))
|
||||||
|
compiledMessages := make([]*template.Template, 0, len(rawMessages))
|
||||||
|
for index, message := range rawMessages {
|
||||||
|
path := base.Key("messages").Index(index)
|
||||||
|
if message.Role == nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: role is required", path.Key("role"))
|
||||||
|
}
|
||||||
|
if _, err := domain.NormalizeMessageRole(*message.Role); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: %w", path.Key("role"), err)
|
||||||
|
}
|
||||||
|
if (message.Content != nil) == (message.ContentFile != nil) {
|
||||||
|
return nil, nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
|
||||||
|
}
|
||||||
|
content := ""
|
||||||
|
if message.Content != nil {
|
||||||
|
content = *message.Content
|
||||||
|
} else {
|
||||||
|
var err error
|
||||||
|
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var control *domain.CacheControl
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
if message.CacheControl.Type == nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
|
||||||
|
}
|
||||||
|
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
|
||||||
|
if message.CacheControl.TTL != nil {
|
||||||
|
control.TTL = *message.CacheControl.TTL
|
||||||
|
}
|
||||||
|
if _, err := domain.NormalizeCacheControl(control); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
|
||||||
|
}
|
||||||
|
compiledMessages = append(compiledMessages, compiled)
|
||||||
|
templates = append(templates, domain.PromptMessageTemplate{
|
||||||
|
Role: *message.Role, Content: content, CacheControl: control,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return templates, compiledMessages, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func capturePromptOutput(output *promptOutputConfig, base configkit.Path) (domain.OutputContract, error) {
|
||||||
|
if output == nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: output is required", base.Key("output"))
|
||||||
|
}
|
||||||
|
if output.Format == nil || output.ValidationMode == nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
|
||||||
|
}
|
||||||
|
contract := domain.OutputContract{
|
||||||
|
Format: domain.OutputFormat(*output.Format),
|
||||||
|
ValidationMode: domain.ValidationMode(*output.ValidationMode),
|
||||||
|
RepairAttempts: output.RepairAttempts,
|
||||||
|
}
|
||||||
|
if output.SchemaPath != nil {
|
||||||
|
contract.SchemaPath = *output.SchemaPath
|
||||||
|
}
|
||||||
|
if err := domain.ValidateOutputContract(contract); err != nil {
|
||||||
|
return domain.OutputContract{}, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
|
||||||
|
}
|
||||||
|
return contract, nil
|
||||||
|
}
|
||||||
|
|
||||||
func stringPointer(value string) *string {
|
func stringPointer(value string) *string {
|
||||||
copy := value
|
copy := value
|
||||||
return ©
|
return ©
|
||||||
@@ -305,12 +329,6 @@ func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domai
|
|||||||
return ownPromptInspection(entry.inspection), nil
|
return ownPromptInspection(entry.inspection), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptDefinition implements the runner's prompt lookup boundary.
|
|
||||||
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
|
|
||||||
definition, _, err := state.GetPromptSnapshot(ctx, id)
|
|
||||||
return definition, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
// GetCompiledTemplates returns construction-time parses for one prompt ID.
|
||||||
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
||||||
if state == nil {
|
if state == nil {
|
||||||
|
|||||||
@@ -17,109 +17,115 @@ func (state *loaded) buildExecutionProfiles() error {
|
|||||||
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
definitions := make(map[string]domain.Backend, len(backendConfigs))
|
||||||
for _, id := range sortedKeys(backendConfigs) {
|
for _, id := range sortedKeys(backendConfigs) {
|
||||||
config := backendConfigs[id]
|
config := backendConfigs[id]
|
||||||
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
definitions[id] = captureBackend(id, config)
|
||||||
if config.SmokeTests != nil {
|
|
||||||
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
|
||||||
if config.SmokeTests.Reachability != nil {
|
|
||||||
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
|
||||||
}
|
|
||||||
if config.SmokeTests.Authentication != nil {
|
|
||||||
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
|
||||||
Kind: config.SmokeTests.Authentication.Kind,
|
|
||||||
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
definition.SmokeTests = smokeTests
|
|
||||||
}
|
|
||||||
if config.Endpoint != nil {
|
|
||||||
definition.Endpoint = *config.Endpoint
|
|
||||||
}
|
|
||||||
if config.APIKeyEnv != nil {
|
|
||||||
definition.APIKeyEnv = *config.APIKeyEnv
|
|
||||||
}
|
|
||||||
if config.ConcurrencyLimit != nil {
|
|
||||||
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
|
||||||
}
|
|
||||||
if config.QueueCapacity != nil {
|
|
||||||
definition.QueueCapacity = *config.QueueCapacity
|
|
||||||
definition.QueueCapacitySet = definition.ConcurrencyLimit > 0
|
|
||||||
}
|
|
||||||
definitions[id] = definition
|
|
||||||
}
|
}
|
||||||
registry, err := backend.NewEffectiveRegistry(definitions)
|
state.registry = backend.NewEffectiveRegistry(definitions)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
state.registry = registry
|
|
||||||
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
|
||||||
for _, id := range sortedKeys(profileConfigs) {
|
for _, id := range sortedKeys(profileConfigs) {
|
||||||
config := profileConfigs[id]
|
config := profileConfigs[id]
|
||||||
fields := state.profiles[id]
|
profile, err := captureExecutionProfile(id, config, state.profiles[id], state.registry)
|
||||||
target := defaults.ExecutionTargetDefault()
|
if err != nil {
|
||||||
if config.Backend != nil && *config.Backend != "" {
|
return err
|
||||||
selected, err := registry.GetBackend(*config.Backend)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("profile %q backend: %w", id, err)
|
|
||||||
}
|
|
||||||
target.BackendID = selected.ID
|
|
||||||
target.Endpoint = selected.Endpoint
|
|
||||||
target.APIKeyEnv = selected.APIKeyEnv
|
|
||||||
target.ExtraParams = selected.ExtraParams
|
|
||||||
}
|
}
|
||||||
presence := domain.ExecutionTargetPresence{}
|
|
||||||
if fields["endpoint"] != fieldAbsent {
|
|
||||||
target.Endpoint = valueOrZero(config.Endpoint)
|
|
||||||
}
|
|
||||||
if fields["model"] != fieldAbsent {
|
|
||||||
target.Model = valueOrZero(config.Model)
|
|
||||||
}
|
|
||||||
if fields["temperature"] != fieldAbsent {
|
|
||||||
target.Temperature = valueOrZero(config.Temperature)
|
|
||||||
presence.Temperature = fields["temperature"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["top_p"] != fieldAbsent {
|
|
||||||
target.TopP = valueOrZero(config.TopP)
|
|
||||||
presence.TopP = fields["top_p"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["max_tokens"] != fieldAbsent {
|
|
||||||
target.MaxTokens = valueOrZero(config.MaxTokens)
|
|
||||||
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
|
||||||
}
|
|
||||||
if fields["timeout_seconds"] == fieldValue {
|
|
||||||
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
|
||||||
presence.TimeoutSeconds = true
|
|
||||||
}
|
|
||||||
if fields["service_tier"] != fieldAbsent {
|
|
||||||
target.ServiceTier = valueOrZero(config.ServiceTier)
|
|
||||||
}
|
|
||||||
if fields["reasoning_effort"] != fieldAbsent {
|
|
||||||
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
|
||||||
}
|
|
||||||
if fields["api_key_env"] != fieldAbsent {
|
|
||||||
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
|
||||||
}
|
|
||||||
if fields["api_key_required"] != fieldAbsent {
|
|
||||||
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
|
||||||
}
|
|
||||||
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
|
||||||
target.APIKeyEnv = ""
|
|
||||||
}
|
|
||||||
switch fields["extra_params"] {
|
|
||||||
case fieldNull:
|
|
||||||
target.ExtraParams = nil
|
|
||||||
case fieldValue:
|
|
||||||
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("profile %q extra parameters: %w", id, err)
|
|
||||||
}
|
|
||||||
target.ExtraParams = merged
|
|
||||||
}
|
|
||||||
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
|
||||||
state.indexedProfiles[id] = profile
|
state.indexedProfiles[id] = profile
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func captureBackend(id string, config backendConfig) domain.Backend {
|
||||||
|
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
||||||
|
if config.SmokeTests != nil {
|
||||||
|
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
||||||
|
if config.SmokeTests.Reachability != nil {
|
||||||
|
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
|
||||||
|
}
|
||||||
|
if config.SmokeTests.Authentication != nil {
|
||||||
|
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
|
||||||
|
Kind: config.SmokeTests.Authentication.Kind,
|
||||||
|
Endpoint: config.SmokeTests.Authentication.Endpoint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
definition.SmokeTests = smokeTests
|
||||||
|
}
|
||||||
|
if config.Endpoint != nil {
|
||||||
|
definition.Endpoint = *config.Endpoint
|
||||||
|
}
|
||||||
|
if config.APIKeyEnv != nil {
|
||||||
|
definition.APIKeyEnv = *config.APIKeyEnv
|
||||||
|
}
|
||||||
|
if config.ConcurrencyLimit != nil {
|
||||||
|
definition.ConcurrencyLimit = *config.ConcurrencyLimit
|
||||||
|
}
|
||||||
|
if config.QueueCapacity != nil {
|
||||||
|
definition.QueueCapacity = *config.QueueCapacity
|
||||||
|
}
|
||||||
|
return definition
|
||||||
|
}
|
||||||
|
|
||||||
|
func captureExecutionProfile(id string, config profileConfig, fields map[string]fieldState, registry *backend.Registry) (domain.ExecutionProfile, error) {
|
||||||
|
target := defaults.ExecutionTargetDefault()
|
||||||
|
if config.Backend != nil && *config.Backend != "" {
|
||||||
|
selected, err := registry.GetBackend(*config.Backend)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w", id, err)
|
||||||
|
}
|
||||||
|
target.BackendID = selected.ID
|
||||||
|
target.Endpoint = selected.Endpoint
|
||||||
|
target.APIKeyEnv = selected.APIKeyEnv
|
||||||
|
target.ExtraParams = selected.ExtraParams
|
||||||
|
}
|
||||||
|
presence := domain.ExecutionTargetPresence{}
|
||||||
|
if fields["endpoint"] != fieldAbsent {
|
||||||
|
target.Endpoint = valueOrZero(config.Endpoint)
|
||||||
|
}
|
||||||
|
if fields["model"] != fieldAbsent {
|
||||||
|
target.Model = valueOrZero(config.Model)
|
||||||
|
}
|
||||||
|
if fields["temperature"] != fieldAbsent {
|
||||||
|
target.Temperature = valueOrZero(config.Temperature)
|
||||||
|
presence.Temperature = fields["temperature"] == fieldValue
|
||||||
|
}
|
||||||
|
if fields["top_p"] != fieldAbsent {
|
||||||
|
target.TopP = valueOrZero(config.TopP)
|
||||||
|
presence.TopP = fields["top_p"] == fieldValue
|
||||||
|
}
|
||||||
|
if fields["max_tokens"] != fieldAbsent {
|
||||||
|
target.MaxTokens = valueOrZero(config.MaxTokens)
|
||||||
|
presence.MaxTokens = fields["max_tokens"] == fieldValue
|
||||||
|
}
|
||||||
|
if fields["timeout_seconds"] == fieldValue {
|
||||||
|
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
|
||||||
|
presence.TimeoutSeconds = true
|
||||||
|
}
|
||||||
|
if fields["service_tier"] != fieldAbsent {
|
||||||
|
target.ServiceTier = valueOrZero(config.ServiceTier)
|
||||||
|
}
|
||||||
|
if fields["reasoning_effort"] != fieldAbsent {
|
||||||
|
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
|
||||||
|
}
|
||||||
|
if fields["api_key_env"] != fieldAbsent {
|
||||||
|
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
|
||||||
|
}
|
||||||
|
if fields["api_key_required"] != fieldAbsent {
|
||||||
|
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
|
||||||
|
}
|
||||||
|
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
|
||||||
|
target.APIKeyEnv = ""
|
||||||
|
}
|
||||||
|
switch fields["extra_params"] {
|
||||||
|
case fieldNull:
|
||||||
|
target.ExtraParams = nil
|
||||||
|
case fieldValue:
|
||||||
|
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ExecutionProfile{}, fmt.Errorf("profile %q extra parameters: %w", id, err)
|
||||||
|
}
|
||||||
|
target.ExtraParams = merged
|
||||||
|
}
|
||||||
|
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
|
||||||
|
return profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
func valueOrZero[T any](value *T) T {
|
func valueOrZero[T any](value *T) T {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
var zero T
|
var zero T
|
||||||
@@ -138,24 +144,33 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
|
|||||||
if merged == nil {
|
if merged == nil {
|
||||||
merged = make(map[string]any)
|
merged = make(map[string]any)
|
||||||
}
|
}
|
||||||
|
if err := overlayOwnedProviderMap(merged, upper); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return merged, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlayOwnedProviderMap(merged, upper map[string]any) error {
|
||||||
for key, value := range upper {
|
for key, value := range upper {
|
||||||
if child, ok := value.(map[string]any); ok {
|
if child, ok := value.(map[string]any); ok {
|
||||||
if existing, ok := merged[key].(map[string]any); ok {
|
if existing, ok := merged[key].(map[string]any); ok {
|
||||||
combined, err := overlayProviderMap(existing, child)
|
if existing == nil {
|
||||||
if err != nil {
|
existing = make(map[string]any)
|
||||||
return nil, err
|
merged[key] = existing
|
||||||
|
}
|
||||||
|
if err := overlayOwnedProviderMap(existing, child); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
merged[key] = combined
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
copied, err := jsonvalue.Copy(value)
|
copied, err := jsonvalue.Copy(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return err
|
||||||
}
|
}
|
||||||
merged[key] = copied
|
merged[key] = copied
|
||||||
}
|
}
|
||||||
return merged, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
|
||||||
@@ -180,11 +195,7 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
target := *profile.ResolvedTarget
|
target := *profile.ResolvedTarget
|
||||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
target.ExtraParams = params
|
|
||||||
profile.ResolvedTarget = &target
|
profile.ResolvedTarget = &target
|
||||||
return &profile, nil
|
return &profile, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,6 +42,28 @@ func TestLoadIndexesIdentifiedTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOverlayProviderMapOwnsInputsAndReplacesValues(t *testing.T) {
|
||||||
|
lower := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"old"}, "null": "old"}, "disjoint": []any{"original"}}
|
||||||
|
upper := map[string]any{"nested": map[string]any{"replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}}
|
||||||
|
got, err := overlayProviderMap(lower, upper)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}, "disjoint": []any{"original"}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("overlay = %#v", got)
|
||||||
|
}
|
||||||
|
lower["disjoint"].([]any)[0] = "changed"
|
||||||
|
upper["nested"].(map[string]any)["replace"].([]any)[0] = "changed"
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("overlay shares input storage: %#v", got)
|
||||||
|
}
|
||||||
|
got["nested"].(map[string]any)["added"].(map[string]any)["value"] = "changed"
|
||||||
|
if upper["nested"].(map[string]any)["added"].(map[string]any)["value"] != "upper" {
|
||||||
|
t.Fatal("result mutation reached upper")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
||||||
state, err := load(mapping("partial profile", map[string]any{
|
state, err := load(mapping("partial profile", map[string]any{
|
||||||
"profiles": []any{map[string]any{"id": "partial"}},
|
"profiles": []any{map[string]any{"id": "partial"}},
|
||||||
@@ -53,7 +75,7 @@ func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
runner := usecase.NewRunner(state, state, nil, prompt.NewGoRenderer(), nil, nil, nil)
|
runner := usecase.NewRunner(state, state, nil, prompt.NewCompiledRenderer(state), nil, nil, nil)
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "example", ProfileID: "partial",
|
PromptID: "example", ProfileID: "partial",
|
||||||
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
|
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
|
||||||
@@ -97,7 +119,7 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
selected, err := state.registry.GetBackend("second")
|
selected, err := state.registry.GetBackend("second")
|
||||||
if err != nil || selected.QueueCapacity != 0 || !selected.QueueCapacitySet {
|
if err != nil || selected.QueueCapacity != 0 || selected.ConcurrencyLimit != 2 {
|
||||||
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
|
||||||
}
|
}
|
||||||
child, err := state.GetProfile(context.Background(), "child")
|
child, err := state.GetProfile(context.Background(), "child")
|
||||||
@@ -197,10 +219,7 @@ func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
|
||||||
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,
|
||||||
|
|||||||
@@ -145,7 +145,6 @@ type PromptDefinition struct {
|
|||||||
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
SessionID string `yaml:"session_id" json:"session_id,omitempty"`
|
||||||
Inputs []PromptInput `yaml:"inputs"`
|
Inputs []PromptInput `yaml:"inputs"`
|
||||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
Templates []PromptMessageTemplate `yaml:"templates"`
|
||||||
OutputFormat OutputFormat `yaml:"output_format"`
|
|
||||||
Validation OutputContract `yaml:"validation"`
|
Validation OutputContract `yaml:"validation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,7 +169,6 @@ type PromptInput struct {
|
|||||||
type PromptMessageTemplate struct {
|
type PromptMessageTemplate struct {
|
||||||
Role string `yaml:"role"`
|
Role string `yaml:"role"`
|
||||||
Content string `yaml:"content"`
|
Content string `yaml:"content"`
|
||||||
ContentFile string `yaml:"content_file"`
|
|
||||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -184,7 +182,6 @@ type Backend struct {
|
|||||||
SmokeTests *BackendSmokeTestConfig
|
SmokeTests *BackendSmokeTestConfig
|
||||||
ConcurrencyLimit int
|
ConcurrencyLimit int
|
||||||
QueueCapacity int
|
QueueCapacity int
|
||||||
QueueCapacitySet bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackendCapacityPolicy describes normalized run and generation capacity for
|
// BackendCapacityPolicy describes normalized run and generation capacity for
|
||||||
|
|||||||
@@ -31,14 +31,6 @@ func qualifyResourceID(id, namespace string) (string, bool) {
|
|||||||
return namespace + "/" + id, true
|
return namespace + "/" + id, true
|
||||||
}
|
}
|
||||||
|
|
||||||
func resourceNamespaceFromID(id string) (string, bool) {
|
|
||||||
if !validResourceID(id) || !strings.Contains(id, "/") {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
namespace, _, _ := strings.Cut(id, "/")
|
|
||||||
return namespace, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
// IsValidResourceID reports whether id is an unqualified resource ID or one
|
||||||
// valid namespace/local-ID pair.
|
// valid namespace/local-ID pair.
|
||||||
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
func IsValidResourceID(id string) bool { return validResourceID(id) }
|
||||||
|
|||||||
@@ -86,25 +86,3 @@ func TestQualifyResourceID(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResourceNamespaceFromID(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
id string
|
|
||||||
namespace string
|
|
||||||
wantOK bool
|
|
||||||
}{
|
|
||||||
{name: "qualified", id: "team/item", namespace: "team", wantOK: true},
|
|
||||||
{name: "same-prefix namespace stays distinct", id: "team-extra/item", namespace: "team-extra", wantOK: true},
|
|
||||||
{name: "unqualified", id: "item", wantOK: false},
|
|
||||||
{name: "malformed", id: "team/sub/item", wantOK: false},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
got, ok := resourceNamespaceFromID(test.id)
|
|
||||||
if got != test.namespace || ok != test.wantOK {
|
|
||||||
t.Fatalf("resourceNamespaceFromID(%q) = (%q, %t), want (%q, %t)", test.id, got, ok, test.namespace, test.wantOK)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,5 +18,5 @@ func NormalizeSessionID(raw string) (string, error) {
|
|||||||
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
||||||
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
||||||
}
|
}
|
||||||
return normalized, nil
|
return strings.Clone(normalized), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,24 +112,21 @@ type SmokeTestAttempt struct {
|
|||||||
type SmokeTestReason string
|
type SmokeTestReason string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
SmokeTestReasonNone SmokeTestReason = ""
|
SmokeTestReasonNone SmokeTestReason = ""
|
||||||
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
|
||||||
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
|
||||||
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
|
||||||
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
|
||||||
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
|
||||||
|
SmokeTestReasonAuthenticationResponseLimitExceeded SmokeTestReason = "authentication_response_limit_exceeded"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CloneBackend returns a copy of a backend, including its mutable settings.
|
// CloneBackend returns a copy of a backend, including its mutable settings.
|
||||||
func CloneBackend(backend Backend) (Backend, error) {
|
func CloneBackend(backend Backend) Backend {
|
||||||
cloned := backend
|
cloned := backend
|
||||||
var err error
|
cloned.ExtraParams = jsonvalue.CloneTrusted(backend.ExtraParams).(map[string]any)
|
||||||
cloned.ExtraParams, err = jsonvalue.CopyMap(backend.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return Backend{}, err
|
|
||||||
}
|
|
||||||
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
|
||||||
return cloned, nil
|
return cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
|
||||||
@@ -153,105 +150,6 @@ func CloneBackendSmokeTestConfig(config *BackendSmokeTestConfig) *BackendSmokeTe
|
|||||||
return &cloned
|
return &cloned
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneSmokeTestRequest returns a copy that owns its optional settings.
|
|
||||||
func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
|
|
||||||
cloned := request
|
|
||||||
cloned.TransientRetries = cloneInt(request.TransientRetries)
|
|
||||||
if request.Execution == nil {
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
execution := *request.Execution
|
|
||||||
execution.Temperature = cloneFloat64(request.Execution.Temperature)
|
|
||||||
execution.MaxTokens = cloneInt(request.Execution.MaxTokens)
|
|
||||||
execution.TopP = cloneFloat64(request.Execution.TopP)
|
|
||||||
execution.TimeoutSeconds = cloneInt(request.Execution.TimeoutSeconds)
|
|
||||||
execution.ReasoningEffort = cloneString(request.Execution.ReasoningEffort)
|
|
||||||
var err error
|
|
||||||
execution.ExtraParams, err = jsonvalue.CopyMap(request.Execution.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return SmokeTestRequest{}, err
|
|
||||||
}
|
|
||||||
cloned.Execution = &execution
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CloneSmokeTestReport returns a copy with independent optional facts and maps.
|
|
||||||
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
|
|
||||||
cloned := report
|
|
||||||
cloned.Applicable = cloneBool(report.Applicable)
|
|
||||||
cloned.Facts = cloneSmokeTestFacts(report.Facts)
|
|
||||||
cloned.Timing = cloneSmokeTestTiming(report.Timing)
|
|
||||||
var err error
|
|
||||||
cloned.Target.EffectiveSettings.ExtraParams, err = jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return SmokeTestReport{}, err
|
|
||||||
}
|
|
||||||
if report.Attempts != nil {
|
|
||||||
cloned.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
|
|
||||||
for index, attempt := range report.Attempts {
|
|
||||||
cloned.Attempts[index] = attempt
|
|
||||||
cloned.Attempts[index].HTTPStatus = cloneInt(attempt.HTTPStatus)
|
|
||||||
cloned.Attempts[index].Facts = cloneSmokeTestFacts(attempt.Facts)
|
|
||||||
cloned.Attempts[index].Timing = cloneSmokeTestTiming(attempt.Timing)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cloned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneSmokeTestFacts(facts SmokeTestFacts) SmokeTestFacts {
|
|
||||||
facts.CredentialsSupplied = cloneBool(facts.CredentialsSupplied)
|
|
||||||
facts.EndpointReachable = cloneBool(facts.EndpointReachable)
|
|
||||||
facts.CredentialsAccepted = cloneBool(facts.CredentialsAccepted)
|
|
||||||
facts.ResponseReceived = cloneBool(facts.ResponseReceived)
|
|
||||||
facts.GenerationCompleted = cloneBool(facts.GenerationCompleted)
|
|
||||||
facts.TextReturned = cloneBool(facts.TextReturned)
|
|
||||||
facts.ExpectedTextMatched = cloneBool(facts.ExpectedTextMatched)
|
|
||||||
facts.StructuredOutputRequested = cloneBool(facts.StructuredOutputRequested)
|
|
||||||
facts.ValidJSON = cloneBool(facts.ValidJSON)
|
|
||||||
facts.SchemaMatched = cloneBool(facts.SchemaMatched)
|
|
||||||
return facts
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneSmokeTestTiming(timing SmokeTestTiming) SmokeTestTiming {
|
|
||||||
timing.CapacityWait = cloneDuration(timing.CapacityWait)
|
|
||||||
timing.RequestDuration = cloneDuration(timing.RequestDuration)
|
|
||||||
timing.RetryDelay = cloneDuration(timing.RetryDelay)
|
|
||||||
timing.EvaluationDuration = cloneDuration(timing.EvaluationDuration)
|
|
||||||
return timing
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneBool(value *bool) *bool {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneDuration(value *time.Duration) *time.Duration {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneFloat64(value *float64) *float64 {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneInt(value *int) *int {
|
|
||||||
if value == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cloned := *value
|
|
||||||
return &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneString(value *string) *string {
|
func cloneString(value *string) *string {
|
||||||
if value == nil {
|
if value == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
package domain
|
package domain
|
||||||
|
|
||||||
import (
|
import "testing"
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
||||||
profile := "profile"
|
profile := "profile"
|
||||||
@@ -22,10 +19,7 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cloned, err := CloneBackend(backend)
|
cloned := CloneBackend(backend)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.SmokeTests.DefaultProfile = "changed"
|
*cloned.SmokeTests.DefaultProfile = "changed"
|
||||||
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
*cloned.SmokeTests.Reachability.Endpoint = "changed"
|
||||||
*cloned.SmokeTests.Authentication.Kind = "changed"
|
*cloned.SmokeTests.Authentication.Kind = "changed"
|
||||||
@@ -41,72 +35,12 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCloneSmokeTestRequestOwnsOptionalSettings(t *testing.T) {
|
func TestCloneBackendPreservesProviderMapPresence(t *testing.T) {
|
||||||
temperature := 0.25
|
if cloned := CloneBackend(Backend{}); cloned.ExtraParams != nil {
|
||||||
retries := 2
|
t.Fatalf("nil provider map became %#v", cloned.ExtraParams)
|
||||||
reasoning := "low"
|
|
||||||
request := SmokeTestRequest{
|
|
||||||
APIKey: "secret",
|
|
||||||
TransientRetries: &retries,
|
|
||||||
Execution: &ExecutionTargetOverride{
|
|
||||||
Temperature: &temperature,
|
|
||||||
ReasoningEffort: &reasoning,
|
|
||||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
cloned := CloneBackend(Backend{ExtraParams: map[string]any{"empty": map[string]any{}, "null": nil}})
|
||||||
cloned, err := CloneSmokeTestRequest(request)
|
if cloned.ExtraParams == nil || cloned.ExtraParams["empty"].(map[string]any) == nil || cloned.ExtraParams["null"] != nil {
|
||||||
if err != nil {
|
t.Fatalf("provider map presence changed: %#v", cloned.ExtraParams)
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.TransientRetries = 3
|
|
||||||
*cloned.Execution.Temperature = 0.5
|
|
||||||
*cloned.Execution.ReasoningEffort = "high"
|
|
||||||
cloned.Execution.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
|
|
||||||
if retries != 2 || temperature != 0.25 || reasoning != "low" {
|
|
||||||
t.Fatal("cloned request shared optional scalar pointers")
|
|
||||||
}
|
|
||||||
if got := request.Execution.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
|
||||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
|
|
||||||
applicable := true
|
|
||||||
reachable := true
|
|
||||||
status := 200
|
|
||||||
wait := 3 * time.Millisecond
|
|
||||||
report := SmokeTestReport{
|
|
||||||
Applicable: &applicable,
|
|
||||||
Target: SmokeTestTarget{
|
|
||||||
EffectiveSettings: ExecutionTarget{
|
|
||||||
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
|
||||||
Attempts: []SmokeTestAttempt{{
|
|
||||||
HTTPStatus: &status,
|
|
||||||
Facts: SmokeTestFacts{EndpointReachable: &reachable},
|
|
||||||
Timing: SmokeTestTiming{CapacityWait: &wait},
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
|
|
||||||
cloned, err := CloneSmokeTestReport(report)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
*cloned.Applicable = false
|
|
||||||
*cloned.Facts.EndpointReachable = false
|
|
||||||
*cloned.Attempts[0].HTTPStatus = 503
|
|
||||||
*cloned.Attempts[0].Facts.EndpointReachable = false
|
|
||||||
*cloned.Attempts[0].Timing.CapacityWait = time.Second
|
|
||||||
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
|
|
||||||
|
|
||||||
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond {
|
|
||||||
t.Fatal("cloned report shared optional scalar pointers")
|
|
||||||
}
|
|
||||||
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
|
|
||||||
t.Fatalf("source provider parameter changed through clone: %v", got)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ type ConfigurationSources struct {
|
|||||||
type Options struct {
|
type Options struct {
|
||||||
llmClient llm.Client
|
llmClient llm.Client
|
||||||
artifactReader artifact.Reader
|
artifactReader artifact.Reader
|
||||||
validator validate.Validator
|
validator validate.ValidationPreparer
|
||||||
validatorSource bool
|
validatorSource bool
|
||||||
artifactSource bool
|
artifactSource bool
|
||||||
disableBundledOpenRouter bool
|
disableBundledOpenRouter bool
|
||||||
@@ -73,7 +73,7 @@ func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
|
|||||||
|
|
||||||
// SetSchemaFile selects a single schema file.
|
// SetSchemaFile selects a single schema file.
|
||||||
func (o *Options) SetSchemaFile(path string) error {
|
func (o *Options) SetSchemaFile(path string) error {
|
||||||
fsys, root, _, err := fileSource(path)
|
fsys, root, err := fileSource(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -98,9 +98,9 @@ type resourceNamespaces struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type identityInventory interface {
|
type identityInventory interface {
|
||||||
ListBackends(context.Context) ([]string, error)
|
ListBackends(context.Context, configuration.IdentityFilter) ([]string, error)
|
||||||
ListProfiles(context.Context) ([]string, error)
|
ListProfiles(context.Context, configuration.IdentityFilter) ([]string, error)
|
||||||
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
|
ListPrompts(context.Context, configuration.IdentityFilter) ([]domain.PromptIdentity, error)
|
||||||
ListNamespaces(context.Context, []string) ([]string, error)
|
ListNamespaces(context.Context, []string) ([]string, error)
|
||||||
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
|
||||||
}
|
}
|
||||||
@@ -157,10 +157,7 @@ func New(cfg Config, options Options) (*Engine, error) {
|
|||||||
var client llm.Client
|
var client llm.Client
|
||||||
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
var diagnosticHTTPClient *llm.OpenAICompatibleClient
|
||||||
if options.llmClient == nil {
|
if options.llmClient == nil {
|
||||||
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
builtIn := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
|
||||||
if newErr != nil {
|
|
||||||
return nil, newErr
|
|
||||||
}
|
|
||||||
diagnosticHTTPClient = builtIn
|
diagnosticHTTPClient = builtIn
|
||||||
client = generation.NewBuiltIn(manager, builtIn)
|
client = generation.NewBuiltIn(manager, builtIn)
|
||||||
} else {
|
} else {
|
||||||
@@ -208,23 +205,23 @@ func validateSourceFS(fsys fs.FS, root string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, string, error) {
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
if strings.TrimSpace(name) == "" {
|
if strings.TrimSpace(name) == "" {
|
||||||
return nil, "", "", fmt.Errorf("source path is required")
|
return nil, "", fmt.Errorf("source path is required")
|
||||||
}
|
}
|
||||||
directory := filepath.Dir(name)
|
directory := filepath.Dir(name)
|
||||||
base := filepath.Base(name)
|
base := filepath.Base(name)
|
||||||
if base == "." || base == string(filepath.Separator) {
|
if base == "." || base == string(filepath.Separator) {
|
||||||
return nil, "", "", fmt.Errorf("source path must name a file")
|
return nil, "", fmt.Errorf("source path must name a file")
|
||||||
}
|
}
|
||||||
info, err := os.Stat(name)
|
info, err := os.Stat(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
return nil, "", fmt.Errorf("failed to access source file %q: %w", name, err)
|
||||||
}
|
}
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return nil, "", "", fmt.Errorf("source path %q must be a file", name)
|
return nil, "", fmt.Errorf("source path %q must be a file", name)
|
||||||
}
|
}
|
||||||
return os.DirFS(directory), filepath.ToSlash(base), directory, nil
|
return os.DirFS(directory), filepath.ToSlash(base), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
@@ -268,24 +265,11 @@ func (e *Engine) ListProfiles(ctx context.Context, filters ...ListingFilter) ([]
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
|
||||||
mode, namespace, err := resolveListingFilter(filters, e.namespaces.prompt)
|
filter, err := inventoryFilter(filters, e.namespaces.prompt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
prompts, err := e.listing.ListPrompts(ctx)
|
return e.listing.ListPrompts(ctx, filter)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var filtered []domain.PromptIdentity
|
|
||||||
for _, prompt := range prompts {
|
|
||||||
if matchesListingNamespace(prompt.ID, mode, namespace) {
|
|
||||||
filtered = append(filtered, prompt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return filtered, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
|
||||||
@@ -307,26 +291,21 @@ func filterIdentityList(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
filters []ListingFilter,
|
filters []ListingFilter,
|
||||||
scope string,
|
scope string,
|
||||||
list func(context.Context) ([]string, error),
|
list func(context.Context, configuration.IdentityFilter) ([]string, error),
|
||||||
) ([]string, error) {
|
) ([]string, error) {
|
||||||
|
filter, err := inventoryFilter(filters, scope)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return list(ctx, filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func inventoryFilter(filters []ListingFilter, scope string) (configuration.IdentityFilter, error) {
|
||||||
mode, namespace, err := resolveListingFilter(filters, scope)
|
mode, namespace, err := resolveListingFilter(filters, scope)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return configuration.IdentityFilter{}, err
|
||||||
}
|
}
|
||||||
ids, err := list(ctx)
|
return configuration.IdentityFilter{All: mode == ListingAll, Namespace: namespace}, nil
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var filtered []string
|
|
||||||
for _, id := range ids {
|
|
||||||
if matchesListingNamespace(id, mode, namespace) {
|
|
||||||
filtered = append(filtered, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return filtered, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
|
||||||
@@ -355,19 +334,6 @@ func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterM
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func matchesListingNamespace(id string, mode ListingFilterMode, namespace string) bool {
|
|
||||||
switch mode {
|
|
||||||
case ListingNamed:
|
|
||||||
idNamespace, _, qualified := strings.Cut(id, "/")
|
|
||||||
return qualified && idNamespace == namespace
|
|
||||||
case ListingUnqualified:
|
|
||||||
return !strings.Contains(id, "/")
|
|
||||||
case ListingAll:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
|
||||||
request, err := e.qualifyRunRequest(request)
|
request, err := e.qualifyRunRequest(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -67,15 +67,6 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
|
|||||||
return files, err
|
return files, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// RelativePath computes a clean relative path from root to path.
|
|
||||||
func RelativePath(root string, filePath string) string {
|
|
||||||
rel, err := filepath.Rel(root, filePath)
|
|
||||||
if err != nil {
|
|
||||||
return filepath.Clean(filePath)
|
|
||||||
}
|
|
||||||
return filepath.Clean(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
// CleanFSRoot normalizes a root path for use with fs.FS while preserving
|
||||||
// nonblank leading and trailing whitespace.
|
// nonblank leading and trailing whitespace.
|
||||||
func CleanFSRoot(root string) string {
|
func CleanFSRoot(root string) string {
|
||||||
|
|||||||
@@ -81,16 +81,6 @@ func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRelativePathNested(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
|
||||||
got := RelativePath(root, path)
|
|
||||||
want := filepath.Join("nested", "profiles", "local.yaml")
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("expected relative path %q, got %q", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCleanFSRoot(t *testing.T) {
|
func TestCleanFSRoot(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -85,6 +85,12 @@ func (c *Coordinator) GenerateWithEvidence(
|
|||||||
return c.generate(ctx, req, true)
|
return c.generate(ctx, req, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreparationError identifies a built-in request preparation failure.
|
||||||
|
type PreparationError struct{ Cause error }
|
||||||
|
|
||||||
|
func (e *PreparationError) Error() string { return e.Cause.Error() }
|
||||||
|
func (e *PreparationError) Unwrap() error { return e.Cause }
|
||||||
|
|
||||||
func (c *Coordinator) generate(
|
func (c *Coordinator) generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -95,7 +101,7 @@ func (c *Coordinator) generate(
|
|||||||
}
|
}
|
||||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err, GenerationEvidence{}
|
return nil, &PreparationError{Cause: err}, GenerationEvidence{PreparationFailed: true}
|
||||||
}
|
}
|
||||||
|
|
||||||
evidence := GenerationEvidence{}
|
evidence := GenerationEvidence{}
|
||||||
@@ -120,66 +126,37 @@ func (c *Coordinator) generate(
|
|||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
ordinal := attempts + 1
|
ordinal := attempts + 1
|
||||||
outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
result := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
|
||||||
if dispatched {
|
if result.Dispatched {
|
||||||
attempts++
|
attempts++
|
||||||
}
|
}
|
||||||
if captureEvidence {
|
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
|
||||||
attempt := AttemptEvidence{
|
if result.Outcome.Err == nil {
|
||||||
StartedAt: startedAt.UTC(),
|
|
||||||
Dispatched: dispatched,
|
|
||||||
ResponseReceived: outcome.Evidence.ResponseReceived,
|
|
||||||
HTTP: cloneHTTPEvidence(outcome.Evidence),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
RequestDuration: generationDuration,
|
|
||||||
Failure: ClassifyFailure(outcome.Err),
|
|
||||||
ResponseDecoded: outcome.ResponseDecoded,
|
|
||||||
GenerationCompleted: outcome.GenerationCompleted,
|
|
||||||
}
|
|
||||||
if outcome.StructuredOutputRequested != nil {
|
|
||||||
requested := *outcome.StructuredOutputRequested
|
|
||||||
attempt.StructuredOutputRequested = &requested
|
|
||||||
}
|
|
||||||
if dispatched {
|
|
||||||
usage.Add(outcome.Usage, outcome.UsagePresence)
|
|
||||||
var attemptUsage observation.UsageAccumulator
|
|
||||||
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
|
|
||||||
attempt.Usage = attemptUsage.Observed()
|
|
||||||
}
|
|
||||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
|
||||||
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
|
|
||||||
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
|
|
||||||
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
|
||||||
}
|
|
||||||
if outcome.Err == nil {
|
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(outcome.Response, err)
|
return finish(result.Outcome.Response, err)
|
||||||
}
|
}
|
||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
return finish(outcome.Response, nil)
|
return finish(result.Outcome.Response, nil)
|
||||||
}
|
}
|
||||||
if !dispatched || !eligibleStatus(outcome) {
|
if !result.Dispatched || !eligibleStatus(result.Outcome) {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(outcome.Response, outcome.Err)
|
return finish(result.Outcome.Response, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
return finish(nil, outcome.Err)
|
return finish(nil, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
if retriesUsed == req.TransientRetries {
|
if retriesUsed == req.TransientRetries {
|
||||||
if retriesUsed == 0 {
|
if retriesUsed == 0 {
|
||||||
return finish(nil, outcome.Err)
|
return finish(nil, result.Outcome.Err)
|
||||||
}
|
}
|
||||||
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err})
|
return finish(nil, &ExhaustedError{attempts: attempts, cause: result.Outcome.Err})
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return finish(nil, err)
|
return finish(nil, err)
|
||||||
}
|
}
|
||||||
retryOrdinal := retriesUsed + 1
|
retryOrdinal := retriesUsed + 1
|
||||||
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now())
|
delay := retryDelay(c.backoff(retryOrdinal), result.Outcome.RetryNotBefore, c.now())
|
||||||
waitStart := c.now()
|
waitStart := c.now()
|
||||||
waitErr := c.wait(ctx, delay)
|
waitErr := c.wait(ctx, delay)
|
||||||
waitDuration := elapsed(waitStart, c.now())
|
waitDuration := elapsed(waitStart, c.now())
|
||||||
@@ -200,127 +177,133 @@ func (c *Coordinator) generate(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) generateInjected(
|
func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest, captureEvidence bool) (*domain.GenerateResponse, error, GenerationEvidence) {
|
||||||
ctx context.Context,
|
|
||||||
req domain.GenerateRequest,
|
|
||||||
captureEvidence bool,
|
|
||||||
) (*domain.GenerateResponse, error, GenerationEvidence) {
|
|
||||||
evidence := GenerationEvidence{}
|
evidence := GenerationEvidence{}
|
||||||
var usage observation.UsageAccumulator
|
var usage observation.UsageAccumulator
|
||||||
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Usage = usage.Observed()
|
|
||||||
}
|
|
||||||
return response, err, evidence
|
|
||||||
}
|
|
||||||
recorder := observation.FromContext(ctx)
|
|
||||||
generation := req.Generation
|
generation := req.Generation
|
||||||
if generation == 0 {
|
if generation == 0 {
|
||||||
generation = 1
|
generation = 1
|
||||||
}
|
}
|
||||||
start := c.now()
|
result := c.executeInjected(ctx, req)
|
||||||
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
|
||||||
capacityWait := c.capacityWait(req.Target.BackendID, start)
|
|
||||||
if err != nil {
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
|
||||||
StartedAt: start.UTC(),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
|
||||||
}
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
release()
|
|
||||||
if captureEvidence {
|
|
||||||
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
|
|
||||||
StartedAt: start.UTC(),
|
|
||||||
CapacityWait: capacityWait,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if recorder != nil {
|
|
||||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
|
|
||||||
}
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
|
||||||
dispatchStart := c.now()
|
|
||||||
response, err := c.injected.Generate(ctx, req)
|
|
||||||
duration := elapsed(dispatchStart, c.now())
|
|
||||||
if err == nil && response == nil {
|
|
||||||
err = llm.ErrMalformedResponse
|
|
||||||
}
|
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
attempt := AttemptEvidence{
|
evidence.Usage = usage.Observed()
|
||||||
StartedAt: start.UTC(),
|
|
||||||
Dispatched: true,
|
|
||||||
ResponseReceived: response != nil,
|
|
||||||
RequestDuration: duration,
|
|
||||||
Failure: ClassifyFailure(err),
|
|
||||||
GenerationCompleted: err == nil && response != nil,
|
|
||||||
}
|
|
||||||
requested := req.StructuredOutput != nil
|
|
||||||
attempt.StructuredOutputRequested = &requested
|
|
||||||
var reportedUsage domain.TokenUsage
|
|
||||||
var reportedPresence *domain.TokenUsagePresence
|
|
||||||
if err == nil && response != nil {
|
|
||||||
reportedUsage = response.Usage
|
|
||||||
reportedPresence = response.UsagePresence
|
|
||||||
}
|
|
||||||
usage.Add(reportedUsage, reportedPresence)
|
|
||||||
var attemptUsage observation.UsageAccumulator
|
|
||||||
attemptUsage.Add(reportedUsage, reportedPresence)
|
|
||||||
attempt.Usage = attemptUsage.Observed()
|
|
||||||
evidence.Attempts = append(evidence.Attempts, attempt)
|
|
||||||
}
|
}
|
||||||
release()
|
if result.Outcome.Err != nil {
|
||||||
if recorder != nil {
|
return nil, result.Outcome.Err, evidence
|
||||||
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
|
|
||||||
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
|
|
||||||
if err == nil {
|
|
||||||
data.Usage = response.Usage
|
|
||||||
data.UsagePresence = response.UsagePresence
|
|
||||||
}
|
|
||||||
recorder.Attempt(data)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return finish(nil, err)
|
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
return finish(response, err)
|
return result.Outcome.Response, err, evidence
|
||||||
}
|
}
|
||||||
return finish(nil, err)
|
return nil, err, evidence
|
||||||
}
|
}
|
||||||
return finish(response, nil)
|
return result.Outcome.Response, nil, evidence
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) {
|
func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRequest) attemptResult {
|
||||||
start := c.now()
|
result := attemptResult{StartedAt: c.now()}
|
||||||
startedAt = start
|
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
|
||||||
release, err := c.manager.Acquire(ctx, backendID)
|
result.CapacityWait = c.capacityWait(req.Target.BackendID, result.StartedAt)
|
||||||
capacityWait = c.capacityWait(backendID, start)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
start := c.now()
|
||||||
|
result.Dispatched = true
|
||||||
|
requested := req.StructuredOutput != nil
|
||||||
|
result.Outcome.StructuredOutputRequested = &requested
|
||||||
|
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
|
||||||
|
result.RequestDuration = elapsed(start, c.now())
|
||||||
|
if result.Outcome.Err == nil && result.Outcome.Response == nil {
|
||||||
|
result.Outcome.Err = llm.ErrMalformedResponse
|
||||||
|
}
|
||||||
|
if result.Outcome.Err == nil {
|
||||||
|
result.Outcome.Usage = result.Outcome.Response.Usage
|
||||||
|
result.Outcome.UsagePresence = result.Outcome.Response.UsagePresence
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
type attemptResult struct {
|
||||||
|
Outcome llm.AttemptOutcome
|
||||||
|
Dispatched bool
|
||||||
|
CapacityWait time.Duration
|
||||||
|
RequestDuration time.Duration
|
||||||
|
StartedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordAttempt runs after the attempt has released its permit, including when
|
||||||
|
// the recorder calls application code. Injected clients expose no HTTP facts.
|
||||||
|
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
|
||||||
|
outcome := result.Outcome
|
||||||
|
failure := ClassifyFailure(outcome.Err)
|
||||||
|
if capture {
|
||||||
|
attempt := AttemptEvidence{
|
||||||
|
StartedAt: result.StartedAt.UTC(), Dispatched: result.Dispatched,
|
||||||
|
CapacityWait: result.CapacityWait, RequestDuration: result.RequestDuration,
|
||||||
|
Failure: failure,
|
||||||
|
}
|
||||||
|
if injected {
|
||||||
|
attempt.ResponseReceived = outcome.Response != nil
|
||||||
|
attempt.GenerationCompleted = outcome.Err == nil && outcome.Response != nil
|
||||||
|
if result.Dispatched {
|
||||||
|
attempt.CapacityWait = 0
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
attempt.ResponseReceived = outcome.Evidence.ResponseReceived
|
||||||
|
attempt.HTTP = cloneHTTPEvidence(outcome.Evidence)
|
||||||
|
attempt.ResponseDecoded = outcome.ResponseDecoded
|
||||||
|
attempt.GenerationCompleted = outcome.GenerationCompleted
|
||||||
|
}
|
||||||
|
if outcome.StructuredOutputRequested != nil {
|
||||||
|
requested := *outcome.StructuredOutputRequested
|
||||||
|
attempt.StructuredOutputRequested = &requested
|
||||||
|
}
|
||||||
|
if result.Dispatched {
|
||||||
|
usage.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
var single observation.UsageAccumulator
|
||||||
|
single.Add(outcome.Usage, outcome.UsagePresence)
|
||||||
|
attempt.Usage = single.Observed()
|
||||||
|
}
|
||||||
|
evidence.Attempts = append(evidence.Attempts, attempt)
|
||||||
|
}
|
||||||
|
if recorder != nil {
|
||||||
|
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||||
|
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
|
||||||
|
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
|
||||||
|
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) attemptResult {
|
||||||
|
result := attemptResult{StartedAt: c.now()}
|
||||||
|
release, err := c.manager.Acquire(ctx, backendID)
|
||||||
|
result.CapacityWait = c.capacityWait(backendID, result.StartedAt)
|
||||||
|
if err != nil {
|
||||||
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
result.Outcome.Err = err
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
dispatchStart := c.now()
|
dispatchStart := c.now()
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
outcome = prepared.ExecuteWithEvidence(ctx)
|
result.Outcome = prepared.ExecuteWithEvidence(ctx)
|
||||||
dispatched = outcome.Evidence.DispatchInvoked
|
result.Dispatched = result.Outcome.Evidence.DispatchInvoked
|
||||||
} else {
|
} else {
|
||||||
outcome = prepared.Execute(ctx)
|
result.Outcome = prepared.Execute(ctx)
|
||||||
dispatched = true
|
result.Dispatched = true
|
||||||
}
|
}
|
||||||
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt
|
result.RequestDuration = elapsed(dispatchStart, c.now())
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {
|
||||||
|
|||||||
@@ -40,10 +40,7 @@ func testCoordinator(t *testing.T, transport transportFunc) *Coordinator {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
coordinator := NewBuiltIn(manager, client)
|
coordinator := NewBuiltIn(manager, client)
|
||||||
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
coordinator.jitter = func(time.Duration) time.Duration { return 0 }
|
||||||
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
// GenerationEvidence contains invocation-local request evidence without raw
|
// GenerationEvidence contains invocation-local request evidence without raw
|
||||||
// provider bodies, headers, credentials, or generated content.
|
// provider bodies, headers, credentials, or generated content.
|
||||||
type GenerationEvidence struct {
|
type GenerationEvidence struct {
|
||||||
|
// PreparationFailed identifies a built-in request rejected before attempts.
|
||||||
|
PreparationFailed bool
|
||||||
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
||||||
// Injected generation leaves it unknown.
|
// Injected generation leaves it unknown.
|
||||||
InitialEndpoint string
|
InitialEndpoint string
|
||||||
|
|||||||
@@ -144,6 +144,32 @@ func (c fixedClient) Generate(context.Context, domain.GenerateRequest) (*domain.
|
|||||||
return c.response, c.err
|
return c.response, c.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInjectedObserverRunsAfterPermitRelease(t *testing.T) {
|
||||||
|
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP request"); return nil, nil })
|
||||||
|
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{}})
|
||||||
|
called := false
|
||||||
|
recorder, err := observation.New(func(event domain.ExecutionEvent) {
|
||||||
|
if event.Kind != domain.ExecutionAttempt {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
called = true
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
release, err := coordinator.manager.Acquire(ctx, "limited")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("observer could not acquire permit: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := coordinator.Generate(observation.WithRecorder(context.Background(), recorder), testRequest(0)); err != nil || !called {
|
||||||
|
t.Fatalf("generation error = %v, observer called = %v", err, called)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
|
||||||
manager, err := capacity.NewManager(nil)
|
manager, err := capacity.NewManager(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
97
internal/jsonvalue/trusted.go
Normal file
97
internal/jsonvalue/trusted.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import "reflect"
|
||||||
|
|
||||||
|
func cloneTrustedMap(src map[string]any) map[string]any {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
out[k] = CloneTrusted(v)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// CloneTrusted detaches a previously accepted value without revalidating it.
|
||||||
|
func CloneTrusted(value any) any {
|
||||||
|
if value == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch v := value.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return cloneTrustedMap(v)
|
||||||
|
case []any:
|
||||||
|
out := make([]any, len(v))
|
||||||
|
for i, item := range v {
|
||||||
|
out[i] = CloneTrusted(item)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case []string:
|
||||||
|
if v == nil {
|
||||||
|
return []string(nil)
|
||||||
|
}
|
||||||
|
return append([]string{}, v...)
|
||||||
|
case []byte:
|
||||||
|
if v == nil {
|
||||||
|
return []byte(nil)
|
||||||
|
}
|
||||||
|
return append([]byte{}, v...)
|
||||||
|
default:
|
||||||
|
return cloneTrustedReflect(reflect.ValueOf(value)).Interface()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneTrustedReflect(value reflect.Value) reflect.Value {
|
||||||
|
if !value.IsValid() {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
switch value.Kind() {
|
||||||
|
case reflect.Interface:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
copied := cloneTrustedReflect(value.Elem())
|
||||||
|
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
|
||||||
|
return copied
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
out.Set(copied)
|
||||||
|
return out
|
||||||
|
case reflect.Pointer:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.New(value.Type().Elem())
|
||||||
|
out.Elem().Set(cloneTrustedReflect(value.Elem()))
|
||||||
|
return out
|
||||||
|
case reflect.Map:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeMapWithSize(value.Type(), value.Len())
|
||||||
|
iter := value.MapRange()
|
||||||
|
for iter.Next() {
|
||||||
|
out.SetMapIndex(cloneTrustedReflect(iter.Key()), cloneTrustedReflect(iter.Value()))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Slice:
|
||||||
|
if value.IsNil() {
|
||||||
|
return reflect.Zero(value.Type())
|
||||||
|
}
|
||||||
|
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case reflect.Array:
|
||||||
|
out := reflect.New(value.Type()).Elem()
|
||||||
|
for i := 0; i < value.Len(); i++ {
|
||||||
|
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
default:
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
}
|
||||||
27
internal/jsonvalue/trusted_test.go
Normal file
27
internal/jsonvalue/trusted_test.go
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
package jsonvalue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type trustedNamed string
|
||||||
|
|
||||||
|
func TestCloneTrustedPreservesTypesAndDetaches(t *testing.T) {
|
||||||
|
named := trustedNamed("value")
|
||||||
|
original := map[string]any{"": map[string]any{"number": json.Number("9007199254740993")}, "typed": []string{"original"}, "pointer": &named, "nilmap": map[string]any(nil), "nilslice": []string(nil), "empty": []any{}}
|
||||||
|
copy := CloneTrusted(original).(map[string]any)
|
||||||
|
if !reflect.DeepEqual(copy, original) {
|
||||||
|
t.Fatalf("clone = %#v", copy)
|
||||||
|
}
|
||||||
|
copy[""].(map[string]any)["number"] = json.Number("1")
|
||||||
|
copy["typed"].([]string)[0] = "changed"
|
||||||
|
*copy["pointer"].(*trustedNamed) = "changed"
|
||||||
|
if original[""].(map[string]any)["number"] != json.Number("9007199254740993") || original["typed"].([]string)[0] != "original" || named != "value" {
|
||||||
|
t.Fatal("clone shares mutable storage")
|
||||||
|
}
|
||||||
|
if copy["nilmap"].(map[string]any) != nil || copy["nilslice"].([]string) != nil || copy["empty"].([]any) == nil {
|
||||||
|
t.Fatal("nil/empty shape changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,10 +25,7 @@ func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
|
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
|
||||||
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
|
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -88,7 +85,7 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
|
||||||
var authorization string
|
var authorization string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
authorization = request.Header.Get("Authorization")
|
authorization = request.Header.Get("Authorization")
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -98,9 +95,6 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, evidence, err := client.GetWithAuthenticationEvidence(
|
response, evidence, err := client.GetWithAuthenticationEvidence(
|
||||||
context.Background(), "https://example.invalid/key", 0, tc.target,
|
context.Background(), "https://example.invalid/key", 0, tc.target,
|
||||||
@@ -269,10 +263,7 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
|
|||||||
|
|
||||||
transport := http.DefaultTransport
|
transport := http.DefaultTransport
|
||||||
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
|
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
const calls = 8
|
const calls = 8
|
||||||
evidences := make([]HTTPExecutionEvidence, calls)
|
evidences := make([]HTTPExecutionEvidence, calls)
|
||||||
@@ -307,15 +298,12 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
|
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
|
||||||
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
<-request.Context().Done()
|
<-request.Context().Done()
|
||||||
return nil, request.Context().Err()
|
return nil, request.Context().Err()
|
||||||
}),
|
}),
|
||||||
}})
|
}})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
|
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
|
||||||
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
|
if response != nil || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
|||||||
@@ -17,10 +17,10 @@ import (
|
|||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
|
||||||
ErrInvalidRequest = errors.New("invalid generate request")
|
ErrInvalidRequest = errors.New("invalid generate request")
|
||||||
ErrRequestFailed = errors.New("llm request failed")
|
ErrRequestFailed = errors.New("llm request failed")
|
||||||
ErrTransport = errors.New("llm transport failed")
|
ErrTransport = errors.New("llm transport failed")
|
||||||
@@ -43,28 +43,15 @@ func (e *requestFailedError) Unwrap() []error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type OpenAICompatibleConfig struct {
|
type OpenAICompatibleConfig struct {
|
||||||
BaseURL string
|
|
||||||
Model string
|
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
type OpenAICompatibleClient struct {
|
type OpenAICompatibleClient struct {
|
||||||
baseURL string
|
httpClient *http.Client
|
||||||
defaultModel string
|
|
||||||
httpClient *http.Client
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) *OpenAICompatibleClient {
|
||||||
baseURL := ""
|
|
||||||
if strings.TrimSpace(cfg.BaseURL) != "" {
|
|
||||||
var err error
|
|
||||||
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout := cfg.Timeout
|
timeout := cfg.Timeout
|
||||||
if timeout <= 0 {
|
if timeout <= 0 {
|
||||||
timeout = defaults.LLMRequestTimeoutDefault
|
timeout = defaults.LLMRequestTimeoutDefault
|
||||||
@@ -82,10 +69,8 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &OpenAICompatibleClient{
|
return &OpenAICompatibleClient{
|
||||||
baseURL: baseURL,
|
httpClient: client,
|
||||||
defaultModel: cfg.Model,
|
}
|
||||||
httpClient: client,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
@@ -132,11 +117,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
selectedEndpoint := req.Target.Endpoint
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(req.Target.Endpoint)
|
||||||
if strings.TrimSpace(selectedEndpoint) == "" {
|
|
||||||
selectedEndpoint = c.baseURL
|
|
||||||
}
|
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -145,12 +126,10 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
// Custom JSON encoders may mutate their receivers. Both payload encoding
|
||||||
if err != nil {
|
// and encoding-failure diagnostics must use values detached from metadata.
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
req.Target.ExtraParams = jsonvalue.CloneTrusted(req.Target.ExtraParams).(map[string]any)
|
||||||
}
|
wirePayload, err := openAIChatRequestPayload(req)
|
||||||
|
|
||||||
wirePayload, err := openAIChatRequestPayload(wireReq)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -179,7 +158,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
|||||||
return &PreparedGeneration{
|
return &PreparedGeneration{
|
||||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||||
structuredOutputRequested: wireReq.ResponseFormat != nil,
|
structuredOutputRequested: wirePayload["response_format"] != nil,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -370,40 +349,69 @@ func tokenCount(value *int) int {
|
|||||||
return *value
|
return *value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type responseErrorReader struct {
|
||||||
|
reader io.Reader
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *responseErrorReader) Read(p []byte) (int, error) {
|
||||||
|
n, err := r.reader.Read(p)
|
||||||
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
r.err = err
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
|
||||||
|
observed := &responseErrorReader{reader: body}
|
||||||
limited := &io.LimitedReader{
|
limited := &io.LimitedReader{
|
||||||
R: body,
|
R: observed,
|
||||||
N: maxOpenAIChatResponseBytes + 1,
|
N: maxOpenAIChatResponseBytes + 1,
|
||||||
}
|
}
|
||||||
decoder := json.NewDecoder(limited)
|
decoder := json.NewDecoder(limited)
|
||||||
|
|
||||||
var response openAIChatResponse
|
var response openAIChatResponse
|
||||||
if err := decoder.Decode(&response); err != nil {
|
if err := decoder.Decode(&response); err != nil {
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
}
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
|
||||||
var trailing any
|
var trailing any
|
||||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
return openAIChatResponse{}, &requestFailedError{cause: err}
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
}
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
if limited.N == 0 {
|
if limited.N == 0 {
|
||||||
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
return openAIChatResponse{}, openAIChatResponseTooLargeError()
|
||||||
}
|
}
|
||||||
|
if observed.err != nil {
|
||||||
|
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
|
||||||
|
}
|
||||||
|
|
||||||
return response, nil
|
return response, nil
|
||||||
}
|
}
|
||||||
@@ -416,99 +424,52 @@ func openAIChatResponseTooLargeError() error {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
func openAIChatRequestPayload(req domain.GenerateRequest) (map[string]any, error) {
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
model := strings.TrimSpace(req.Target.Model)
|
||||||
if model == "" {
|
if model == "" {
|
||||||
model = strings.TrimSpace(defaultModel)
|
return nil, errors.New("model is required")
|
||||||
}
|
|
||||||
if model == "" {
|
|
||||||
return openAIChatRequest{}, errors.New("model is required")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
wireReq := openAIChatRequest{
|
|
||||||
Model: model,
|
|
||||||
}
|
|
||||||
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return openAIChatRequest{}, err
|
return nil, err
|
||||||
}
|
}
|
||||||
wireReq.SessionID = sessionID
|
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
messages := make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||||
for _, msg := range req.Prompt.Messages {
|
for _, msg := range req.Prompt.Messages {
|
||||||
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
messages = append(messages, openAIChatRequestMessageFromRenderedMessage(msg))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
out := map[string]any{"model": model, "messages": messages}
|
||||||
|
if sessionID != "" {
|
||||||
|
out["session_id"] = sessionID
|
||||||
|
}
|
||||||
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
|
||||||
wireReq.Temperature = &req.Target.Temperature
|
out["temperature"] = req.Target.Temperature
|
||||||
}
|
}
|
||||||
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
|
||||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
out["max_tokens"] = req.Target.MaxTokens
|
||||||
}
|
}
|
||||||
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
if req.Target.TopP != 0 || req.TargetPresence.TopP {
|
||||||
wireReq.TopP = &req.Target.TopP
|
out["top_p"] = req.Target.TopP
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
||||||
wireReq.ServiceTier = req.Target.ServiceTier
|
out["service_tier"] = req.Target.ServiceTier
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
|
||||||
wireReq.ReasoningEffort = req.Target.ReasoningEffort
|
out["reasoning_effort"] = req.Target.ReasoningEffort
|
||||||
}
|
|
||||||
if len(req.Target.ExtraParams) > 0 {
|
|
||||||
wireReq.ExtraParams = req.Target.ExtraParams
|
|
||||||
}
|
}
|
||||||
if req.StructuredOutput != nil {
|
if req.StructuredOutput != nil {
|
||||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return openAIChatRequest{}, err
|
return nil, err
|
||||||
|
}
|
||||||
|
if responseFormat != nil {
|
||||||
|
out["response_format"] = responseFormat
|
||||||
}
|
}
|
||||||
wireReq.ResponseFormat = responseFormat
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return wireReq, nil
|
for key, value := range req.Target.ExtraParams {
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
|
||||||
Model string `json:"model"`
|
|
||||||
SessionID string `json:"session_id,omitempty"`
|
|
||||||
Messages []openAIChatRequestMessage `json:"messages"`
|
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
|
||||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
|
||||||
ExtraParams map[string]any `json:"-"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|
||||||
out := map[string]any{
|
|
||||||
"model": req.Model,
|
|
||||||
"messages": req.Messages,
|
|
||||||
}
|
|
||||||
if req.SessionID != "" {
|
|
||||||
out["session_id"] = req.SessionID
|
|
||||||
}
|
|
||||||
if req.Temperature != nil {
|
|
||||||
out["temperature"] = *req.Temperature
|
|
||||||
}
|
|
||||||
if req.MaxTokens != nil {
|
|
||||||
out["max_tokens"] = *req.MaxTokens
|
|
||||||
}
|
|
||||||
if req.TopP != nil {
|
|
||||||
out["top_p"] = *req.TopP
|
|
||||||
}
|
|
||||||
if req.ServiceTier != "" {
|
|
||||||
out["service_tier"] = req.ServiceTier
|
|
||||||
}
|
|
||||||
if req.ReasoningEffort != "" {
|
|
||||||
out["reasoning_effort"] = req.ReasoningEffort
|
|
||||||
}
|
|
||||||
if req.ResponseFormat != nil {
|
|
||||||
out["response_format"] = req.ResponseFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
for key, value := range req.ExtraParams {
|
|
||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, errors.New("extra_params key must not be empty")
|
return nil, errors.New("extra_params key must not be empty")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -123,26 +123,19 @@ func (p *recordingProvider) lastRequest(t *testing.T) recordedProviderRequest {
|
|||||||
return p.requests[len(p.requests)-1]
|
return p.requests[len(p.requests)-1]
|
||||||
}
|
}
|
||||||
|
|
||||||
func newProviderClient(t *testing.T, provider *recordingProvider, config OpenAICompatibleConfig) *OpenAICompatibleClient {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if config.BaseURL == "" {
|
|
||||||
config.BaseURL = provider.endpoint("/v1")
|
|
||||||
}
|
|
||||||
client, err := NewOpenAICompatibleClient(config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
|
|
||||||
func ordinaryGenerateRequest() domain.GenerateRequest {
|
func ordinaryGenerateRequest() domain.GenerateRequest {
|
||||||
return domain.GenerateRequest{
|
return domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "model"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *recordingProvider) generateRequest() domain.GenerateRequest {
|
||||||
|
req := ordinaryGenerateRequest()
|
||||||
|
req.Target.Endpoint = p.endpoint("/v1")
|
||||||
|
return req
|
||||||
|
}
|
||||||
|
|
||||||
type deadlineCapturingTransport struct {
|
type deadlineCapturingTransport struct {
|
||||||
deadline time.Time
|
deadline time.Time
|
||||||
hasDeadline bool
|
hasDeadline bool
|
||||||
@@ -253,35 +246,13 @@ func assertDeadlineNear(t *testing.T, deadline, before, after time.Time, duratio
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewOpenAICompatibleClientRejectsInvalidBaseURL(t *testing.T) {
|
|
||||||
for _, endpoint := range []string{
|
|
||||||
"://invalid",
|
|
||||||
"/v1",
|
|
||||||
"https:///v1",
|
|
||||||
"ftp://provider.example/v1",
|
|
||||||
"https://user@provider.example/v1",
|
|
||||||
"https://provider.example/v1?mode=chat",
|
|
||||||
"https://provider.example/v1#chat",
|
|
||||||
} {
|
|
||||||
t.Run(endpoint, func(t *testing.T) {
|
|
||||||
_, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: endpoint})
|
|
||||||
if !errors.Is(err, ErrInvalidConfig) {
|
|
||||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
|
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
|
||||||
transport := http.DefaultTransport
|
transport := http.DefaultTransport
|
||||||
supplied := &http.Client{Transport: transport}
|
supplied := &http.Client{Transport: transport}
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != 0 {
|
if supplied.Timeout != 0 {
|
||||||
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
|
||||||
@@ -305,13 +276,10 @@ func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *t
|
|||||||
Transport: transport,
|
Transport: transport,
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: 2 * time.Second,
|
Timeout: 2 * time.Second,
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != suppliedTimeout {
|
if supplied.Timeout != suppliedTimeout {
|
||||||
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
|
||||||
@@ -335,13 +303,10 @@ func TestNewOpenAICompatibleClientTreatsSuppliedNegativeTimeoutAsUnset(t *testin
|
|||||||
}
|
}
|
||||||
configuredTimeout := 23 * time.Second
|
configuredTimeout := 23 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: configuredTimeout,
|
Timeout: configuredTimeout,
|
||||||
HTTPClient: supplied,
|
HTTPClient: supplied,
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if supplied.Timeout != -time.Second {
|
if supplied.Timeout != -time.Second {
|
||||||
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
|
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
|
||||||
@@ -370,7 +335,6 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
|||||||
{name: "optional response fields", run: checkOptionalResponseFieldOmission},
|
{name: "optional response fields", run: checkOptionalResponseFieldOmission},
|
||||||
{name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping},
|
{name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping},
|
||||||
{name: "optional request field presence", run: checkRequestFieldPresence},
|
{name: "optional request field presence", run: checkRequestFieldPresence},
|
||||||
{name: "configured model fallback", run: checkConfiguredModelFallback},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -380,14 +344,14 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
|
|||||||
|
|
||||||
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
|
func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
const content = " exact content\nwith whitespace "
|
const content = " exact content\nwith whitespace "
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
|
||||||
Role: domain.RoleDeveloper,
|
Role: domain.RoleDeveloper,
|
||||||
Content: content,
|
Content: content,
|
||||||
}}},
|
}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Generate() error = %v", err)
|
t.Fatalf("Generate() error = %v", err)
|
||||||
@@ -413,7 +377,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
|
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
|
||||||
}`)
|
}`)
|
||||||
|
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
Timeout: 2 * time.Second,
|
Timeout: 2 * time.Second,
|
||||||
})
|
})
|
||||||
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key")
|
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key")
|
||||||
@@ -424,6 +388,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
{Role: "user", Content: "Say hello"},
|
{Role: "user", Content: "Say hello"},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: provider.endpoint("/v1"),
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
Temperature: 0.4,
|
Temperature: 0.4,
|
||||||
MaxTokens: 123,
|
MaxTokens: 123,
|
||||||
@@ -594,9 +559,11 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
tc.configureEnv(t)
|
tc.configureEnv(t)
|
||||||
}
|
}
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Target = tc.target
|
request.Target = tc.target
|
||||||
|
request.Target.Endpoint = provider.endpoint("/v1")
|
||||||
|
request.Target.Model = "model"
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), request)
|
_, err := client.Generate(context.Background(), request)
|
||||||
if tc.wantError != nil {
|
if tc.wantError != nil {
|
||||||
@@ -625,7 +592,7 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
|
|
||||||
func checkCacheControlledMessageMapping(t *testing.T) {
|
func checkCacheControlledMessageMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
@@ -639,7 +606,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{Role: "user", Content: "Dynamic request."},
|
{Role: "user", Content: "Dynamic request."},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -691,7 +658,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
|
|||||||
|
|
||||||
func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||||
@@ -703,7 +670,7 @@ func checkEmptyCacheControlTTLOmission(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -745,8 +712,8 @@ func checkSessionIDMapping(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Prompt.SessionID = tc.sessionID
|
request.Prompt.SessionID = tc.sessionID
|
||||||
|
|
||||||
if _, err := client.Generate(context.Background(), request); err != nil {
|
if _, err := client.Generate(context.Background(), request); err != nil {
|
||||||
@@ -766,19 +733,14 @@ func checkSessionIDMapping(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "model",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{
|
Prompt: domain.RenderedPrompt{
|
||||||
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
|
||||||
},
|
},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected invalid request error")
|
t.Fatal("expected invalid request error")
|
||||||
@@ -828,9 +790,9 @@ func checkContentPresence(t *testing.T) {
|
|||||||
"cache_write_tokens": 5
|
"cache_write_tokens": 5
|
||||||
}
|
}
|
||||||
}`)
|
}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
response, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
}
|
}
|
||||||
@@ -865,10 +827,11 @@ func checkCacheUsageMapping(t *testing.T) {
|
|||||||
"cache_write_tokens": 60
|
"cache_write_tokens": 60
|
||||||
}
|
}
|
||||||
}`)
|
}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -901,8 +864,8 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
|
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
resp, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
resp, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -913,17 +876,17 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
|
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
if resp, err := client.Generate(context.Background(), ordinaryGenerateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
|
if resp, err := client.Generate(context.Background(), provider.generateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
|
||||||
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
|
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
_, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -938,11 +901,12 @@ func checkOptionalResponseFieldOmission(t *testing.T) {
|
|||||||
|
|
||||||
func checkReasoningAndExtraParameterMapping(t *testing.T) {
|
func checkReasoningAndExtraParameterMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: provider.endpoint("/v1"),
|
||||||
Model: "model",
|
Model: "model",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: "high",
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
@@ -1020,8 +984,8 @@ func checkRequestFieldPresence(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.TargetPresence = tc.presence
|
request.TargetPresence = tc.presence
|
||||||
|
|
||||||
if _, err := client.Generate(context.Background(), request); err != nil {
|
if _, err := client.Generate(context.Background(), request); err != nil {
|
||||||
@@ -1073,21 +1037,17 @@ func checkOmittedGenerationTimeout(t *testing.T) {
|
|||||||
transport := &deadlineCapturingTransport{}
|
transport := &deadlineCapturingTransport{}
|
||||||
clientTimeout := 5 * time.Second
|
clientTimeout := 5 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Timeout: clientTimeout,
|
Timeout: clientTimeout,
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model", TimeoutSeconds: 0},
|
||||||
})
|
})
|
||||||
after := time.Now()
|
after := time.Now()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -1146,11 +1106,11 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams},
|
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model", ExtraParams: tc.extraParams},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected invalid request error")
|
t.Fatal("expected invalid request error")
|
||||||
@@ -1192,8 +1152,8 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
request := ordinaryGenerateRequest()
|
request := provider.generateRequest()
|
||||||
request.Target.ExtraParams = tc.extraParams
|
request.Target.ExtraParams = tc.extraParams
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
@@ -1218,30 +1178,12 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkConfiguredModelFallback(t *testing.T) {
|
|
||||||
provider := newRecordingProvider(t)
|
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
|
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
var model string
|
|
||||||
if !provider.lastRequest(t).decodeField(t, "model", &model) || model != "default-model" {
|
|
||||||
t.Fatalf("model = %q, want default-model", model)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
run func(*testing.T)
|
run func(*testing.T)
|
||||||
}{
|
}{
|
||||||
{name: "request endpoint overrides configured endpoint", run: checkEndpointOverride},
|
{name: "request endpoint", run: checkSelectedEndpoint},
|
||||||
{name: "request endpoint works without configured endpoint", run: checkEmptyConfiguredEndpoint},
|
|
||||||
{name: "completion URL preserves valid base paths", run: checkCompletionURLComposition},
|
{name: "completion URL preserves valid base paths", run: checkCompletionURLComposition},
|
||||||
{name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection},
|
{name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection},
|
||||||
{name: "an endpoint is required", run: checkRequiredEndpoint},
|
{name: "an endpoint is required", run: checkRequiredEndpoint},
|
||||||
@@ -1252,35 +1194,6 @@ func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkEndpointOverride(t *testing.T) {
|
|
||||||
configuredProvider := newRecordingProvider(t)
|
|
||||||
configuredProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"default"}}]}`)
|
|
||||||
selectedProvider := newRecordingProvider(t)
|
|
||||||
selectedProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"override"}}]}`)
|
|
||||||
|
|
||||||
client := newProviderClient(t, configuredProvider, OpenAICompatibleConfig{Model: "m"})
|
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{Endpoint: selectedProvider.endpoint("/v1")},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if resp.Content != "override" {
|
|
||||||
t.Fatalf("expected override response, got %q", resp.Content)
|
|
||||||
}
|
|
||||||
if got := configuredProvider.requestCount(); got != 0 {
|
|
||||||
t.Fatalf("configured provider calls = %d, want 0", got)
|
|
||||||
}
|
|
||||||
if got := selectedProvider.requestCount(); got != 1 {
|
|
||||||
t.Fatalf("selected provider calls = %d, want 1", got)
|
|
||||||
}
|
|
||||||
if got := selectedProvider.lastRequest(t).url; got != "/v1/chat/completions" {
|
|
||||||
t.Fatalf("selected URL = %q, want /v1/chat/completions", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkCommonResponseFailures(t *testing.T) {
|
func checkCommonResponseFailures(t *testing.T) {
|
||||||
const sensitiveBody = `provider-secret-fragment request_payload_details`
|
const sensitiveBody = `provider-secret-fragment request_payload_details`
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -1310,9 +1223,9 @@ func checkCommonResponseFailures(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(tc.statusCode, tc.body)
|
provider.respond(tc.statusCode, tc.body)
|
||||||
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "m"})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
response, err := client.Generate(context.Background(), provider.generateRequest())
|
||||||
if response != nil {
|
if response != nil {
|
||||||
t.Fatalf("expected no partial response, got %+v", response)
|
t.Fatalf("expected no partial response, got %+v", response)
|
||||||
}
|
}
|
||||||
@@ -1357,9 +1270,7 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
body := sizedSuccessResponseBody(tc.size)
|
body := sizedSuccessResponseBody(tc.size)
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1370,12 +1281,10 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if tc.wantErr {
|
if tc.wantErr {
|
||||||
if response != nil {
|
if response != nil {
|
||||||
@@ -1425,9 +1334,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
strings.NewReader(prefix),
|
strings.NewReader(prefix),
|
||||||
continuation,
|
continuation,
|
||||||
)}
|
)}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1438,9 +1345,6 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type outcome struct {
|
type outcome struct {
|
||||||
response *domain.GenerateResponse
|
response *domain.GenerateResponse
|
||||||
@@ -1450,6 +1354,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
|
|||||||
go func() {
|
go func() {
|
||||||
response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{
|
response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
done <- outcome{response: response, err: generateErr}
|
done <- outcome{response: response, err: generateErr}
|
||||||
}()
|
}()
|
||||||
@@ -1497,9 +1402,7 @@ func checkSingleResponseDocument(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
body := &countingReadCloser{reader: strings.NewReader(tc.body)}
|
body := &countingReadCloser{reader: strings.NewReader(tc.body)}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -1510,12 +1413,10 @@ func checkSingleResponseDocument(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if tc.wantErr {
|
if tc.wantErr {
|
||||||
if response != nil {
|
if response != nil {
|
||||||
@@ -1541,22 +1442,19 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
|
|||||||
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
||||||
generationTimeout := 2 * time.Second
|
generationTimeout := 2 * time.Second
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: 10 * time.Second,
|
Timeout: 10 * time.Second,
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://example.com/v1",
|
||||||
|
Model: "m",
|
||||||
TimeoutSeconds: int(generationTimeout / time.Second),
|
TimeoutSeconds: int(generationTimeout / time.Second),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -1578,25 +1476,22 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
|
|||||||
|
|
||||||
func checkCallerDeadlinePrecedence(t *testing.T) {
|
func checkCallerDeadlinePrecedence(t *testing.T) {
|
||||||
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: 10 * time.Second,
|
Timeout: 10 * time.Second,
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
callerDeadline := time.Now().Add(time.Second)
|
callerDeadline := time.Now().Add(time.Second)
|
||||||
ctx, cancel := context.WithDeadline(context.Background(), callerDeadline)
|
ctx, cancel := context.WithDeadline(context.Background(), callerDeadline)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{
|
Target: domain.ExecutionTarget{
|
||||||
|
Endpoint: "http://example.com/v1",
|
||||||
|
Model: "m",
|
||||||
TimeoutSeconds: 2,
|
TimeoutSeconds: 2,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -1618,22 +1513,18 @@ func checkCallerDeadlinePrecedence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkCallerCancellationIdentity(t *testing.T) {
|
func checkCallerCancellationIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: contextErrorTransport{},
|
Transport: contextErrorTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected cancellation error")
|
t.Fatal("expected cancellation error")
|
||||||
@@ -1647,21 +1538,17 @@ func checkCallerCancellationIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Transport: contextErrorTransport{},
|
Transport: contextErrorTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0))
|
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0))
|
||||||
defer cancel()
|
defer cancel()
|
||||||
_, err = client.Generate(ctx, domain.GenerateRequest{
|
_, err := client.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
t.Fatalf("expected request failure and caller deadline identities, got %v", err)
|
t.Fatalf("expected request failure and caller deadline identities, got %v", err)
|
||||||
@@ -1669,20 +1556,16 @@ func checkExpiredCallerDeadlineIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkWholeRequestTimeoutIdentity(t *testing.T) {
|
func checkWholeRequestTimeoutIdentity(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{
|
HTTPClient: &http.Client{
|
||||||
Timeout: time.Millisecond,
|
Timeout: time.Millisecond,
|
||||||
Transport: waitingContextTransport{},
|
Transport: waitingContextTransport{},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
|
||||||
t.Fatalf("expected request failure and client timeout identities, got %v", err)
|
t.Fatalf("expected request failure and client timeout identities, got %v", err)
|
||||||
@@ -1696,20 +1579,15 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
|
|||||||
apiKey = "sensitive-api-key"
|
apiKey = "sensitive-api-key"
|
||||||
content = "sensitive prompt content"
|
content = "sensitive prompt content"
|
||||||
)
|
)
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: endpoint,
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return nil, transportCause
|
return nil, transportCause
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}},
|
||||||
Target: domain.ExecutionTarget{APIKey: apiKey},
|
Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m", APIKey: apiKey},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) {
|
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) {
|
||||||
t.Fatalf("expected request failure and transport cause identities, got %v", err)
|
t.Fatalf("expected request failure and transport cause identities, got %v", err)
|
||||||
@@ -1726,13 +1604,7 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type testCase struct {
|
type testCase struct {
|
||||||
name string
|
name string
|
||||||
@@ -1765,11 +1637,9 @@ func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkEmptyConfiguredEndpoint(t *testing.T) {
|
func checkSelectedEndpoint(t *testing.T) {
|
||||||
var selectedURL string
|
var selectedURL string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
selectedURL = req.URL.String()
|
selectedURL = req.URL.String()
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -1782,13 +1652,10 @@ func checkEmptyConfiguredEndpoint(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected empty configured base URL to be allowed, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
response, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1"},
|
Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1", Model: "m"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate with request endpoint: %v", err)
|
t.Fatalf("generate with request endpoint: %v", err)
|
||||||
@@ -1817,9 +1684,7 @@ func checkCompletionURLComposition(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
var selectedURL string
|
var selectedURL string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: tc.baseURL,
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
selectedURL = req.URL.String()
|
selectedURL = req.URL.String()
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
@@ -1830,12 +1695,10 @@ func checkCompletionURLComposition(t *testing.T) {
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: tc.baseURL, Model: "m"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
@@ -1860,21 +1723,16 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
|
|||||||
for _, endpoint := range invalidEndpoints {
|
for _, endpoint := range invalidEndpoints {
|
||||||
t.Run(endpoint, func(t *testing.T) {
|
t.Run(endpoint, func(t *testing.T) {
|
||||||
transportCalls := 0
|
transportCalls := 0
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://configured.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
transportCalls++
|
transportCalls++
|
||||||
return nil, errors.New("transport must not be called")
|
return nil, errors.New("transport must not be called")
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{Endpoint: endpoint},
|
Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m"},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
@@ -1890,17 +1748,11 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func checkRequiredEndpoint(t *testing.T) {
|
func checkRequiredEndpoint(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
_, err := client.Generate(context.Background(), domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{},
|
Target: domain.ExecutionTarget{Model: "m"},
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected endpoint-required error")
|
t.Fatal("expected endpoint-required error")
|
||||||
@@ -1909,3 +1761,13 @@ func checkRequiredEndpoint(t *testing.T) {
|
|||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAICompatibleClientRequiresModel(t *testing.T) {
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
_, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1"},
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("missing model error = %v, want invalid request", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
108
internal/llm/openrouter_authentication.go
Normal file
108
internal/llm/openrouter_authentication.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxAuthenticationResponseBytes = 1 << 20
|
||||||
|
|
||||||
|
// AuthenticationEvidence describes the observable result of a key probe.
|
||||||
|
type AuthenticationEvidence struct {
|
||||||
|
HTTP HTTPExecutionEvidence
|
||||||
|
Accepted *bool
|
||||||
|
Overflow bool
|
||||||
|
Unrecognized bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenRouterKeyEndpoint derives the default key endpoint from a base endpoint.
|
||||||
|
func OpenRouterKeyEndpoint(base string) (string, error) {
|
||||||
|
return url.JoinPath(base, "key")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProbeOpenRouterKey executes and interprets an OpenRouter key request.
|
||||||
|
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
|
||||||
|
parsed, err := url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return AuthenticationEvidence{}, err
|
||||||
|
}
|
||||||
|
parsed.User = nil
|
||||||
|
response, evidence, requestErr := c.GetWithAuthenticationEvidence(ctx, parsed.String(), timeoutSeconds, target)
|
||||||
|
result := AuthenticationEvidence{HTTP: evidence}
|
||||||
|
if response != nil && response.Body != nil {
|
||||||
|
defer response.Body.Close()
|
||||||
|
}
|
||||||
|
if !evidence.ResponseReceived && response == nil {
|
||||||
|
return result, requestErr
|
||||||
|
}
|
||||||
|
status := evidence.HTTPStatus
|
||||||
|
if status == nil && response != nil {
|
||||||
|
status = &response.StatusCode
|
||||||
|
}
|
||||||
|
if status == nil {
|
||||||
|
result.Unrecognized = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
switch *status {
|
||||||
|
case http.StatusUnauthorized:
|
||||||
|
accepted := false
|
||||||
|
result.Accepted = &accepted
|
||||||
|
case http.StatusOK:
|
||||||
|
if response == nil || response.Body == nil {
|
||||||
|
result.Unrecognized = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
contents, err := io.ReadAll(io.LimitReader(response.Body, maxAuthenticationResponseBytes+1))
|
||||||
|
if err != nil && ctx.Err() != nil {
|
||||||
|
return result, ctx.Err()
|
||||||
|
}
|
||||||
|
if len(contents) > maxAuthenticationResponseBytes {
|
||||||
|
result.Overflow = true
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
if recognizedOpenRouterKeyResponse(contents) {
|
||||||
|
accepted := true
|
||||||
|
result.Accepted = &accepted
|
||||||
|
} else {
|
||||||
|
result.Unrecognized = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recognizedOpenRouterKeyResponse(contents []byte) bool {
|
||||||
|
var envelope map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var data map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(envelope["data"], &data); err != nil || data == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
label := bytes.TrimSpace(data["label"])
|
||||||
|
var labelValue string
|
||||||
|
if len(label) == 0 || label[0] != '"' || json.Unmarshal(label, &labelValue) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
freeTier := bytes.TrimSpace(data["is_free_tier"])
|
||||||
|
if !bytes.Equal(freeTier, []byte("true")) && !bytes.Equal(freeTier, []byte("false")) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data["usage"]))
|
||||||
|
decoder.UseNumber()
|
||||||
|
var number any
|
||||||
|
if decoder.Decode(&number) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, ok := number.(json.Number)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
134
internal/llm/openrouter_authentication_test.go
Normal file
134
internal/llm/openrouter_authentication_test.go
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKey(t *testing.T) {
|
||||||
|
valid := `{"data":{"label":"x","is_free_tier":false,"usage":0}}`
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
body string
|
||||||
|
key string
|
||||||
|
accepted *bool
|
||||||
|
unrecognized, overflow bool
|
||||||
|
}{
|
||||||
|
{name: "recognized", status: 200, body: valid, key: "secret", accepted: boolValue(true)},
|
||||||
|
{name: "exact limit", status: 200, body: valid + strings.Repeat(" ", maxAuthenticationResponseBytes-len(valid)), accepted: boolValue(true)},
|
||||||
|
{name: "unknown", status: 200, body: `{"data":{"label":"x","is_free_tier":false,"usage":"0"}}`, unrecognized: true},
|
||||||
|
{name: "trailing value", status: 200, body: valid + ` {}`, unrecognized: true},
|
||||||
|
{name: "overflow", status: 200, body: strings.Repeat("x", maxAuthenticationResponseBytes+1), overflow: true},
|
||||||
|
{name: "unauthorized anonymous", status: 401, accepted: boolValue(false)},
|
||||||
|
{name: "other status", status: 503},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var authorization string
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
if req.URL.User != nil {
|
||||||
|
t.Fatal("request retained URL credentials")
|
||||||
|
}
|
||||||
|
authorization = req.Header.Get("Authorization")
|
||||||
|
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.HTTP.Endpoint != "https://example.invalid/key" || got.HTTP.HTTPStatus == nil || *got.HTTP.HTTPStatus != tc.status || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow {
|
||||||
|
t.Fatalf("evidence = %+v", got)
|
||||||
|
}
|
||||||
|
if (got.Accepted == nil) != (tc.accepted == nil) || got.Accepted != nil && *got.Accepted != *tc.accepted {
|
||||||
|
t.Fatalf("acceptance = %v", got.Accepted)
|
||||||
|
}
|
||||||
|
if (authorization != "") != (tc.key != "") || got.HTTP.CredentialsSupplied != (tc.key != "") {
|
||||||
|
t.Fatalf("credential evidence = %+v", got.HTTP)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func boolValue(value bool) *bool { return &value }
|
||||||
|
|
||||||
|
type authenticationFailingBody struct{ err error }
|
||||||
|
|
||||||
|
func (b authenticationFailingBody) Read([]byte) (int, error) { return 0, b.err }
|
||||||
|
func (b authenticationFailingBody) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
|
||||||
|
want := errors.New("interrupted")
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
|
||||||
|
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type observedAuthenticationBody struct {
|
||||||
|
reader io.Reader
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *observedAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n, err := b.reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *observedAuthenticationBody) Close() error {
|
||||||
|
b.closed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
wantRead int
|
||||||
|
}{
|
||||||
|
{"overflow", http.StatusOK, maxAuthenticationResponseBytes + 1},
|
||||||
|
{"unauthorized", http.StatusUnauthorized, 0},
|
||||||
|
{"other status", http.StatusServiceUnavailable, 0},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
body := &observedAuthenticationBody{reader: strings.NewReader(strings.Repeat("x", 2*maxAuthenticationResponseBytes))}
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
|
||||||
|
})}})
|
||||||
|
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
|
||||||
|
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenRouterKeyEndpoint(t *testing.T) {
|
||||||
|
got, err := OpenRouterKeyEndpoint("https://example.invalid/api/v1/")
|
||||||
|
if err != nil || got != "https://example.invalid/api/v1/key" {
|
||||||
|
t.Fatalf("endpoint = %q, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
|
||||||
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
<-req.Context().Done()
|
||||||
|
return nil, req.Context().Err()
|
||||||
|
})}})
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||||
|
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
|
||||||
|
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,13 +12,7 @@ import (
|
|||||||
var prepareGenerationBenchmarkSink *PreparedGeneration
|
var prepareGenerationBenchmarkSink *PreparedGeneration
|
||||||
|
|
||||||
func BenchmarkPrepareGeneration(b *testing.B) {
|
func BenchmarkPrepareGeneration(b *testing.B) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
BaseURL: "https://api.example.test/v1",
|
|
||||||
Model: "benchmark-model",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("construct benchmark client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prompt := domain.RenderedPrompt{
|
prompt := domain.RenderedPrompt{
|
||||||
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
|
||||||
@@ -31,14 +25,14 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "no_extras",
|
name: "no_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "small_extras",
|
name: "small_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"priority": "low",
|
"priority": "low",
|
||||||
"attempt": 3,
|
"attempt": 3,
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
@@ -49,7 +43,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "1MiB_string_extra",
|
name: "1MiB_string_extra",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"payload": strings.Repeat("x", 1<<20),
|
"payload": strings.Repeat("x", 1<<20),
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
@@ -58,7 +52,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "nested_json_extras",
|
name: "nested_json_extras",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
|
||||||
"options": map[string]any{
|
"options": map[string]any{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"limits": []any{2, 4, 8},
|
"limits": []any{2, 4, 8},
|
||||||
@@ -71,7 +65,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
|
|||||||
name: "structured_output",
|
name: "structured_output",
|
||||||
request: domain.GenerateRequest{
|
request: domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
Target: domain.ExecutionTarget{Model: "benchmark-model"},
|
Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
|
||||||
StructuredOutput: &domain.StructuredOutputSpec{
|
StructuredOutput: &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
|
|||||||
@@ -73,9 +73,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
response := responseFor(request, test.status, "")
|
response := responseFor(request, test.status, "")
|
||||||
response.Body = io.NopCloser(strings.NewReader(test.body))
|
response.Body = io.NopCloser(strings.NewReader(test.body))
|
||||||
@@ -83,9 +81,6 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
return response, nil
|
return response, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
request := ordinaryGenerateRequest()
|
request := ordinaryGenerateRequest()
|
||||||
if test.structured {
|
if test.structured {
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
@@ -136,9 +131,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
|
|||||||
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
|
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
|
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: http.StatusOK,
|
StatusCode: http.StatusOK,
|
||||||
@@ -149,9 +142,6 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
|
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -174,16 +164,11 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
|
|||||||
|
|
||||||
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
|
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
|
||||||
transportErr := errors.New("provider unavailable")
|
transportErr := errors.New("provider unavailable")
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return nil, transportErr
|
return nil, transportErr
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
request := ordinaryGenerateRequest()
|
request := ordinaryGenerateRequest()
|
||||||
request.StructuredOutput = &domain.StructuredOutputSpec{
|
request.StructuredOutput = &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
|
|||||||
@@ -26,8 +26,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
|
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
|
||||||
t.Setenv(envName, tc.initialKey)
|
t.Setenv(envName, tc.initialKey)
|
||||||
var bodies, auth, urls []string
|
var bodies, auth, urls []string
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "default-model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
body, err := io.ReadAll(req.Body)
|
body, err := io.ReadAll(req.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -39,12 +38,9 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req := domain.GenerateRequest{
|
req := domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
|
||||||
Target: domain.ExecutionTarget{APIKeyEnv: envName},
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "default-model", APIKeyEnv: envName},
|
||||||
}
|
}
|
||||||
prepared, err := client.PrepareGeneration(req)
|
prepared, err := client.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -68,8 +64,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
|
|||||||
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
|
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
|
||||||
var contexts []context.Context
|
var contexts []context.Context
|
||||||
var closed int
|
var closed int
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
if err := req.Context().Err(); err != nil {
|
if err := req.Context().Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -81,12 +76,9 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
|
|||||||
}}, nil
|
}}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
Target: domain.ExecutionTarget{TimeoutSeconds: 1},
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model", TimeoutSeconds: 1},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -115,17 +107,14 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -137,17 +126,14 @@ func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -160,17 +146,14 @@ func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
|
|||||||
|
|
||||||
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
|
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
|
||||||
closed := false
|
closed := false
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://example.invalid/v1", Model: "model",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
|
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
prepared, err := client.PrepareGeneration(domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
||||||
|
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -127,9 +127,7 @@ func TestOpenAICompatibleClientNonSuccessBodyOwnership(t *testing.T) {
|
|||||||
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
||||||
BaseURL: "https://provider.example/v1",
|
|
||||||
Model: "m",
|
|
||||||
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: statusCode,
|
StatusCode: statusCode,
|
||||||
@@ -138,8 +136,5 @@ func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int
|
|||||||
}, nil
|
}, nil
|
||||||
})},
|
})},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("construct client: %v", err)
|
|
||||||
}
|
|
||||||
return client
|
return client
|
||||||
}
|
}
|
||||||
|
|||||||
48
internal/llm/response_read_error_test.go
Normal file
48
internal/llm/response_read_error_test.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type responseBytesAndError struct {
|
||||||
|
data []byte
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *responseBytesAndError) Read(p []byte) (int, error) {
|
||||||
|
if len(r.data) == 0 {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
n := copy(p, r.data)
|
||||||
|
r.data = r.data[n:]
|
||||||
|
if len(r.data) == 0 {
|
||||||
|
return n, r.err
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSuccessfulResponseReadFailuresRetainCause(t *testing.T) {
|
||||||
|
cause := errors.New("private reader diagnostic")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, prefix string
|
||||||
|
together bool
|
||||||
|
}{
|
||||||
|
{"partial", `{"choices":[`, false},
|
||||||
|
{"complete then failure", successfulProviderResponse, false},
|
||||||
|
{"bytes and error together", successfulProviderResponse, true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
var reader io.Reader = io.MultiReader(strings.NewReader(test.prefix), failingReader{err: cause})
|
||||||
|
if test.together {
|
||||||
|
reader = &responseBytesAndError{data: []byte(test.prefix), err: cause}
|
||||||
|
}
|
||||||
|
_, err := decodeOpenAIChatResponse(reader)
|
||||||
|
if !errors.Is(err, ErrTransport) || !errors.Is(err, cause) || errors.Is(err, ErrMalformedResponse) || strings.Contains(err.Error(), cause.Error()) {
|
||||||
|
t.Fatalf("read failure = %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,4 +13,5 @@ var ErrProfileNotFound = errors.New("execution profile not found")
|
|||||||
// Repository selects execution profiles from a captured index.
|
// Repository selects execution profiles from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
||||||
|
GetProfileIdentity(ctx context.Context, id string) (string, error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,10 +24,6 @@ type goRenderer struct {
|
|||||||
templates TemplateLookup
|
templates TemplateLookup
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGoRenderer() Renderer {
|
|
||||||
return &goRenderer{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewCompiledRenderer renders definitions using templates captured with the
|
// NewCompiledRenderer renders definitions using templates captured with the
|
||||||
// configuration snapshot. Per-call clones bind request-specific input helpers.
|
// configuration snapshot. Per-call clones bind request-specific input helpers.
|
||||||
func NewCompiledRenderer(templates TemplateLookup) Renderer {
|
func NewCompiledRenderer(templates TemplateLookup) Renderer {
|
||||||
@@ -56,13 +52,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
funcs := template.FuncMap{
|
funcs := template.FuncMap{
|
||||||
"input": resolver.resolve,
|
"input": resolver.resolve,
|
||||||
}
|
}
|
||||||
var compiled CompiledTemplates
|
if r.templates == nil {
|
||||||
if r.templates != nil {
|
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
||||||
var found bool
|
}
|
||||||
compiled, found = r.templates.GetCompiledTemplates(definition.ID)
|
compiled, found := r.templates.GetCompiledTemplates(definition.ID)
|
||||||
if !found || len(compiled.Messages) != len(definition.Templates) {
|
if !found || len(compiled.Messages) != len(definition.Templates) {
|
||||||
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -70,15 +65,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}
|
}
|
||||||
var sessionID string
|
var sessionID string
|
||||||
var err error
|
var err error
|
||||||
if r.templates != nil {
|
if definition.SessionID != "" {
|
||||||
if definition.SessionID != "" {
|
if compiled.Session == nil {
|
||||||
if compiled.Session == nil {
|
return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
|
||||||
return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
|
|
||||||
}
|
|
||||||
sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
|
|
||||||
}
|
}
|
||||||
} else {
|
sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
|
||||||
sessionID, err = renderSessionID(ctx, definition.SessionID, funcs, vars)
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -102,18 +93,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
var tmpl *template.Template
|
if compiled.Messages[i] == nil {
|
||||||
var parseErr error
|
return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure)
|
||||||
if r.templates != nil {
|
}
|
||||||
if compiled.Messages[i] == nil {
|
tmpl, parseErr := compiled.Messages[i].Clone()
|
||||||
return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure)
|
if parseErr == nil {
|
||||||
}
|
tmpl.Funcs(funcs)
|
||||||
tmpl, parseErr = compiled.Messages[i].Clone()
|
|
||||||
if parseErr == nil {
|
|
||||||
tmpl.Funcs(funcs)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
tmpl, parseErr = parseTemplate(fmt.Sprintf("msg_%d", i), tmplMsg.Content, funcs)
|
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -201,20 +186,6 @@ func (r *artifactTextResolver) resolve(name string) (string, error) {
|
|||||||
return text, nil
|
return text, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func renderSessionID(ctx context.Context, raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
tmpl, parseErr := parseTemplate("session_id", raw, funcs)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if parseErr != nil {
|
|
||||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, parseErr)
|
|
||||||
}
|
|
||||||
return executeSessionTemplate(ctx, tmpl, vars)
|
|
||||||
}
|
|
||||||
|
|
||||||
func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) {
|
func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestGoRenderer_Render(t *testing.T) {
|
func TestGoRenderer_Render(t *testing.T) {
|
||||||
renderer := NewGoRenderer()
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
inputs := map[string]*domain.Artifact{
|
inputs := map[string]*domain.Artifact{
|
||||||
@@ -33,7 +32,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -49,11 +48,11 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "user", Content: "From file: {{input \"transcript\"}}", ContentFile: "/tmp/user.tmpl"},
|
{Role: "user", Content: "From file: {{input \"transcript\"}}"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -71,7 +70,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -99,7 +98,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -129,7 +128,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -151,7 +150,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -169,7 +168,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, map[string]string{
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": "agent-session-123",
|
"session_id": "agent-session-123",
|
||||||
})
|
})
|
||||||
@@ -190,7 +189,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -208,7 +207,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -223,7 +222,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
|
||||||
})
|
})
|
||||||
@@ -241,7 +240,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, map[string]string{
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
|
||||||
"tone": "concise",
|
"tone": "concise",
|
||||||
"session_id": "session" + string([]byte{0xff}),
|
"session_id": "session" + string([]byte{0xff}),
|
||||||
})
|
})
|
||||||
@@ -258,7 +257,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -278,7 +277,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -298,7 +297,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -315,7 +314,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, map[string]*domain.Artifact{}, vars)
|
||||||
if !errors.Is(err, ErrMissingRequiredInput) {
|
if !errors.Is(err, ErrMissingRequiredInput) {
|
||||||
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
|
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -329,9 +328,9 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := CompileTemplate("message", def.Templates[0].Content)
|
||||||
if !errors.Is(err, ErrInvalidTemplate) {
|
if err == nil {
|
||||||
t.Fatalf("expected ErrInvalidTemplate, got %v", err)
|
t.Fatal("expected template compilation to fail")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -343,7 +342,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
if !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -359,7 +358,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
{Role: "", Content: "Hello"},
|
{Role: "", Content: "Hello"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
|
||||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
if !errors.Is(err, ErrInvalidMessageRole) {
|
||||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -371,7 +370,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
|
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
res, err := renderer.Render(ctx, def, nil, vars)
|
res, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
@@ -385,7 +384,7 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
def := &domain.PromptDefinition{
|
def := &domain.PromptDefinition{
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
|
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
|
||||||
}
|
}
|
||||||
_, err := renderer.Render(ctx, def, nil, vars)
|
_, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
|
||||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
if !errors.Is(err, ErrInvalidMessageRole) {
|
||||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -396,9 +395,9 @@ func TestGoRenderer_Render(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGoRendererCancellation(t *testing.T) {
|
func TestGoRendererCancellation(t *testing.T) {
|
||||||
t.Run("before session parsing", func(t *testing.T) {
|
t.Run("before rendering", func(t *testing.T) {
|
||||||
definition := &domain.PromptDefinition{
|
definition := &domain.PromptDefinition{
|
||||||
SessionID: "{{ malformed",
|
SessionID: "{{.session}}",
|
||||||
Templates: []domain.PromptMessageTemplate{
|
Templates: []domain.PromptMessageTemplate{
|
||||||
{Role: "user", Content: "not rendered"},
|
{Role: "user", Content: "not rendered"},
|
||||||
},
|
},
|
||||||
@@ -406,18 +405,11 @@ func TestGoRendererCancellation(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
result, err := renderer.Render(ctx, definition, nil, nil)
|
||||||
if result != nil || !errors.Is(err, context.Canceled) {
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
}
|
}
|
||||||
if errors.Is(err, ErrInvalidTemplate) {
|
|
||||||
t.Fatalf("pre-canceled render parsed the malformed session: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil)
|
|
||||||
if result != nil || !errors.Is(err, ErrInvalidTemplate) {
|
|
||||||
t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("during artifact text conversion", func(t *testing.T) {
|
t.Run("during artifact text conversion", func(t *testing.T) {
|
||||||
@@ -447,14 +439,15 @@ func TestGoRendererCancellation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
counter := &checkCountingContext{Context: context.Background()}
|
counter := &checkCountingContext{Context: context.Background()}
|
||||||
if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil {
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
if _, err := renderer.Render(counter, definition, nil, nil); err != nil {
|
||||||
t.Fatalf("count render checkpoints: %v", err)
|
t.Fatalf("count render checkpoints: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The final three checks occur after template execution, after the
|
// The final three checks occur after template execution, after the
|
||||||
// message is assembled, and immediately before publication.
|
// message is assembled, and immediately before publication.
|
||||||
ctx := newCancelOnCheckContext(counter.checks - 2)
|
ctx := newCancelOnCheckContext(counter.checks - 2)
|
||||||
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
|
result, err := renderer.Render(ctx, definition, nil, nil)
|
||||||
if result != nil || !errors.Is(err, context.Canceled) {
|
if result != nil || !errors.Is(err, context.Canceled) {
|
||||||
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
|
||||||
}
|
}
|
||||||
@@ -472,7 +465,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
renderer := capturedTestRenderer(t, definition)
|
||||||
|
first, err := renderer.Render(context.Background(), definition, inputs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("first render: %v", err)
|
t.Fatalf("first render: %v", err)
|
||||||
}
|
}
|
||||||
@@ -488,7 +482,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
|
||||||
t.Fatal("completed render aliases the artifact body")
|
t.Fatal("completed render aliases the artifact body")
|
||||||
}
|
}
|
||||||
second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
|
second, err := renderer.Render(context.Background(), definition, inputs, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("second render: %v", err)
|
t.Fatalf("second render: %v", err)
|
||||||
}
|
}
|
||||||
@@ -498,7 +492,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
nilInputs := map[string]*domain.Artifact{"document": nil}
|
nilInputs := map[string]*domain.Artifact{"document": nil}
|
||||||
result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil)
|
result, err := renderer.Render(context.Background(), definition, nilInputs, nil)
|
||||||
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
|
||||||
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
|
||||||
}
|
}
|
||||||
@@ -559,11 +553,56 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCompiledRendererRequiresCapturedTemplates(t *testing.T) {
|
||||||
|
message, err := CompileTemplate("message", "hello")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
lookup TemplateLookup
|
||||||
|
sessionID string
|
||||||
|
}{
|
||||||
|
{name: "missing lookup"},
|
||||||
|
{name: "missing prompt", lookup: compiledRendererTestLookup{id: "other"}},
|
||||||
|
{name: "missing message", lookup: compiledRendererTestLookup{id: "prompt"}},
|
||||||
|
{name: "nil message", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{nil}}}},
|
||||||
|
{name: "missing session", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{message}}}, sessionID: "session"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
definition := &domain.PromptDefinition{ID: "prompt", SessionID: tc.sessionID, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hello"}}}
|
||||||
|
if _, err := NewCompiledRenderer(tc.lookup).Render(context.Background(), definition, nil, nil); !errors.Is(err, ErrRenderFailure) {
|
||||||
|
t.Fatalf("render error = %v, want render failure", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type compiledRendererTestLookup struct {
|
type compiledRendererTestLookup struct {
|
||||||
id string
|
id string
|
||||||
templates CompiledTemplates
|
templates CompiledTemplates
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func capturedTestRenderer(t testing.TB, definition *domain.PromptDefinition) Renderer {
|
||||||
|
t.Helper()
|
||||||
|
var compiled CompiledTemplates
|
||||||
|
var err error
|
||||||
|
if definition.SessionID != "" {
|
||||||
|
compiled.Session, err = CompileTemplate("session", definition.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, message := range definition.Templates {
|
||||||
|
parsed, err := CompileTemplate("message", message.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
compiled.Messages = append(compiled.Messages, parsed)
|
||||||
|
}
|
||||||
|
return NewCompiledRenderer(compiledRendererTestLookup{id: definition.ID, templates: compiled})
|
||||||
|
}
|
||||||
|
|
||||||
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
|
||||||
if id != lookup.id {
|
if id != lookup.id {
|
||||||
return CompiledTemplates{}, false
|
return CompiledTemplates{}, false
|
||||||
@@ -571,8 +610,7 @@ func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (Compil
|
|||||||
return lookup.templates, true
|
return lookup.templates, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
|
// BenchmarkGoRendererArtifactReferences measures rendering captured templates.
|
||||||
// template-parsing fallback rather than the captured compiled path.
|
|
||||||
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
||||||
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
|
||||||
inputs := map[string]*domain.Artifact{
|
inputs := map[string]*domain.Artifact{
|
||||||
@@ -604,7 +642,7 @@ func BenchmarkGoRendererArtifactReferences(b *testing.B) {
|
|||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
b.Run(tc.name, func(b *testing.B) {
|
b.Run(tc.name, func(b *testing.B) {
|
||||||
renderer := NewGoRenderer()
|
renderer := capturedTestRenderer(b, tc.definition)
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
b.SetBytes(int64(len(body)))
|
b.SetBytes(int64(len(body)))
|
||||||
for range b.N {
|
for range b.N {
|
||||||
|
|||||||
@@ -319,24 +319,27 @@ func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluat
|
|||||||
result := conditionEvaluation{
|
result := conditionEvaluation{
|
||||||
unsupported: left.unsupported || right.unsupported,
|
unsupported: left.unsupported || right.unsupported,
|
||||||
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
|
||||||
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
|
missingVariables: mergeOwnedNames(left.missingVariables, right.missingVariables),
|
||||||
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
|
contentInputs: mergeOwnedNames(left.contentInputs, right.contentInputs),
|
||||||
}
|
|
||||||
for name := range left.missingVariables {
|
|
||||||
result.missingVariables[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range right.missingVariables {
|
|
||||||
result.missingVariables[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range left.contentInputs {
|
|
||||||
result.contentInputs[name] = struct{}{}
|
|
||||||
}
|
|
||||||
for name := range right.contentInputs {
|
|
||||||
result.contentInputs[name] = struct{}{}
|
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Condition results own their sets; callers retain operand values and flags,
|
||||||
|
// not independent snapshots of these maps after a merge.
|
||||||
|
func mergeOwnedNames(left, right map[string]struct{}) map[string]struct{} {
|
||||||
|
if len(left) < len(right) {
|
||||||
|
left, right = right, left
|
||||||
|
}
|
||||||
|
if left == nil {
|
||||||
|
return right
|
||||||
|
}
|
||||||
|
for name := range right {
|
||||||
|
left[name] = struct{}{}
|
||||||
|
}
|
||||||
|
return left
|
||||||
|
}
|
||||||
|
|
||||||
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
|
||||||
if parent == requirementInactive {
|
if parent == requirementInactive {
|
||||||
return requirementInactive
|
return requirementInactive
|
||||||
@@ -448,6 +451,34 @@ func (walker *requirementWalker) walkTemplate(compiled *template.Template) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (walker *requirementWalker) visitBranch(branch *parse.BranchNode, reach requirementReachability, dot dotIdentity, changesDot bool) error {
|
||||||
|
condition := conditionEvaluation{}
|
||||||
|
if reach == requirementInactive {
|
||||||
|
if err := walker.visit(branch.Pipe, requirementInactive, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
condition = walker.evaluateCondition(branch.Pipe, reach, dot)
|
||||||
|
}
|
||||||
|
before := walker.dependencies
|
||||||
|
bodyDot := dot
|
||||||
|
if changesDot {
|
||||||
|
bodyDot = walker.withDotIdentity(branch.Pipe, dot, condition)
|
||||||
|
}
|
||||||
|
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
||||||
|
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
||||||
|
if err := walker.visit(branch.List, thenReach, bodyDot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := walker.visit(branch.ElseList, elseReach, dot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if reach != requirementInactive && !condition.known && walker.dependencies != before {
|
||||||
|
walker.recordConditionLimitation(condition)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
|
||||||
if err := walker.ctx.Err(); err != nil {
|
if err := walker.ctx.Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -468,55 +499,9 @@ func (walker *requirementWalker) visit(node parse.Node, reach requirementReachab
|
|||||||
case *parse.ActionNode:
|
case *parse.ActionNode:
|
||||||
return walker.visit(node.Pipe, reach, dot)
|
return walker.visit(node.Pipe, reach, dot)
|
||||||
case *parse.IfNode:
|
case *parse.IfNode:
|
||||||
if reach == requirementInactive {
|
return walker.visitBranch(&node.BranchNode, reach, dot, false)
|
||||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
|
||||||
}
|
|
||||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
|
||||||
before := walker.dependencies
|
|
||||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
|
||||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
|
||||||
if err := walker.visit(node.List, thenReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !condition.known && walker.dependencies != before {
|
|
||||||
walker.recordConditionLimitation(condition)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case *parse.WithNode:
|
case *parse.WithNode:
|
||||||
if reach == requirementInactive {
|
return walker.visitBranch(&node.BranchNode, reach, dot, true)
|
||||||
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
|
|
||||||
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return walker.visit(node.ElseList, requirementInactive, dot)
|
|
||||||
}
|
|
||||||
condition := walker.evaluateCondition(node.Pipe, reach, dot)
|
|
||||||
before := walker.dependencies
|
|
||||||
thenReach := branchReach(reach, condition.known, condition.truth(), true)
|
|
||||||
elseReach := branchReach(reach, condition.known, condition.truth(), false)
|
|
||||||
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
|
|
||||||
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !condition.known && walker.dependencies != before {
|
|
||||||
walker.recordConditionLimitation(condition)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
case *parse.RangeNode:
|
case *parse.RangeNode:
|
||||||
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
if err := walker.visit(node.Pipe, reach, dot); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -314,7 +314,7 @@ func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
|
|||||||
"active": {Name: "active", Body: []byte("active")},
|
"active": {Name: "active", Body: []byte("active")},
|
||||||
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
"inactive": {Name: "inactive", Body: []byte("inactive")},
|
||||||
}
|
}
|
||||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -406,6 +406,35 @@ func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *tes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConditionEvaluationMergesOwnedEvidence(t *testing.T) {
|
||||||
|
left := conditionEvaluation{missingVariables: map[string]struct{}{"a": {}, "b": {}}, contentInputs: map[string]struct{}{"document": {}}, unsupported: true}
|
||||||
|
right := conditionEvaluation{missingVariables: map[string]struct{}{"b": {}, "c": {}}, inputNameUnknown: true}
|
||||||
|
merged := mergeConditionEvaluations(left, right)
|
||||||
|
if !merged.unsupported || !merged.inputNameUnknown || len(merged.missingVariables) != 3 || len(merged.contentInputs) != 1 {
|
||||||
|
t.Fatalf("merged evidence = %+v", merged)
|
||||||
|
}
|
||||||
|
transferred := mergeConditionEvaluations(conditionEvaluation{}, conditionEvaluation{missingVariables: map[string]struct{}{"only": {}}})
|
||||||
|
if len(transferred.missingVariables) != 1 || transferred.contentInputs != nil {
|
||||||
|
t.Fatalf("transferred evidence = %+v", transferred)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeRequirementsRetainsLeftHeavyMissingConditions(t *testing.T) {
|
||||||
|
compiled := compileRequirementTemplate(t, "nested", `{{if eq (eq (eq .a .b) .c) .d}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
|
||||||
|
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{compiled}}}, nil, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Complete || len(result.Variables) != 4 || len(result.Inputs) != 2 {
|
||||||
|
t.Fatalf("requirements = %+v", result)
|
||||||
|
}
|
||||||
|
for i, name := range []string{"a", "b", "c", "d"} {
|
||||||
|
if result.Variables[i].Name != name {
|
||||||
|
t.Fatalf("variables = %+v", result.Variables)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
|
||||||
description := "declared for a reason"
|
description := "declared for a reason"
|
||||||
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
|
||||||
@@ -497,7 +526,7 @@ func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
|
|||||||
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
|
||||||
}
|
}
|
||||||
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
|
||||||
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
|
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/prompt"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
|
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
|
||||||
@@ -12,5 +13,7 @@ var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
|||||||
|
|
||||||
// Repository selects prompt definitions from a captured index.
|
// Repository selects prompt definitions from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error)
|
GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
|
||||||
|
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
|
||||||
|
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
||||||
}
|
}
|
||||||
|
|||||||
38
internal/usecase/operation_error.go
Normal file
38
internal/usecase/operation_error.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OperationError identifies the operation that owns a failure while retaining
|
||||||
|
// the original cause for inspection.
|
||||||
|
type OperationError struct {
|
||||||
|
Category error
|
||||||
|
Cause error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *OperationError) Error() string { return fmt.Sprintf("%v: %v", e.Category, e.Cause) }
|
||||||
|
func (e *OperationError) Unwrap() error { return e.Cause }
|
||||||
|
func (e *OperationError) Is(target error) bool { return target == e.Category }
|
||||||
|
|
||||||
|
func promptLookupError(cause error) error {
|
||||||
|
category := ErrPromptLoad
|
||||||
|
if errors.Is(cause, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
category = promptdef.ErrPromptDefinitionNotFound
|
||||||
|
cause = fmt.Errorf("%w: %w", ErrPromptLoad, cause)
|
||||||
|
}
|
||||||
|
return &OperationError{Category: category, Cause: cause}
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileLookupError(cause error) error {
|
||||||
|
category := ErrProfileLoad
|
||||||
|
if errors.Is(cause, profile.ErrProfileNotFound) {
|
||||||
|
category = profile.ErrProfileNotFound
|
||||||
|
cause = fmt.Errorf("%w: %w", ErrProfileLoad, cause)
|
||||||
|
}
|
||||||
|
return &OperationError{Category: category, Cause: cause}
|
||||||
|
}
|
||||||
34
internal/usecase/operation_error_test.go
Normal file
34
internal/usecase/operation_error_test.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOperationErrorRetainsOwnerAndCause(t *testing.T) {
|
||||||
|
cause := fmt.Errorf("nested: %w", promptdef.ErrPromptDefinitionNotFound)
|
||||||
|
outer := &OperationError{Category: ErrLLMGenerate, Cause: cause}
|
||||||
|
if !errors.Is(outer, ErrLLMGenerate) || !errors.Is(outer, promptdef.ErrPromptDefinitionNotFound) || errors.Unwrap(outer) != cause {
|
||||||
|
t.Fatalf("operation error = %v", outer)
|
||||||
|
}
|
||||||
|
for _, test := range []struct {
|
||||||
|
err error
|
||||||
|
category error
|
||||||
|
}{
|
||||||
|
{promptLookupError(cause), promptdef.ErrPromptDefinitionNotFound},
|
||||||
|
{profileLookupError(profile.ErrProfileNotFound), profile.ErrProfileNotFound},
|
||||||
|
{promptLookupError(contextFailure), ErrPromptLoad},
|
||||||
|
{profileLookupError(contextFailure), ErrProfileLoad},
|
||||||
|
} {
|
||||||
|
var operation *OperationError
|
||||||
|
if !errors.As(test.err, &operation) || operation.Category != test.category {
|
||||||
|
t.Errorf("operation = %+v", operation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var contextFailure = errors.New("lookup failed")
|
||||||
@@ -172,14 +172,13 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
|
|||||||
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
||||||
collaborators.validator.prepareCalls != 0 || collaborators.validator.directValidateCalls != 0 ||
|
collaborators.validator.prepareCalls != 0 ||
|
||||||
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d validation=%d admissions=%d generation=%d",
|
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d admissions=%d generation=%d",
|
||||||
collaborators.artifacts.calls,
|
collaborators.artifacts.calls,
|
||||||
collaborators.renderer.calls,
|
collaborators.renderer.calls,
|
||||||
collaborators.validator.prepareCalls,
|
collaborators.validator.prepareCalls,
|
||||||
collaborators.validator.directValidateCalls,
|
|
||||||
len(collaborators.admitter.backendIDs),
|
len(collaborators.admitter.backendIDs),
|
||||||
collaborators.llm.calls,
|
collaborators.llm.calls,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
@@ -22,7 +21,7 @@ const (
|
|||||||
|
|
||||||
// PreparedExecution owns one frozen, single-use runner execution.
|
// PreparedExecution owns one frozen, single-use runner execution.
|
||||||
type PreparedExecution struct {
|
type PreparedExecution struct {
|
||||||
owner *Runner
|
owner *runnerIdentity
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
state preparedExecutionState
|
state preparedExecutionState
|
||||||
details *domain.PreparedRun
|
details *domain.PreparedRun
|
||||||
@@ -50,24 +49,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
executionSnapshot := operation.run
|
|
||||||
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
|
|
||||||
|
|
||||||
details, err := clonePreparedRun(executionSnapshot)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &PreparedExecution{
|
return &PreparedExecution{
|
||||||
owner: r,
|
owner: r.identity,
|
||||||
state: preparedExecutionReady,
|
state: preparedExecutionReady,
|
||||||
details: details,
|
details: operation.run,
|
||||||
payload: &preparedExecutionPayload{
|
payload: &preparedExecutionPayload{
|
||||||
prepared: executionSnapshot,
|
prepared: operation.run,
|
||||||
validation: operation.validation,
|
validation: operation.validation,
|
||||||
directKey: state.effectiveModel.APIKey,
|
directKey: state.effectiveModel.APIKey,
|
||||||
transientRetries: state.transientRetries,
|
transientRetries: state.transientRetries,
|
||||||
@@ -76,7 +63,8 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Details returns a fresh credential-redacted copy of the prepared run.
|
// Details returns immutable, credential-redacted metadata borrowed from the
|
||||||
|
// handle. Callers must copy it before exposing it to mutable external code.
|
||||||
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||||
if p == nil {
|
if p == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -85,11 +73,7 @@ func (p *PreparedExecution) Details() *domain.PreparedRun {
|
|||||||
detailsSnapshot := p.details
|
detailsSnapshot := p.details
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
details, err := clonePreparedRun(detailsSnapshot)
|
return detailsSnapshot
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return details
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Discard invalidates an unclaimed execution and drops its private payload.
|
// Discard invalidates an unclaimed execution and drops its private payload.
|
||||||
@@ -170,7 +154,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
||||||
if p == nil || owner == nil || p.owner != owner {
|
if p == nil || owner == nil || owner.identity == nil || p.owner != owner.identity {
|
||||||
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,9 +173,6 @@ func (p *preparedExecutionPayload) clear() {
|
|||||||
if p == nil {
|
if p == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if p.prepared != nil {
|
|
||||||
p.prepared.EffectiveModelParams.APIKey = ""
|
|
||||||
}
|
|
||||||
p.prepared = nil
|
p.prepared = nil
|
||||||
p.validation = nil
|
p.validation = nil
|
||||||
p.directKey = ""
|
p.directKey = ""
|
||||||
@@ -222,43 +203,3 @@ func (p noOpPreparedValidation) Validate(
|
|||||||
func (noOpPreparedValidation) SchemaDocument() any {
|
func (noOpPreparedValidation) SchemaDocument() any {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
|
||||||
if source == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
copied := *source
|
|
||||||
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
copied.EffectiveModelParams.ExtraParams = extraParams
|
|
||||||
|
|
||||||
if source.InputHashes != nil {
|
|
||||||
copied.InputHashes = make(map[string]string, len(source.InputHashes))
|
|
||||||
for name, hash := range source.InputHashes {
|
|
||||||
copied.InputHashes[name] = hash
|
|
||||||
}
|
|
||||||
}
|
|
||||||
copied.Messages = domain.CloneRenderedMessages(source.Messages)
|
|
||||||
|
|
||||||
if source.StructuredOutput != nil {
|
|
||||||
structuredOutput := *source.StructuredOutput
|
|
||||||
copied.StructuredOutput = &structuredOutput
|
|
||||||
if source.StructuredOutput.JSONSchema != nil {
|
|
||||||
jsonSchema := *source.StructuredOutput.JSONSchema
|
|
||||||
copied.StructuredOutput.JSONSchema = &jsonSchema
|
|
||||||
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
|
|
||||||
if copyErr != nil {
|
|
||||||
return nil, copyErr
|
|
||||||
}
|
|
||||||
copied.StructuredOutput.JSONSchema.Schema = schema
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &copied, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneJSONValue(source any) (any, error) {
|
|
||||||
return jsonvalue.Copy(source)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
@@ -47,25 +48,15 @@ func (p *recordingPreparedValidation) SchemaDocument() any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type recordingValidationPreparer struct {
|
type recordingValidationPreparer struct {
|
||||||
plan *recordingPreparedValidation
|
plan *recordingPreparedValidation
|
||||||
prepareErr error
|
prepareErr error
|
||||||
prepareCalls int
|
prepareCalls int
|
||||||
directValidateCalls int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type validationOnly struct{}
|
type nilPlanPreparer struct{}
|
||||||
|
|
||||||
func (validationOnly) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
|
func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||||
return domain.ValidationResult{}, nil
|
return nil, nil
|
||||||
}
|
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) Validate(
|
|
||||||
context.Context,
|
|
||||||
*domain.Artifact,
|
|
||||||
domain.OutputContract,
|
|
||||||
) (domain.ValidationResult, error) {
|
|
||||||
v.directValidateCalls++
|
|
||||||
return domain.ValidationResult{}, errors.New("live validation must not be used")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *recordingValidationPreparer) PrepareValidation(
|
func (v *recordingValidationPreparer) PrepareValidation(
|
||||||
@@ -129,12 +120,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
|||||||
}
|
}
|
||||||
defer prepared.Discard()
|
defer prepared.Discard()
|
||||||
|
|
||||||
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
|
if validator.prepareCalls != 1 {
|
||||||
t.Fatalf(
|
t.Fatalf("validation preparations = %d, want 1", validator.prepareCalls)
|
||||||
"validation calls=(prepare=%d direct=%d), want (1, 0)",
|
|
||||||
validator.prepareCalls,
|
|
||||||
validator.directValidateCalls,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
if reader.calls != 1 || renderer.calls != 1 {
|
if reader.calls != 1 || renderer.calls != 1 {
|
||||||
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
||||||
@@ -156,90 +143,37 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
|||||||
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
||||||
}
|
}
|
||||||
|
|
||||||
first.Messages[0].Content = "caller mutation"
|
|
||||||
first.InputHashes["input"] = "caller mutation"
|
|
||||||
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
|
|
||||||
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
|
|
||||||
renderer.rendered.Messages[0].Content = "source mutation"
|
renderer.rendered.Messages[0].Content = "source mutation"
|
||||||
|
|
||||||
second := prepared.Details()
|
second := prepared.Details()
|
||||||
if second.Messages[0].Content != "original message" ||
|
if second != first || second.Messages[0].Content != "original message" ||
|
||||||
second.InputHashes["input"] == "caller mutation" ||
|
|
||||||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||||
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
|
t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) {
|
func TestRunnerPreparationDetachesDeepPreparedSchema(t *testing.T) {
|
||||||
operations := []struct {
|
var nested any = "original"
|
||||||
name string
|
for range 150 {
|
||||||
run func(*Runner, domain.RunRequest) error
|
nested = map[string]any{"child": nested}
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "Prepare",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.Prepare(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Run",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.Run(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "PrepareExecution",
|
|
||||||
run: func(runner *Runner, request domain.RunRequest) error {
|
|
||||||
_, err := runner.PrepareExecution(context.Background(), request)
|
|
||||||
return err
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
schema := map[string]any{"annotation": nested}
|
||||||
for _, operation := range operations {
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
t.Run(operation.name, func(t *testing.T) {
|
def.Validation.SchemaPath = "schema.json"
|
||||||
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{schemaDocument: schema}}
|
||||||
def.Validation.SchemaPath = "schema.json"
|
runner := NewRunner(&fakePromptRepo{def: def},
|
||||||
llmClient := &fakeLLM{forbid: true}
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
validator := &recordingValidationPreparer{
|
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, validator, nil)
|
||||||
plan: &recordingPreparedValidation{schemaDocument: excessivelyDeepPreparedJSONValue()},
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
}
|
if err != nil {
|
||||||
runner := NewRunner(
|
t.Fatalf("prepare execution: %v", err)
|
||||||
&fakePromptRepo{def: def},
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
|
||||||
|
|
||||||
defaultArtifactReader(),
|
|
||||||
defaultRenderer(),
|
|
||||||
llmClient,
|
|
||||||
validator,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
err := operation.run(runner, domain.RunRequest{
|
|
||||||
PromptID: "p",
|
|
||||||
ProfileID: "exec",
|
|
||||||
Inputs: singleInputRef(),
|
|
||||||
})
|
|
||||||
if !errors.Is(err, ErrValidation) {
|
|
||||||
t.Fatalf("expected ErrValidation, got %v", err)
|
|
||||||
}
|
|
||||||
if llmClient.calls != 0 {
|
|
||||||
t.Fatalf("invalid prepared schema reached generation: %d calls", llmClient.calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
schema["annotation"].(map[string]any)["child"] = "source mutation"
|
||||||
|
first := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||||
func excessivelyDeepPreparedJSONValue() any {
|
if _, ok := first["annotation"].(map[string]any)["child"].(map[string]any); !ok {
|
||||||
const clearlyUnsafeContainerDepth = 1_000
|
t.Fatal("prepared schema aliases the validator document")
|
||||||
var value any = true
|
|
||||||
for level := 0; level < clearlyUnsafeContainerDepth; level++ {
|
|
||||||
value = map[string]any{"child": value}
|
|
||||||
}
|
}
|
||||||
return value
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
||||||
@@ -421,8 +355,8 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
|||||||
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
||||||
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
||||||
}
|
}
|
||||||
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
if repairer.calls != 1 {
|
||||||
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
t.Fatalf("repair calls = %d, want 1", repairer.calls)
|
||||||
}
|
}
|
||||||
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
|
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
|
||||||
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
|
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
|
||||||
@@ -498,7 +432,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
repairer: &fakeRepairer{err: repairInvalidRequest},
|
repairer: &fakeRepairer{err: repairInvalidRequest},
|
||||||
wantError: ErrInvalidRequest,
|
wantError: ErrLLMGenerate,
|
||||||
wantSource: repairInvalidRequest,
|
wantSource: repairInvalidRequest,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -600,6 +534,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
owner := newRunner()
|
owner := newRunner()
|
||||||
|
if other := newRunner(); owner.identity == nil || owner.identity == other.identity {
|
||||||
|
t.Fatal("constructed runners must have independent identity tokens")
|
||||||
|
}
|
||||||
prepared, err := owner.PrepareExecution(context.Background(), request)
|
prepared, err := owner.PrepareExecution(context.Background(), request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare execution: %v", err)
|
t.Fatalf("prepare execution: %v", err)
|
||||||
@@ -616,6 +553,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
if prepared.Details() == nil {
|
if prepared.Details() == nil {
|
||||||
t.Fatal("details unavailable after execution")
|
t.Fatal("details unavailable after execution")
|
||||||
}
|
}
|
||||||
|
if prepared.owner != owner.identity || prepared.payload != nil {
|
||||||
|
t.Fatal("claimed handle retained payload or lost owner identity")
|
||||||
|
}
|
||||||
|
|
||||||
discarded, err := owner.PrepareExecution(context.Background(), request)
|
discarded, err := owner.PrepareExecution(context.Background(), request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -623,6 +563,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
discarded.Discard()
|
discarded.Discard()
|
||||||
discarded.Discard()
|
discarded.Discard()
|
||||||
|
if discarded.owner != owner.identity || discarded.payload != nil {
|
||||||
|
t.Fatal("discarded handle retained payload or lost owner identity")
|
||||||
|
}
|
||||||
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
||||||
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
||||||
}
|
}
|
||||||
@@ -687,7 +630,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
wantLabel := "profile source"
|
wantLabel := "profile source"
|
||||||
if test.requestOverride {
|
if test.requestOverride {
|
||||||
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras}
|
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: jsonvalue.CloneTrusted(requestExtras).(map[string]any)}
|
||||||
wantLabel = "request source"
|
wantLabel = "request source"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -711,11 +654,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
|
||||||
t.Fatalf("prepared details retained a source alias: %#v", details)
|
t.Fatalf("prepared details retained a source alias: %#v", details)
|
||||||
}
|
}
|
||||||
details.Messages[0].Content = "mutated details"
|
|
||||||
if again := prepared.Details(); again == nil || len(again.Messages) != 1 || again.Messages[0].Content != "captured rendering" {
|
|
||||||
t.Fatalf("details mutation escaped into the retained snapshot: %+v", again)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
|
||||||
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
|
||||||
@@ -737,22 +675,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
|
|
||||||
profile := defaultExecutionProfile()
|
|
||||||
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
|
|
||||||
runner := NewRunner(
|
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
|
||||||
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
|
|
||||||
)
|
|
||||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
||||||
PromptID: "p", ProfileID: "exec",
|
|
||||||
})
|
|
||||||
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
|
|
||||||
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||||
@@ -782,7 +704,7 @@ func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
func TestRunnerPrepareExecutionRejectsNilValidationPlan(t *testing.T) {
|
||||||
reader := defaultArtifactReader()
|
reader := defaultArtifactReader()
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
@@ -791,7 +713,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
|||||||
reader,
|
reader,
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
validationOnly{},
|
nilPlanPreparer{},
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type resolvedProfileSelection struct {
|
type resolvedProfileSelection struct {
|
||||||
@@ -15,10 +14,6 @@ type resolvedProfileSelection struct {
|
|||||||
profile *domain.ExecutionProfile
|
profile *domain.ExecutionProfile
|
||||||
}
|
}
|
||||||
|
|
||||||
type profileIdentityRepository interface {
|
|
||||||
GetProfileIdentity(context.Context, string) (string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
func (r *Runner) profileLookupID(profileID string) (string, error) {
|
||||||
if strings.TrimSpace(profileID) == "" {
|
if strings.TrimSpace(profileID) == "" {
|
||||||
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
|
||||||
@@ -39,7 +34,7 @@ func (r *Runner) resolveProfileSelection(
|
|||||||
}
|
}
|
||||||
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, profileLookupError(err)
|
||||||
}
|
}
|
||||||
if selectedProfile == nil {
|
if selectedProfile == nil {
|
||||||
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
|
||||||
@@ -61,18 +56,11 @@ func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if indexed, ok := r.profiles.(profileIdentityRepository); ok {
|
identity, err := r.profiles.GetProfileIdentity(ctx, lookupID)
|
||||||
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
return identity, nil
|
|
||||||
}
|
|
||||||
selection, err := r.resolveProfileSelection(ctx, lookupID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", profileLookupError(err)
|
||||||
}
|
}
|
||||||
return selection.id, nil
|
return identity, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
|
||||||
@@ -108,7 +96,7 @@ func (r *Runner) InspectProfile(
|
|||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
|
return nil, &OperationError{Category: ErrProfileLoad, Cause: ctx.Err()}
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,12 +106,6 @@ func (r *Runner) InspectProfile(
|
|||||||
}
|
}
|
||||||
target, _ := resolveExecutionTarget(selection.profile, nil)
|
target, _ := resolveExecutionTarget(selection.profile, nil)
|
||||||
target.APIKey = ""
|
target.APIKey = ""
|
||||||
params, err := jsonvalue.CopyMap(target.ExtraParams)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: profile extra parameters: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
target.ExtraParams = params
|
|
||||||
|
|
||||||
return &domain.ProfileInspection{
|
return &domain.ProfileInspection{
|
||||||
ProfileID: selection.id,
|
ProfileID: selection.id,
|
||||||
EffectiveModelParams: target,
|
EffectiveModelParams: target,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/profile"
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,7 +24,21 @@ func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (
|
|||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
return nil, r.err
|
return nil, r.err
|
||||||
}
|
}
|
||||||
return r.profile, nil
|
if r.profile == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
profile := *r.profile
|
||||||
|
if profile.ResolvedTarget != nil {
|
||||||
|
target := *profile.ResolvedTarget
|
||||||
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
|
profile.ResolvedTarget = &target
|
||||||
|
}
|
||||||
|
return &profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *inspectionProfileRepository) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
_, err := r.GetProfile(ctx, id)
|
||||||
|
return id, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {
|
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {
|
||||||
|
|||||||
@@ -13,10 +13,6 @@ type resolvedPromptDefinition struct {
|
|||||||
hash string
|
hash string
|
||||||
}
|
}
|
||||||
|
|
||||||
type promptInspectionRepository interface {
|
|
||||||
GetPromptInspection(context.Context, string) (*domain.PromptInspection, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
|
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
|
||||||
if strings.TrimSpace(promptID) == "" {
|
if strings.TrimSpace(promptID) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
@@ -25,29 +21,14 @@ func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (
|
|||||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
|
|
||||||
var definition *domain.PromptDefinition
|
definition, hash, err := r.promptDefs.GetPromptSnapshot(ctx, promptID)
|
||||||
var hash string
|
|
||||||
var err error
|
|
||||||
if indexed, ok := r.promptDefs.(interface {
|
|
||||||
GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error)
|
|
||||||
}); ok {
|
|
||||||
definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID)
|
|
||||||
} else {
|
|
||||||
definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID)
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, promptLookupError(err)
|
||||||
}
|
}
|
||||||
if definition == nil {
|
if definition == nil {
|
||||||
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
|
|
||||||
if hash == "" {
|
|
||||||
hash, err = hashPromptDefinition(definition)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,29 +39,14 @@ func (r *Runner) resolvePromptInspection(ctx context.Context, promptID string) (
|
|||||||
if r == nil || r.promptDefs == nil {
|
if r == nil || r.promptDefs == nil {
|
||||||
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
|
inspection, err := r.promptDefs.GetPromptInspection(ctx, promptID)
|
||||||
inspection, err := indexed.GetPromptInspection(ctx, promptID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
|
||||||
}
|
|
||||||
if inspection == nil {
|
|
||||||
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
|
||||||
}
|
|
||||||
return inspection, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
selection, err := r.resolvePromptDefinition(ctx, promptID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, promptLookupError(err)
|
||||||
}
|
}
|
||||||
definition := selection.definition
|
if inspection == nil {
|
||||||
return &domain.PromptInspection{
|
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
|
||||||
PromptID: definition.ID,
|
}
|
||||||
PromptHash: selection.hash,
|
return inspection, nil
|
||||||
DefaultProfileID: definition.DefaultProfile,
|
|
||||||
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
|
|
||||||
OutputContract: definition.Validation,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// InspectPrompt resolves one explicit prompt without execution work.
|
// InspectPrompt resolves one explicit prompt without execution work.
|
||||||
@@ -90,7 +56,7 @@ func (r *Runner) InspectPrompt(ctx context.Context, promptID string) (*domain.Pr
|
|||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: ctx.Err()}
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/prompt"
|
||||||
"go.maximumdirect.net/promptkit/internal/promptdef"
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,20 +24,33 @@ type promptMetadataRepository struct {
|
|||||||
lookupID string
|
lookupID string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *inspectionPromptRepository) GetPromptDefinition(
|
func (r *inspectionPromptRepository) GetPromptSnapshot(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
) (*domain.PromptDefinition, error) {
|
) (*domain.PromptDefinition, string, error) {
|
||||||
r.calls++
|
r.calls++
|
||||||
r.id = id
|
r.id = id
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
return nil, r.err
|
return nil, "", r.err
|
||||||
}
|
}
|
||||||
return r.definition, nil
|
return r.definition, "captured-hash", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string) (*domain.PromptDefinition, error) {
|
func (r *inspectionPromptRepository) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
return nil, errors.New("full prompt definition should not be requested")
|
definition, hash, err := r.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*inspectionPromptRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *promptMetadataRepository) GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error) {
|
||||||
|
return nil, "", errors.New("full prompt definition should not be requested")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*promptMetadataRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *promptMetadataRepository) GetPromptInspection(
|
func (r *promptMetadataRepository) GetPromptInspection(
|
||||||
@@ -71,15 +85,11 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect prompt: %v", err)
|
t.Fatalf("inspect prompt: %v", err)
|
||||||
}
|
}
|
||||||
wantHash, err := hashPromptDefinition(definition)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("hash prompt definition: %v", err)
|
|
||||||
}
|
|
||||||
if repository.calls != 1 || repository.id != " prompt-id " {
|
if repository.calls != 1 || repository.id != " prompt-id " {
|
||||||
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
|
||||||
}
|
}
|
||||||
if inspection.PromptID != definition.ID ||
|
if inspection.PromptID != definition.ID ||
|
||||||
inspection.PromptHash != wantHash ||
|
inspection.PromptHash != "captured-hash" ||
|
||||||
inspection.DefaultProfileID != definition.DefaultProfile ||
|
inspection.DefaultProfileID != definition.DefaultProfile ||
|
||||||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
|
||||||
inspection.OutputContract != definition.Validation {
|
inspection.OutputContract != definition.Validation {
|
||||||
@@ -96,7 +106,7 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
|
func TestRunnerInspectPromptUsesOwnedCapturedMetadata(t *testing.T) {
|
||||||
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
|
||||||
PromptID: "captured.prompt", PromptHash: "cached-prompt-hash",
|
PromptID: "captured.prompt", PromptHash: "cached-prompt-hash",
|
||||||
DefaultProfileID: "partial-profile",
|
DefaultProfileID: "partial-profile",
|
||||||
@@ -119,7 +129,7 @@ func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
|
func TestRunnerInspectPromptUsesCapturedHashAndMetadata(t *testing.T) {
|
||||||
repository := &requirementInspectionPromptRepository{
|
repository := &requirementInspectionPromptRepository{
|
||||||
definition: &domain.PromptDefinition{
|
definition: &domain.PromptDefinition{
|
||||||
ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}},
|
ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}},
|
||||||
@@ -128,7 +138,7 @@ func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *te
|
|||||||
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
|
||||||
if err != nil || inspection.PromptID != "captured.prompt" ||
|
if err != nil || inspection.PromptID != "captured.prompt" ||
|
||||||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
|
||||||
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
|
t.Fatalf("captured inspection=(%+v, %v)", inspection, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
48
internal/usecase/repair_prefix.go
Normal file
48
internal/usecase/repair_prefix.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// boundedDiagnosticJSON returns a JSON string prefix and whether it represents
|
||||||
|
// the complete normalized input. It retains at most the useful prefix.
|
||||||
|
func boundedDiagnosticJSON(value string, maxBytes int) ([]byte, bool) {
|
||||||
|
if maxBytes < 2 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
limit := maxBytes - 2
|
||||||
|
var prefix strings.Builder
|
||||||
|
used := 0
|
||||||
|
more := false
|
||||||
|
for i := 0; i < len(value); {
|
||||||
|
r, size := utf8.DecodeRuneInString(value[i:])
|
||||||
|
piece := value[i : i+size]
|
||||||
|
if r == utf8.RuneError && size == 1 {
|
||||||
|
piece = "\uFFFD"
|
||||||
|
i++
|
||||||
|
for i < len(value) {
|
||||||
|
r, size = utf8.DecodeRuneInString(value[i:])
|
||||||
|
if r != utf8.RuneError || size != 1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
i += size
|
||||||
|
}
|
||||||
|
if used+len(piece) > limit {
|
||||||
|
more = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
prefix.WriteString(piece)
|
||||||
|
used += len(piece)
|
||||||
|
}
|
||||||
|
normalized := prefix.String()
|
||||||
|
encoded, _ := json.Marshal(normalized)
|
||||||
|
if len(encoded) > maxBytes {
|
||||||
|
return truncateDiagnosticJSONValue(normalized, maxBytes), false
|
||||||
|
}
|
||||||
|
return encoded, !more
|
||||||
|
}
|
||||||
33
internal/usecase/repair_prefix_test.go
Normal file
33
internal/usecase/repair_prefix_test.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBoundedDiagnosticJSON(t *testing.T) {
|
||||||
|
for _, input := range []string{"", "plain", "<&>\n\"", "雪☃", "a\xff\xfe\xffb", "\xff\xfe" + strings.Repeat("z", 100)} {
|
||||||
|
for budget := 0; budget <= 30; budget++ {
|
||||||
|
encoded, complete := boundedDiagnosticJSON(input, budget)
|
||||||
|
if budget < 2 {
|
||||||
|
if encoded != nil || complete {
|
||||||
|
t.Fatalf("budget %d: %q %v", budget, encoded, complete)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(encoded) > budget || !utf8.Valid(encoded) {
|
||||||
|
t.Fatalf("budget %d: %q", budget, encoded)
|
||||||
|
}
|
||||||
|
var decoded string
|
||||||
|
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
normalized := strings.ToValidUTF8(input, "\uFFFD")
|
||||||
|
if !strings.HasPrefix(normalized, decoded) || complete != (decoded == normalized) {
|
||||||
|
t.Fatalf("input %q budget %d: %q complete=%v normalized=%q", input, budget, decoded, complete, normalized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -117,27 +117,38 @@ func repairGuidance(mode domain.ValidationMode) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func formatRepairDiagnostics(errors []string) string {
|
func formatRepairDiagnostics(errors []string) string {
|
||||||
diagnostics := make([]string, len(errors))
|
complete := []byte{'['}
|
||||||
for index, diagnostic := range errors {
|
fits := true
|
||||||
diagnostics[index] = strings.ToValidUTF8(diagnostic, "\uFFFD")
|
for _, diagnostic := range errors {
|
||||||
|
separator := 0
|
||||||
|
if len(complete) > 1 {
|
||||||
|
separator = 1
|
||||||
|
}
|
||||||
|
entry, whole := boundedDiagnosticJSON(diagnostic, maxRepairDiagnosticBytes-len(complete)-separator-1)
|
||||||
|
if !whole {
|
||||||
|
fits = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if separator != 0 {
|
||||||
|
complete = append(complete, ',')
|
||||||
|
}
|
||||||
|
complete = append(complete, entry...)
|
||||||
}
|
}
|
||||||
|
if fits {
|
||||||
complete, _ := json.Marshal(diagnostics)
|
return string(append(complete, ']'))
|
||||||
if len(complete) <= maxRepairDiagnosticBytes {
|
|
||||||
return string(complete)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
||||||
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
|
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
|
||||||
encoded = append(encoded, '[')
|
encoded = append(encoded, '[')
|
||||||
for _, diagnostic := range diagnostics {
|
for _, diagnostic := range errors {
|
||||||
entry, _ := json.Marshal(diagnostic)
|
|
||||||
separator := 0
|
separator := 0
|
||||||
if len(encoded) > 1 {
|
if len(encoded) > 1 {
|
||||||
separator = 1
|
separator = 1
|
||||||
}
|
}
|
||||||
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
|
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
|
||||||
if len(entry) <= available {
|
entry, whole := boundedDiagnosticJSON(diagnostic, available)
|
||||||
|
if whole {
|
||||||
if separator != 0 {
|
if separator != 0 {
|
||||||
encoded = append(encoded, ',')
|
encoded = append(encoded, ',')
|
||||||
}
|
}
|
||||||
@@ -151,7 +162,7 @@ func formatRepairDiagnostics(errors []string) string {
|
|||||||
if separator != 0 {
|
if separator != 0 {
|
||||||
encoded = append(encoded, ',')
|
encoded = append(encoded, ',')
|
||||||
}
|
}
|
||||||
encoded = append(encoded, truncateDiagnosticJSONValue(diagnostic, available)...)
|
encoded = append(encoded, entry...)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if len(encoded) > 1 {
|
if len(encoded) > 1 {
|
||||||
|
|||||||
@@ -298,6 +298,23 @@ func TestFormatRepairDiagnosticsBoundsAndPreservesData(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFormatRepairDiagnosticsEmptyAndEscapedOversize(t *testing.T) {
|
||||||
|
for _, values := range [][]string{nil, {}} {
|
||||||
|
if got := formatRepairDiagnostics(values); got != "[]" {
|
||||||
|
t.Fatalf("empty diagnostics = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
value := strings.Repeat("<&>\x00\xff\xfe", maxRepairDiagnosticBytes)
|
||||||
|
encoded := formatRepairDiagnostics([]string{value})
|
||||||
|
var got []string
|
||||||
|
if err := json.Unmarshal([]byte(encoded), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(encoded) > maxRepairDiagnosticBytes || len(got) != 2 || got[1] != omittedRepairDiagnostics || !strings.HasPrefix(strings.ToValidUTF8(value, "\uFFFD"), got[0]) {
|
||||||
|
t.Fatalf("oversize diagnostics length=%d, values=%d", len(encoded), len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func manyRepairDiagnostics() []string {
|
func manyRepairDiagnostics() []string {
|
||||||
diagnostics := make([]string, 1_000)
|
diagnostics := make([]string, 1_000)
|
||||||
for index := range diagnostics {
|
for index := range diagnostics {
|
||||||
|
|||||||
@@ -18,10 +18,6 @@ type RequirementInspectionRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
}
|
}
|
||||||
|
|
||||||
type requirementSnapshotRepository interface {
|
|
||||||
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
|
||||||
}
|
|
||||||
|
|
||||||
// InspectRequirements discovers prompt inputs and caller variables without
|
// InspectRequirements discovers prompt inputs and caller variables without
|
||||||
// loading artifacts or performing execution preparation.
|
// loading artifacts or performing execution preparation.
|
||||||
func (r *Runner) InspectRequirements(
|
func (r *Runner) InspectRequirements(
|
||||||
@@ -36,7 +32,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
selection, err := r.resolvePromptInspection(ctx, req.PromptID)
|
selection, err := r.resolvePromptInspection(ctx, req.PromptID)
|
||||||
@@ -44,7 +40,7 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, &OperationError{Category: ErrProfileLoad, Cause: err}
|
||||||
}
|
}
|
||||||
selectedProfileID := req.ProfileID
|
selectedProfileID := req.ProfileID
|
||||||
if strings.TrimSpace(selectedProfileID) == "" {
|
if strings.TrimSpace(selectedProfileID) == "" {
|
||||||
@@ -58,21 +54,17 @@ func (r *Runner) InspectRequirements(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
repository, ok := r.promptDefs.(requirementSnapshotRepository)
|
snapshot, ok := r.promptDefs.GetRequirementSnapshot(selection.PromptID)
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
|
|
||||||
}
|
|
||||||
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID)
|
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
|
||||||
}
|
}
|
||||||
definition := &domain.PromptDefinition{ID: selection.PromptID}
|
definition := &domain.PromptDefinition{ID: selection.PromptID}
|
||||||
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
|
return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
|
||||||
}
|
}
|
||||||
inspection.ProfileID = profileIdentity
|
inspection.ProfileID = profileIdentity
|
||||||
return &inspection, nil
|
return &inspection, nil
|
||||||
|
|||||||
@@ -45,13 +45,6 @@ func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
|
|||||||
return repository.identity, repository.identityErr
|
return repository.identity, repository.identityErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
|
|
||||||
context.Context,
|
|
||||||
string,
|
|
||||||
) (*domain.PromptDefinition, error) {
|
|
||||||
return repository.definition, repository.lookupErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
id string,
|
id string,
|
||||||
@@ -71,6 +64,11 @@ func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
|
|||||||
return repository.definition, "captured-hash", repository.lookupErr
|
return repository.definition, "captured-hash", repository.lookupErr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (repository *requirementInspectionPromptRepository) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
|
definition, hash, err := repository.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
|
||||||
id string,
|
id string,
|
||||||
) (prompt.RequirementSnapshot, bool) {
|
) (prompt.RequirementSnapshot, bool) {
|
||||||
@@ -163,7 +161,7 @@ func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) {
|
func TestRunnerInspectRequirementsUsesProfileIdentity(t *testing.T) {
|
||||||
message, err := prompt.CompileTemplate("message", `{{.name}}`)
|
message, err := prompt.CompileTemplate("message", `{{.name}}`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -177,7 +175,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
|
|||||||
}
|
}
|
||||||
requestedID := " literal-profile "
|
requestedID := " literal-profile "
|
||||||
|
|
||||||
t.Run("optional capability", func(t *testing.T) {
|
t.Run("identity lookup", func(t *testing.T) {
|
||||||
profiles := &inspectionProfileIdentityRepository{
|
profiles := &inspectionProfileIdentityRepository{
|
||||||
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
||||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
||||||
@@ -196,21 +194,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("full-profile fallback", func(t *testing.T) {
|
t.Run("identity failure", func(t *testing.T) {
|
||||||
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
|
|
||||||
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
|
|
||||||
}}
|
|
||||||
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
|
|
||||||
context.Background(), RequirementInspectionRequest{
|
|
||||||
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil || inspection.ProfileID != requestedID || profiles.calls != 1 || profiles.id != requestedID {
|
|
||||||
t.Fatalf("fallback inspection=(%+v, %v), full lookup=(%d,%q)", inspection, err, profiles.calls, profiles.id)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("capability failure", func(t *testing.T) {
|
|
||||||
lookupErr := errors.New("profile identity lookup failed")
|
lookupErr := errors.New("profile identity lookup failed")
|
||||||
profiles := &inspectionProfileIdentityRepository{
|
profiles := &inspectionProfileIdentityRepository{
|
||||||
inspectionProfileRepository: &inspectionProfileRepository{},
|
inspectionProfileRepository: &inspectionProfileRepository{},
|
||||||
@@ -329,14 +313,6 @@ func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *test
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("repository lacks snapshot capability", func(t *testing.T) {
|
|
||||||
_, err := (&Runner{promptDefs: &inspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
|
||||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}},
|
|
||||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
|
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
|
||||||
t.Fatalf("missing capability error=%v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("nil runner error=%v", err)
|
t.Fatalf("nil runner error=%v", err)
|
||||||
}
|
}
|
||||||
@@ -348,11 +324,6 @@ func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *test
|
|||||||
); !errors.Is(err, ErrProfileLoad) {
|
); !errors.Is(err, ErrProfileLoad) {
|
||||||
t.Fatalf("missing profile error=%v", err)
|
t.Fatalf("missing profile error=%v", err)
|
||||||
}
|
}
|
||||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
|
|
||||||
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}},
|
|
||||||
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrProfileLoad) {
|
|
||||||
t.Fatalf("unresolved profile error=%v", err)
|
|
||||||
}
|
|
||||||
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
|
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
|
||||||
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
|
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
|
||||||
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
|
||||||
|
|||||||
@@ -58,6 +58,19 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvedTargetPreservesProviderMapSelection(t *testing.T) {
|
||||||
|
profile := &domain.ExecutionProfile{ResolvedTarget: &domain.ExecutionTarget{ExtraParams: map[string]any{"configured": true}}}
|
||||||
|
selected, _ := resolveExecutionTarget(profile, &domain.ExecutionTargetOverride{ExtraParams: map[string]any{}})
|
||||||
|
if selected.ExtraParams["configured"] != true {
|
||||||
|
t.Fatalf("empty override did not inherit provider settings: %#v", selected.ExtraParams)
|
||||||
|
}
|
||||||
|
profile.ResolvedTarget.ExtraParams = map[string]any{}
|
||||||
|
selected, _ = resolveExecutionTarget(profile, nil)
|
||||||
|
if selected.ExtraParams != nil {
|
||||||
|
t.Fatalf("empty provider settings changed projection: %#v", selected.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvedTargetCredentialAvailability(t *testing.T) {
|
func TestResolvedTargetCredentialAvailability(t *testing.T) {
|
||||||
profile := &domain.ExecutionProfile{ID: "direct", ResolvedTarget: &domain.ExecutionTarget{
|
profile := &domain.ExecutionProfile{ID: "direct", ResolvedTarget: &domain.ExecutionTarget{
|
||||||
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
@@ -42,17 +43,22 @@ var (
|
|||||||
|
|
||||||
// Runner executes the Promptkit core use case.
|
// Runner executes the Promptkit core use case.
|
||||||
type Runner struct {
|
type Runner struct {
|
||||||
|
identity *runnerIdentity
|
||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
artifacts artifact.Reader
|
artifacts artifact.Reader
|
||||||
renderer prompt.Renderer
|
renderer prompt.Renderer
|
||||||
llm llm.Client
|
llm llm.Client
|
||||||
validator validate.Validator
|
validator validate.ValidationPreparer
|
||||||
repairer OutputRepairer
|
repairer OutputRepairer
|
||||||
admitter RunAdmitter
|
admitter RunAdmitter
|
||||||
defaultTransientRetries uint64
|
defaultTransientRetries uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A non-zero-sized token gives each constructed runner a distinct identity
|
||||||
|
// without retaining its collaborators in prepared handles.
|
||||||
|
type runnerIdentity struct{ marker byte }
|
||||||
|
|
||||||
// RunAdmitter reserves capacity for one resolved backend run.
|
// RunAdmitter reserves capacity for one resolved backend run.
|
||||||
type RunAdmitter interface {
|
type RunAdmitter interface {
|
||||||
Admit(context.Context, string) (func(), error)
|
Admit(context.Context, string) (func(), error)
|
||||||
@@ -81,7 +87,7 @@ func NewRunner(
|
|||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
validator validate.Validator,
|
validator validate.ValidationPreparer,
|
||||||
admitter RunAdmitter,
|
admitter RunAdmitter,
|
||||||
) *Runner {
|
) *Runner {
|
||||||
return NewRunnerWithRepairer(
|
return NewRunnerWithRepairer(
|
||||||
@@ -102,7 +108,7 @@ func NewRunnerWithRepairer(
|
|||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
validator validate.Validator,
|
validator validate.ValidationPreparer,
|
||||||
repairer OutputRepairer,
|
repairer OutputRepairer,
|
||||||
admitter RunAdmitter,
|
admitter RunAdmitter,
|
||||||
defaultTransientRetries ...uint64,
|
defaultTransientRetries ...uint64,
|
||||||
@@ -112,6 +118,7 @@ func NewRunnerWithRepairer(
|
|||||||
retries = defaultTransientRetries[0]
|
retries = defaultTransientRetries[0]
|
||||||
}
|
}
|
||||||
return &Runner{
|
return &Runner{
|
||||||
|
identity: &runnerIdentity{},
|
||||||
promptDefs: promptDefs,
|
promptDefs: promptDefs,
|
||||||
profiles: profiles,
|
profiles: profiles,
|
||||||
artifacts: artifacts,
|
artifacts: artifacts,
|
||||||
@@ -204,7 +211,7 @@ func (r *Runner) executePreparedRun(
|
|||||||
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||||
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
|
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.shouldAttemptRepair(prepared.OutputContract, validationResult) {
|
if r.shouldAttemptRepair(prepared.OutputContract, validationResult) {
|
||||||
@@ -238,7 +245,7 @@ func (r *Runner) executePreparedRun(
|
|||||||
|
|
||||||
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
|
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -269,10 +276,10 @@ func (r *Runner) executePreparedRun(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func wrapGenerationError(err error) error {
|
func wrapGenerationError(err error) error {
|
||||||
if errors.Is(err, llm.ErrInvalidRequest) {
|
if _, ok := err.(*generation.PreparationError); ok {
|
||||||
return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return &OperationError{Category: ErrInvalidRequest, Cause: err}
|
||||||
}
|
}
|
||||||
return fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
return &OperationError{Category: ErrLLMGenerate, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
||||||
@@ -402,13 +409,9 @@ func (r *Runner) prepareValidation(
|
|||||||
if r.validator == nil {
|
if r.validator == nil {
|
||||||
return noOpPreparedValidation{contract: contract}, nil
|
return noOpPreparedValidation{contract: contract}, nil
|
||||||
}
|
}
|
||||||
preparer, ok := r.validator.(validate.ValidationPreparer)
|
plan, err := r.validator.PrepareValidation(ctx, contract)
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
|
|
||||||
}
|
|
||||||
plan, err := preparer.PrepareValidation(ctx, contract)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, &OperationError{Category: ErrValidation, Cause: err}
|
||||||
}
|
}
|
||||||
if plan == nil {
|
if plan == nil {
|
||||||
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||||
@@ -432,11 +435,7 @@ func (r *Runner) structuredOutputFromValidationPlan(
|
|||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
||||||
}
|
}
|
||||||
schemaDocument, err := jsonvalue.Copy(schemaDocument)
|
return structuredOutputSpec(def, jsonvalue.CloneTrusted(schemaDocument)), nil
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: invalid prepared json_schema schema document: %v", ErrValidation, err)
|
|
||||||
}
|
|
||||||
return structuredOutputSpec(def, schemaDocument), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) completePreparationWithStructuredOutput(
|
func (r *Runner) completePreparationWithStructuredOutput(
|
||||||
@@ -450,7 +449,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
|
|||||||
for name, ref := range req.Inputs {
|
for name, ref := range req.Inputs {
|
||||||
art, readErr := r.artifacts.Read(ctx, ref)
|
art, readErr := r.artifacts.Read(ctx, ref)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
return nil, fmt.Errorf("%w: input %q: %w", ErrArtifactLoad, name, readErr)
|
return nil, &OperationError{Category: ErrArtifactLoad, Cause: fmt.Errorf("input %q: %w", name, readErr)}
|
||||||
}
|
}
|
||||||
if art.Name == "" {
|
if art.Name == "" {
|
||||||
art.Name = name
|
art.Name = name
|
||||||
@@ -467,7 +466,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
|
|||||||
}
|
}
|
||||||
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
|
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
|
return nil, &OperationError{Category: ErrPromptRender, Cause: err}
|
||||||
}
|
}
|
||||||
effectivePrompt := *renderedPrompt
|
effectivePrompt := *renderedPrompt
|
||||||
if state.directSessionID != "" {
|
if state.directSessionID != "" {
|
||||||
@@ -593,14 +592,16 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, presence domain.E
|
|||||||
out.APIKeyEnv = override.APIKeyEnv
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
}
|
}
|
||||||
if len(override.ExtraParams) > 0 {
|
if len(override.ExtraParams) > 0 {
|
||||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
out.ExtraParams = override.ExtraParams
|
||||||
}
|
}
|
||||||
return out, presence
|
return out, presence
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
|
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
|
||||||
out := *profileValue.ResolvedTarget
|
out := *profileValue.ResolvedTarget
|
||||||
out.ExtraParams = copyExtraParams(out.ExtraParams)
|
if len(out.ExtraParams) == 0 {
|
||||||
|
out.ExtraParams = nil
|
||||||
|
}
|
||||||
presence := profileValue.TargetPresence
|
presence := profileValue.TargetPresence
|
||||||
if override != nil {
|
if override != nil {
|
||||||
return mergeExecutionTargetOverride(out, presence, *override)
|
return mergeExecutionTargetOverride(out, presence, *override)
|
||||||
@@ -625,22 +626,8 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyExtraParams(src map[string]any) map[string]any {
|
|
||||||
if len(src) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
cp := make(map[string]any, len(src))
|
|
||||||
for k, v := range src {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
return cp
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) {
|
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) {
|
||||||
contract := def.Validation
|
contract := def.Validation
|
||||||
if contract.Format == "" {
|
|
||||||
contract.Format = def.OutputFormat
|
|
||||||
}
|
|
||||||
if override != nil {
|
if override != nil {
|
||||||
contract = *override
|
contract = *override
|
||||||
if contract.Format == "" {
|
if contract.Format == "" {
|
||||||
@@ -707,7 +694,3 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
|
|||||||
Hash: hex.EncodeToString(hash[:]),
|
Hash: hex.EncodeToString(hash[:]),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
|
|
||||||
return promptdef.HashDefinition(def)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,13 +13,17 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/prompt"
|
"go.maximumdirect.net/promptkit/internal/prompt"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/promptdef"
|
||||||
"go.maximumdirect.net/promptkit/internal/validate"
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,11 +43,61 @@ type staticPromptRepo struct {
|
|||||||
def *domain.PromptDefinition
|
def *domain.PromptDefinition
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r staticPromptRepo) GetPromptDefinition(
|
type capturedTemplateFixture struct {
|
||||||
|
id string
|
||||||
|
templates prompt.CompiledTemplates
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fixture capturedTemplateFixture) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
|
||||||
|
return fixture.templates, id == fixture.id
|
||||||
|
}
|
||||||
|
|
||||||
|
func testCompiledRenderer(t *testing.T, definition *domain.PromptDefinition) prompt.Renderer {
|
||||||
|
t.Helper()
|
||||||
|
var templates prompt.CompiledTemplates
|
||||||
|
if definition.SessionID != "" {
|
||||||
|
var err error
|
||||||
|
templates.Session, err = prompt.CompileTemplate("session", definition.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
templates.Messages = make([]*template.Template, 0, len(definition.Templates))
|
||||||
|
for _, message := range definition.Templates {
|
||||||
|
compiled, err := prompt.CompileTemplate("message", message.Content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
templates.Messages = append(templates.Messages, compiled)
|
||||||
|
}
|
||||||
|
return prompt.NewCompiledRenderer(capturedTemplateFixture{id: definition.ID, templates: templates})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r staticPromptRepo) GetPromptSnapshot(
|
||||||
context.Context,
|
context.Context,
|
||||||
string,
|
string,
|
||||||
) (*domain.PromptDefinition, error) {
|
) (*domain.PromptDefinition, string, error) {
|
||||||
return r.def, nil
|
hash, err := promptdef.HashDefinition(r.def)
|
||||||
|
return r.def, hash, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r staticPromptRepo) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
|
definition, hash, err := r.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (staticPromptRepo) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixturePromptInspection(definition *domain.PromptDefinition, hash string) *domain.PromptInspection {
|
||||||
|
if definition == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &domain.PromptInspection{
|
||||||
|
PromptID: definition.ID, PromptHash: hash, DefaultProfileID: definition.DefaultProfile,
|
||||||
|
Inputs: append([]domain.PromptInput(nil), definition.Inputs...), OutputContract: definition.Validation,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type staticExecutionProfileRepo struct {
|
type staticExecutionProfileRepo struct {
|
||||||
@@ -58,6 +112,10 @@ func (r staticExecutionProfileRepo) GetProfile(
|
|||||||
return &value, nil
|
return &value, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (staticExecutionProfileRepo) GetProfileIdentity(_ context.Context, id string) (string, error) {
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
type staticArtifactReader struct{}
|
type staticArtifactReader struct{}
|
||||||
|
|
||||||
func (staticArtifactReader) Read(
|
func (staticArtifactReader) Read(
|
||||||
@@ -87,17 +145,40 @@ func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*
|
|||||||
}
|
}
|
||||||
if p, ok := f.profiles[id]; ok {
|
if p, ok := f.profiles[id]; ok {
|
||||||
cp := *p
|
cp := *p
|
||||||
|
if cp.ResolvedTarget != nil {
|
||||||
|
target := *cp.ResolvedTarget
|
||||||
|
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
|
||||||
|
cp.ResolvedTarget = &target
|
||||||
|
}
|
||||||
return &cp, nil
|
return &cp, nil
|
||||||
}
|
}
|
||||||
return nil, errors.New("profile not found")
|
return nil, errors.New("profile not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
|
func (f *fakeExecutionProfileRepo) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
_, err := f.GetProfile(ctx, id)
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakePromptRepo) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error) {
|
||||||
f.lastID = id
|
f.lastID = id
|
||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return nil, f.err
|
return nil, "", f.err
|
||||||
}
|
}
|
||||||
return f.def, nil
|
if f.def == nil {
|
||||||
|
return nil, "", nil
|
||||||
|
}
|
||||||
|
hash, err := promptdef.HashDefinition(f.def)
|
||||||
|
return f.def, hash, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakePromptRepo) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
|
definition, hash, err := f.GetPromptSnapshot(ctx, id)
|
||||||
|
return fixturePromptInspection(definition, hash), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*fakePromptRepo) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
|
||||||
|
return prompt.RequirementSnapshot{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeArtifactReader struct {
|
type fakeArtifactReader struct {
|
||||||
@@ -165,14 +246,6 @@ type fakeValidator struct {
|
|||||||
validateCalls int
|
validateCalls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
|
||||||
f.validateCalls++
|
|
||||||
if f.err != nil {
|
|
||||||
return domain.ValidationResult{}, f.err
|
|
||||||
}
|
|
||||||
return f.result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
|
||||||
var schemaDocument any
|
var schemaDocument any
|
||||||
if contract.ValidationMode == domain.ValidationJSONSchema {
|
if contract.ValidationMode == domain.ValidationJSONSchema {
|
||||||
@@ -186,17 +259,20 @@ func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.Out
|
|||||||
schemaDocument = map[string]any{"type": "object"}
|
schemaDocument = map[string]any{"type": "object"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &fakePreparedValidator{validator: f, contract: contract, schemaDocument: schemaDocument}, nil
|
return &fakePreparedValidator{validator: f, schemaDocument: schemaDocument}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakePreparedValidator struct {
|
type fakePreparedValidator struct {
|
||||||
validator *fakeValidator
|
validator *fakeValidator
|
||||||
contract domain.OutputContract
|
|
||||||
schemaDocument any
|
schemaDocument any
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
||||||
return p.validator.Validate(ctx, artifact, p.contract)
|
p.validator.validateCalls++
|
||||||
|
if p.validator.err != nil {
|
||||||
|
return domain.ValidationResult{}, p.validator.err
|
||||||
|
}
|
||||||
|
return p.validator.result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *fakePreparedValidator) SchemaDocument() any {
|
func (p *fakePreparedValidator) SchemaDocument() any {
|
||||||
@@ -405,7 +481,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -452,7 +528,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -482,7 +558,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -842,7 +918,7 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -869,7 +945,7 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
|
|||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
testCompiledRenderer(t, def),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil, nil)
|
nil, nil)
|
||||||
|
|
||||||
@@ -2074,7 +2150,7 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
func TestRunnerRunInjectedInvalidRequestRemainsGenerationFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
@@ -2089,11 +2165,8 @@ func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
|||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrLLMGenerate) || !errors.Is(err, llm.ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected generation error retaining cause, got %v", err)
|
||||||
}
|
|
||||||
if errors.Is(err, ErrLLMGenerate) {
|
|
||||||
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2401,7 +2474,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
|
|||||||
baseClient := &controlledRepairLLM{
|
baseClient := &controlledRepairLLM{
|
||||||
events: make(chan controlledGeneration, 4),
|
events: make(chan controlledGeneration, 4),
|
||||||
}
|
}
|
||||||
scheduledClient := capacity.NewClient(manager, baseClient)
|
scheduledClient := generation.NewInjected(manager, baseClient)
|
||||||
admitter := &recordingRunAdmitter{next: manager}
|
admitter := &recordingRunAdmitter{next: manager}
|
||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
@@ -2507,7 +2580,6 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
|
|||||||
DefaultProfile: "exec",
|
DefaultProfile: "exec",
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
||||||
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
|
||||||
OutputFormat: format,
|
|
||||||
Validation: domain.OutputContract{
|
Validation: domain.OutputContract{
|
||||||
ValidationMode: mode,
|
ValidationMode: mode,
|
||||||
RepairAttempts: attempts,
|
RepairAttempts: attempts,
|
||||||
@@ -2589,10 +2661,6 @@ func float64Ptr(v float64) *float64 {
|
|||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
func stringPtr(v string) *string {
|
|
||||||
return &v
|
|
||||||
}
|
|
||||||
|
|
||||||
func intPtr(v int) *int {
|
func intPtr(v int) *int {
|
||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,8 @@
|
|||||||
package usecase
|
package usecase
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
@@ -13,10 +10,12 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
type authenticationProbe struct {
|
type smokeAuthenticationClient interface {
|
||||||
client smokeHTTPGetter
|
ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type authenticationProbe struct{ client smokeAuthenticationClient }
|
||||||
|
|
||||||
type authenticationProbeOutcome struct {
|
type authenticationProbeOutcome struct {
|
||||||
supported bool
|
supported bool
|
||||||
applicable *bool
|
applicable *bool
|
||||||
@@ -26,16 +25,33 @@ type authenticationProbeOutcome struct {
|
|||||||
failure domain.ExecutionFailureKind
|
failure domain.ExecutionFailureKind
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p authenticationProbe) run(
|
func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, target domain.SmokeTestTarget) (authenticationProbeOutcome, error) {
|
||||||
ctx context.Context,
|
|
||||||
backend domain.Backend,
|
|
||||||
target domain.SmokeTestTarget,
|
|
||||||
) (authenticationProbeOutcome, error) {
|
|
||||||
outcome, err := p.support(backend)
|
outcome, err := p.support(backend)
|
||||||
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
|
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
|
||||||
return outcome, err
|
return outcome, err
|
||||||
}
|
}
|
||||||
return p.runOpenRouterKey(ctx, target)
|
result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
|
||||||
|
outcome.evidence = result.HTTP
|
||||||
|
outcome.credentialsAccepted = result.Accepted
|
||||||
|
if result.Overflow {
|
||||||
|
outcome.reason = domain.SmokeTestReasonAuthenticationResponseLimitExceeded
|
||||||
|
outcome.failure = domain.ExecutionFailureOther
|
||||||
|
} else if result.Unrecognized {
|
||||||
|
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
||||||
|
outcome.failure = domain.ExecutionFailureMalformedResponse
|
||||||
|
} else if requestErr != nil {
|
||||||
|
if !result.HTTP.DispatchInvoked {
|
||||||
|
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
|
||||||
|
}
|
||||||
|
outcome.failure = httpProbeFailure(ctx, requestErr)
|
||||||
|
} else if result.HTTP.ResponseReceived {
|
||||||
|
if result.HTTP.HTTPStatus != nil && *result.HTTP.HTTPStatus != http.StatusOK && *result.HTTP.HTTPStatus != http.StatusUnauthorized {
|
||||||
|
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
outcome.failure = httpProbeFailure(ctx, nil)
|
||||||
|
}
|
||||||
|
return outcome, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
|
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
|
||||||
@@ -48,11 +64,7 @@ func (p authenticationProbe) support(backend domain.Backend) (authenticationProb
|
|||||||
}
|
}
|
||||||
switch *authentication.Kind {
|
switch *authentication.Kind {
|
||||||
case "none":
|
case "none":
|
||||||
return authenticationProbeOutcome{
|
return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil
|
||||||
supported: true,
|
|
||||||
applicable: boolPointer(false),
|
|
||||||
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
|
|
||||||
}, nil
|
|
||||||
case "openrouter_key":
|
case "openrouter_key":
|
||||||
if p.client == nil {
|
if p.client == nil {
|
||||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
||||||
@@ -62,129 +74,3 @@ func (p authenticationProbe) support(backend domain.Backend) (authenticationProb
|
|||||||
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
|
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p authenticationProbe) runOpenRouterKey(
|
|
||||||
ctx context.Context,
|
|
||||||
target domain.SmokeTestTarget,
|
|
||||||
) (authenticationProbeOutcome, error) {
|
|
||||||
endpoint, err := endpointWithoutUserInfo(target.Endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return authenticationProbeOutcome{}, fmt.Errorf("prepare authentication endpoint: %w", err)
|
|
||||||
}
|
|
||||||
response, evidence, requestErr := p.client.GetWithAuthenticationEvidence(
|
|
||||||
ctx,
|
|
||||||
endpoint,
|
|
||||||
target.EffectiveSettings.TimeoutSeconds,
|
|
||||||
target.EffectiveSettings,
|
|
||||||
)
|
|
||||||
if response != nil && response.Body != nil {
|
|
||||||
defer response.Body.Close()
|
|
||||||
}
|
|
||||||
outcome := authenticationProbeOutcome{
|
|
||||||
supported: true,
|
|
||||||
applicable: boolPointer(true),
|
|
||||||
evidence: evidence,
|
|
||||||
}
|
|
||||||
if evidence.ResponseReceived || response != nil {
|
|
||||||
status := evidence.HTTPStatus
|
|
||||||
if status == nil && response != nil {
|
|
||||||
status = &response.StatusCode
|
|
||||||
}
|
|
||||||
if status == nil {
|
|
||||||
outcome.failure = domain.ExecutionFailureMalformedResponse
|
|
||||||
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
switch *status {
|
|
||||||
case http.StatusUnauthorized:
|
|
||||||
outcome.credentialsAccepted = boolPointer(false)
|
|
||||||
case http.StatusOK:
|
|
||||||
if response == nil || response.Body == nil {
|
|
||||||
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
|
||||||
outcome.failure = domain.ExecutionFailureMalformedResponse
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
recognized, bodyErr := recognizedOpenRouterKeyResponse(response.Body)
|
|
||||||
if bodyErr != nil {
|
|
||||||
outcome.failure = httpProbeFailure(ctx, bodyErr)
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
if recognized {
|
|
||||||
outcome.credentialsAccepted = boolPointer(true)
|
|
||||||
} else {
|
|
||||||
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
|
|
||||||
outcome.failure = domain.ExecutionFailureMalformedResponse
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
|
|
||||||
}
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if evidence.DispatchInvoked {
|
|
||||||
if requestErr != nil {
|
|
||||||
outcome.failure = httpProbeFailure(ctx, requestErr)
|
|
||||||
} else {
|
|
||||||
outcome.failure = httpProbeFailure(ctx, nil)
|
|
||||||
}
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
if requestErr != nil {
|
|
||||||
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
|
|
||||||
}
|
|
||||||
outcome.failure = generation.ClassifyFailure(llm.ErrTransport)
|
|
||||||
return outcome, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func recognizedOpenRouterKeyResponse(body io.Reader) (bool, error) {
|
|
||||||
contents, err := io.ReadAll(body)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
var envelope map[string]json.RawMessage
|
|
||||||
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
dataJSON, ok := envelope["data"]
|
|
||||||
if !ok || !isJSONObject(dataJSON) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
var data map[string]json.RawMessage
|
|
||||||
if err := json.Unmarshal(dataJSON, &data); err != nil || data == nil {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
label, labelOK := data["label"]
|
|
||||||
freeTier, freeTierOK := data["is_free_tier"]
|
|
||||||
usage, usageOK := data["usage"]
|
|
||||||
if !labelOK || !isJSONString(label) || !freeTierOK || !isJSONBoolean(freeTier) || !usageOK {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
var number any
|
|
||||||
usageDecoder := json.NewDecoder(bytes.NewReader(usage))
|
|
||||||
usageDecoder.UseNumber()
|
|
||||||
if usageDecoder.Decode(&number) != nil {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
_, ok = number.(json.Number)
|
|
||||||
return ok, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isJSONObject(value json.RawMessage) bool {
|
|
||||||
value = bytes.TrimSpace(value)
|
|
||||||
return len(value) > 0 && value[0] == '{'
|
|
||||||
}
|
|
||||||
|
|
||||||
func isJSONString(value json.RawMessage) bool {
|
|
||||||
value = bytes.TrimSpace(value)
|
|
||||||
if len(value) == 0 || value[0] != '"' {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
var parsed string
|
|
||||||
return json.Unmarshal(value, &parsed) == nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isJSONBoolean(value json.RawMessage) bool {
|
|
||||||
value = bytes.TrimSpace(value)
|
|
||||||
return bytes.Equal(value, []byte("true")) || bytes.Equal(value, []byte("false"))
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -11,6 +12,58 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type measuredAuthenticationBody struct {
|
||||||
|
reader io.Reader
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *measuredAuthenticationBody) Read(p []byte) (int, error) {
|
||||||
|
n, err := b.reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
func (b *measuredAuthenticationBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestAuthenticationResponseRecognitionBound(t *testing.T) {
|
||||||
|
const maxAuthenticationResponseBytes = 1 << 20
|
||||||
|
valid := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
declared int64
|
||||||
|
status int
|
||||||
|
overflow bool
|
||||||
|
}{
|
||||||
|
{"exact limit", maxAuthenticationResponseBytes, -1, 200, false},
|
||||||
|
{"overflow unknown length", maxAuthenticationResponseBytes + 10, -1, 200, true},
|
||||||
|
{"overflow understated length", maxAuthenticationResponseBytes + 10, 1, 200, true},
|
||||||
|
{"unauthorized does not read", maxAuthenticationResponseBytes + 10, -1, 401, false},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
body := &measuredAuthenticationBody{reader: strings.NewReader(valid + strings.Repeat(" ", test.size-len(valid)))}
|
||||||
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: test.status, Header: make(http.Header), Body: body, ContentLength: test.declared}, nil
|
||||||
|
})
|
||||||
|
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"})
|
||||||
|
if err != nil || !body.closed {
|
||||||
|
t.Fatalf("outcome = %+v, err = %v, closed = %v", outcome, err, body.closed)
|
||||||
|
}
|
||||||
|
if test.overflow {
|
||||||
|
if outcome.reason != domain.SmokeTestReasonAuthenticationResponseLimitExceeded || outcome.failure != domain.ExecutionFailureOther || outcome.credentialsAccepted != nil || body.read != maxAuthenticationResponseBytes+1 {
|
||||||
|
t.Fatalf("overflow = %+v, read = %d", outcome, body.read)
|
||||||
|
}
|
||||||
|
} else if test.status == 401 {
|
||||||
|
if body.read != 0 || outcome.credentialsAccepted == nil || *outcome.credentialsAccepted {
|
||||||
|
t.Fatalf("unauthorized = %+v, read = %d", outcome, body.read)
|
||||||
|
}
|
||||||
|
} else if outcome.credentialsAccepted == nil || !*outcome.credentialsAccepted {
|
||||||
|
t.Fatalf("exact limit = %+v", outcome)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
|
func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
|
||||||
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
t.Fatal("authentication applicability unexpectedly dispatched")
|
t.Fatal("authentication applicability unexpectedly dispatched")
|
||||||
@@ -159,21 +212,26 @@ func TestAuthenticationProbeInterpretsOpenRouterResponses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) {
|
func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) {
|
||||||
for _, body := range []string{
|
for _, tc := range []struct {
|
||||||
`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`,
|
body string
|
||||||
`{"data":[]}`,
|
accepted bool
|
||||||
`{"data":{"label":"member","is_free_tier":false,"usage":null}}`,
|
}{
|
||||||
`{"data":{"label":"member","is_free_tier":0,"usage":1}}`,
|
{`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`, false},
|
||||||
`{"data":{"is_free_tier":false,"usage":1}}`,
|
{`{"data":[]}`, false},
|
||||||
`{"data":{"label":null,"is_free_tier":false,"usage":1}}`,
|
{`{"data":{"label":"member","is_free_tier":false,"usage":null}}`, false},
|
||||||
|
{`{"data":{"label":"member","is_free_tier":0,"usage":1}}`, false},
|
||||||
|
{`{"data":{"is_free_tier":false,"usage":1}}`, false},
|
||||||
|
{`{"data":{"label":null,"is_free_tier":false,"usage":1}}`, false},
|
||||||
|
{`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`, true},
|
||||||
} {
|
} {
|
||||||
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(body)); err != nil || recognized {
|
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
|
||||||
t.Errorf("recognized invalid account response %s", body)
|
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: ioNopCloser(tc.body)}, nil
|
||||||
|
})
|
||||||
|
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"})
|
||||||
|
if err != nil || (outcome.credentialsAccepted != nil && *outcome.credentialsAccepted) != tc.accepted {
|
||||||
|
t.Errorf("account response %s: outcome = %+v, error = %v", tc.body, outcome, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`)); err != nil || !recognized {
|
|
||||||
t.Fatal("valid minimal identifying response was rejected")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) {
|
func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) {
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ type smokeInferenceCase struct {
|
|||||||
kind domain.SmokeTestKind
|
kind domain.SmokeTestKind
|
||||||
request domain.GenerateRequest
|
request domain.GenerateRequest
|
||||||
expectedContent string
|
expectedContent string
|
||||||
jsonValidator validate.Validator
|
jsonValidation validate.PreparedValidation
|
||||||
schemaValidation validate.PreparedValidation
|
schemaValidation validate.PreparedValidation
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -52,11 +52,13 @@ func prepareSmokeInferenceCase(
|
|||||||
}, nil
|
}, nil
|
||||||
case domain.SmokeTestStructuredOutput:
|
case domain.SmokeTestStructuredOutput:
|
||||||
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
||||||
preparer, ok := validator.(validate.ValidationPreparer)
|
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||||
if !ok {
|
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
|
||||||
return nil, fmt.Errorf("%w: private validator does not support prepared validation", ErrValidation)
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
plan, err := preparer.PrepareValidation(ctx, domain.OutputContract{
|
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||||
Format: domain.FormatJSON,
|
Format: domain.FormatJSON,
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
SchemaPath: smokeStructuredSchemaPath,
|
SchemaPath: smokeStructuredSchemaPath,
|
||||||
@@ -68,7 +70,7 @@ func prepareSmokeInferenceCase(
|
|||||||
return &smokeInferenceCase{
|
return &smokeInferenceCase{
|
||||||
kind: kind,
|
kind: kind,
|
||||||
expectedContent: smokeStructuredExpectedContent,
|
expectedContent: smokeStructuredExpectedContent,
|
||||||
jsonValidator: validate.NewStandardValidator(""),
|
jsonValidation: jsonPlan,
|
||||||
schemaValidation: plan,
|
schemaValidation: plan,
|
||||||
request: newGenerationRequest(
|
request: newGenerationRequest(
|
||||||
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
|
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
|
||||||
@@ -101,10 +103,7 @@ func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.Gene
|
|||||||
return facts, nil
|
return facts, nil
|
||||||
case domain.SmokeTestStructuredOutput:
|
case domain.SmokeTestStructuredOutput:
|
||||||
artifact := domain.Artifact{Body: []byte(response.Content)}
|
artifact := domain.Artifact{Body: []byte(response.Content)}
|
||||||
jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{
|
jsonResult, err := c.jsonValidation.Validate(ctx, &artifact)
|
||||||
Format: domain.FormatJSON,
|
|
||||||
ValidationMode: domain.ValidationJSON,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
|
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package usecase
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -102,6 +103,7 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
|
{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
|
||||||
{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
||||||
|
{name: "scalar JSON misses schema", content: `42`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
||||||
{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
||||||
{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
|
{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
|
||||||
{name: "empty content", content: "", wantSchemaMatched: nil},
|
{name: "empty content", content: "", wantSchemaMatched: nil},
|
||||||
@@ -125,3 +127,19 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
|
|||||||
t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
|
t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSmokeStructuredInferenceCaseHonorsCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := prepareSmokeInferenceCase(ctx, domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("prepare error = %v, want cancellation", err)
|
||||||
|
}
|
||||||
|
caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
facts, err := caseValue.evaluate(ctx, &domain.GenerateResponse{Content: `{"status":"ok"}`})
|
||||||
|
if !errors.Is(err, context.Canceled) || facts.ValidJSON != nil || facts.SchemaMatched != nil {
|
||||||
|
t.Fatalf("evaluation facts = %+v, error = %v", facts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
|
|
||||||
type smokeHTTPGetter interface {
|
type smokeHTTPGetter interface {
|
||||||
GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error)
|
GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error)
|
||||||
GetWithAuthenticationEvidence(context.Context, string, int, domain.ExecutionTarget) (*http.Response, llm.HTTPExecutionEvidence, error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type reachabilityProbe struct {
|
type reachabilityProbe struct {
|
||||||
@@ -22,12 +21,11 @@ type reachabilityProbe struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type reachabilityProbeOutcome struct {
|
type reachabilityProbeOutcome struct {
|
||||||
supported bool
|
supported bool
|
||||||
reason domain.SmokeTestReason
|
reason domain.SmokeTestReason
|
||||||
initialEndpoint string
|
reachable *bool
|
||||||
reachable *bool
|
evidence llm.HTTPExecutionEvidence
|
||||||
evidence llm.HTTPExecutionEvidence
|
failure domain.ExecutionFailureKind
|
||||||
failure domain.ExecutionFailureKind
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p reachabilityProbe) run(
|
func (p reachabilityProbe) run(
|
||||||
@@ -45,7 +43,7 @@ func (p reachabilityProbe) run(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return reachabilityProbeOutcome{}, fmt.Errorf("prepare reachability endpoint: %w", err)
|
return reachabilityProbeOutcome{}, fmt.Errorf("prepare reachability endpoint: %w", err)
|
||||||
}
|
}
|
||||||
result := reachabilityProbeOutcome{supported: true, initialEndpoint: endpoint}
|
result := reachabilityProbeOutcome{supported: true}
|
||||||
response, evidence, requestErr := p.client.GetWithEvidence(ctx, endpoint, timeoutSeconds)
|
response, evidence, requestErr := p.client.GetWithEvidence(ctx, endpoint, timeoutSeconds)
|
||||||
if response != nil && response.Body != nil {
|
if response != nil && response.Body != nil {
|
||||||
_ = response.Body.Close()
|
_ = response.Body.Close()
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ func TestReachabilityProbeRemovesURLCredentials(t *testing.T) {
|
|||||||
if observed == nil || observed.URL.User != nil || observed.Header.Get("Authorization") != "" {
|
if observed == nil || observed.URL.User != nil || observed.Header.Get("Authorization") != "" {
|
||||||
t.Fatalf("probe request retained URL credentials: %+v", observed)
|
t.Fatalf("probe request retained URL credentials: %+v", observed)
|
||||||
}
|
}
|
||||||
if result.initialEndpoint != "https://example.invalid/health?check=1" || result.evidence.CredentialsSupplied {
|
if result.evidence.Endpoint != "https://example.invalid/health?check=1" || result.evidence.CredentialsSupplied {
|
||||||
t.Fatalf("credential-free evidence = %+v", result)
|
t.Fatalf("credential-free evidence = %+v", result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -111,12 +111,9 @@ func TestReachabilityProbeIsUnsupportedWithoutHTTPClient(t *testing.T) {
|
|||||||
|
|
||||||
func reachabilityClient(t *testing.T, roundTrip func(*http.Request) (*http.Response, error)) *llm.OpenAICompatibleClient {
|
func reachabilityClient(t *testing.T, roundTrip func(*http.Request) (*http.Response, error)) *llm.OpenAICompatibleClient {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
||||||
HTTPClient: &http.Client{Transport: reachabilityProbeTransport(roundTrip)},
|
HTTPClient: &http.Client{Transport: reachabilityProbeTransport(roundTrip)},
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return client
|
return client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,10 +3,10 @@ package usecase
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -112,7 +112,9 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
|
|||||||
settings.BackendID = selection.backend.ID
|
settings.BackendID = selection.backend.ID
|
||||||
settings.Endpoint = selection.backend.Endpoint
|
settings.Endpoint = selection.backend.Endpoint
|
||||||
settings.APIKeyEnv = selection.backend.APIKeyEnv
|
settings.APIKeyEnv = selection.backend.APIKeyEnv
|
||||||
settings.ExtraParams = copyExtraParams(selection.backend.ExtraParams)
|
if len(selection.backend.ExtraParams) > 0 {
|
||||||
|
settings.ExtraParams = selection.backend.ExtraParams
|
||||||
|
}
|
||||||
presence := domain.ExecutionTargetPresence{}
|
presence := domain.ExecutionTargetPresence{}
|
||||||
if req.Execution != nil {
|
if req.Execution != nil {
|
||||||
settings, presence = mergeExecutionTargetOverride(settings, presence, *req.Execution)
|
settings, presence = mergeExecutionTargetOverride(settings, presence, *req.Execution)
|
||||||
@@ -144,7 +146,7 @@ func (r *Runner) resolveSmokeBackend(id string) (domain.Backend, error) {
|
|||||||
}
|
}
|
||||||
backend, err := lookup.GetBackend(id)
|
backend, err := lookup.GetBackend(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.Backend{}, fmt.Errorf("%w: backend %q: %v", ErrProfileLoad, id, err)
|
return domain.Backend{}, &OperationError{Category: ErrProfileLoad, Cause: fmt.Errorf("backend %q: %w", id, err)}
|
||||||
}
|
}
|
||||||
return backend, nil
|
return backend, nil
|
||||||
}
|
}
|
||||||
@@ -176,7 +178,7 @@ func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, sm
|
|||||||
return *authentication.Endpoint, nil
|
return *authentication.Endpoint, nil
|
||||||
}
|
}
|
||||||
if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
|
if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
|
||||||
return url.JoinPath(baseEndpoint, "key")
|
return llm.OpenRouterKeyEndpoint(baseEndpoint)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/configuration"
|
"go.maximumdirect.net/promptkit/internal/configuration"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
"go.maximumdirect.net/promptkit/internal/profile"
|
"go.maximumdirect.net/promptkit/internal/profile"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
type selectionTestCollaborators struct {
|
type selectionTestCollaborators struct {
|
||||||
@@ -41,9 +42,9 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do
|
|||||||
|
|
||||||
type selectionTestValidator struct{ state *selectionTestCollaborators }
|
type selectionTestValidator struct{ state *selectionTestCollaborators }
|
||||||
|
|
||||||
func (v selectionTestValidator) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) {
|
func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
|
||||||
v.state.validationCalls++
|
v.state.validationCalls++
|
||||||
return domain.ValidationResult{}, errors.New("validation must not occur during target selection")
|
return nil, errors.New("validation must not occur during target selection")
|
||||||
}
|
}
|
||||||
|
|
||||||
type selectionTestRepositories struct {
|
type selectionTestRepositories struct {
|
||||||
@@ -57,6 +58,10 @@ func (r *selectionTestRepositories) GetProfile(ctx context.Context, id string) (
|
|||||||
return r.profiles.GetProfile(ctx, id)
|
return r.profiles.GetProfile(ctx, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *selectionTestRepositories) GetProfileIdentity(ctx context.Context, id string) (string, error) {
|
||||||
|
return r.profiles.GetProfileIdentity(ctx, id)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) {
|
func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) {
|
||||||
return r.backends.GetBackend(id)
|
return r.backends.GetBackend(id)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
type SmokeTester struct {
|
type SmokeTester struct {
|
||||||
runner *Runner
|
runner *Runner
|
||||||
httpClient smokeHTTPGetter
|
httpClient smokeHTTPGetter
|
||||||
|
authClient smokeAuthenticationClient
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -24,18 +25,22 @@ type SmokeTester struct {
|
|||||||
// leaves HTTP operations unsupported rather than falling back to another client.
|
// leaves HTTP operations unsupported rather than falling back to another client.
|
||||||
func NewSmokeTester(runner *Runner, httpClient *llm.OpenAICompatibleClient) *SmokeTester {
|
func NewSmokeTester(runner *Runner, httpClient *llm.OpenAICompatibleClient) *SmokeTester {
|
||||||
var getter smokeHTTPGetter
|
var getter smokeHTTPGetter
|
||||||
|
var authentication smokeAuthenticationClient
|
||||||
if httpClient != nil {
|
if httpClient != nil {
|
||||||
getter = httpClient
|
getter = httpClient
|
||||||
|
authentication = httpClient
|
||||||
}
|
}
|
||||||
return &SmokeTester{runner: runner, httpClient: getter, now: time.Now}
|
return &SmokeTester{runner: runner, httpClient: getter, authClient: authentication, now: time.Now}
|
||||||
}
|
}
|
||||||
|
|
||||||
func newSmokeTesterWithClock(runner *Runner, httpClient *llm.OpenAICompatibleClient, now func() time.Time) *SmokeTester {
|
func newSmokeTesterWithClock(runner *Runner, httpClient *llm.OpenAICompatibleClient, now func() time.Time) *SmokeTester {
|
||||||
var getter smokeHTTPGetter
|
var getter smokeHTTPGetter
|
||||||
|
var authentication smokeAuthenticationClient
|
||||||
if httpClient != nil {
|
if httpClient != nil {
|
||||||
getter = httpClient
|
getter = httpClient
|
||||||
|
authentication = httpClient
|
||||||
}
|
}
|
||||||
return &SmokeTester{runner: runner, httpClient: getter, now: now}
|
return &SmokeTester{runner: runner, httpClient: getter, authClient: authentication, now: now}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run selects and executes one diagnostic, retaining only factual results.
|
// Run selects and executes one diagnostic, retaining only factual results.
|
||||||
@@ -126,7 +131,7 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
|
|||||||
return t.finishHTTP(ctx, report, finish)
|
return t.finishHTTP(ctx, report, finish)
|
||||||
case domain.SmokeTestAuthentication:
|
case domain.SmokeTestAuthentication:
|
||||||
requestStartedAt := now().UTC()
|
requestStartedAt := now().UTC()
|
||||||
outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target)
|
outcome, err := (authenticationProbe{client: t.authClient}).run(ctx, selection.backend, selection.target)
|
||||||
requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
|
requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if contextErr := ctx.Err(); contextErr != nil {
|
if contextErr := ctx.Err(); contextErr != nil {
|
||||||
@@ -210,6 +215,9 @@ func (t *SmokeTester) runInference(
|
|||||||
report.Failure = generation.ClassifyFailure(contextErr)
|
report.Failure = generation.ClassifyFailure(contextErr)
|
||||||
return finish(contextErr)
|
return finish(contextErr)
|
||||||
}
|
}
|
||||||
|
if evidence.PreparationFailed && generationErr != nil {
|
||||||
|
return nil, &OperationError{Category: ErrInvalidRequest, Cause: generationErr}
|
||||||
|
}
|
||||||
if generationErr != nil {
|
if generationErr != nil {
|
||||||
report.Failure = generation.ClassifyFailure(generationErr)
|
report.Failure = generation.ClassifyFailure(generationErr)
|
||||||
report.Completed = true
|
report.Completed = true
|
||||||
@@ -346,7 +354,7 @@ func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smok
|
|||||||
report.Applicable = boolPointer(true)
|
report.Applicable = boolPointer(true)
|
||||||
return nil
|
return nil
|
||||||
case domain.SmokeTestAuthentication:
|
case domain.SmokeTestAuthentication:
|
||||||
outcome, err := (authenticationProbe{client: t.httpClient}).support(selection.backend)
|
outcome, err := (authenticationProbe{client: t.authClient}).support(selection.backend)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -430,12 +438,8 @@ func httpExecutionFacts(evidence llm.HTTPExecutionEvidence) domain.SmokeTestFact
|
|||||||
}
|
}
|
||||||
|
|
||||||
func smokeReportTarget(target domain.SmokeTestTarget) (domain.SmokeTestTarget, error) {
|
func smokeReportTarget(target domain.SmokeTestTarget) (domain.SmokeTestTarget, error) {
|
||||||
cloned, err := domain.CloneSmokeTestReport(domain.SmokeTestReport{Target: target})
|
|
||||||
if err != nil {
|
|
||||||
return domain.SmokeTestTarget{}, fmt.Errorf("clone smoke-test target settings: %w", err)
|
|
||||||
}
|
|
||||||
target = cloned.Target
|
|
||||||
target.EffectiveSettings.APIKey = ""
|
target.EffectiveSettings.APIKey = ""
|
||||||
|
var err error
|
||||||
target.Endpoint, err = endpointWithoutUserInfo(target.Endpoint)
|
target.Endpoint, err = endpointWithoutUserInfo(target.Endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved diagnostic endpoint", ErrInvalidRequest)
|
return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved diagnostic endpoint", ErrInvalidRequest)
|
||||||
|
|||||||
@@ -393,3 +393,22 @@ func (b cancelOnCloseSmokeBody) Close() error {
|
|||||||
b.cancel()
|
b.cancel()
|
||||||
return b.ReadCloser.Close()
|
return b.ReadCloser.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSmokeReportTargetRedactsCredentialsAndSanitizesEndpoints(t *testing.T) {
|
||||||
|
target := domain.SmokeTestTarget{
|
||||||
|
Endpoint: "https://user:password@example.invalid/key",
|
||||||
|
EffectiveSettings: domain.ExecutionTarget{
|
||||||
|
Endpoint: "https://user:password@example.invalid/v1",
|
||||||
|
APIKey: "secret",
|
||||||
|
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
got, err := smokeReportTarget(target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] != "original" ||
|
||||||
|
got.EffectiveSettings.APIKey != "" || got.Endpoint != "https://example.invalid/key" || got.EffectiveSettings.Endpoint != "https://example.invalid/v1" {
|
||||||
|
t.Fatalf("report target not redacted: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestValidationCancellationBeforeWorkDoesNotOpenSchemaSource(t *testing.T) {
|
|||||||
source := &countingSchemaFS{FS: fstest.MapFS{
|
source := &countingSchemaFS{FS: fstest.MapFS{
|
||||||
"schema.json": {Data: []byte(`{"type":"object"}`)},
|
"schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
}}
|
}}
|
||||||
validator := NewFSValidator(source, ".").(ValidationPreparer)
|
validator := NewFSValidator(source, ".")
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
@@ -35,17 +35,16 @@ func TestValidationCancellationBeforeWorkDoesNotOpenSchemaSource(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidationCancellationBetweenReferencedSchemaReadChunks(t *testing.T) {
|
func TestValidationCancellationBetweenRootSchemaReadChunks(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
source := &controlledSchemaFS{
|
source := &controlledSchemaFS{
|
||||||
FS: fstest.MapFS{
|
FS: fstest.MapFS{
|
||||||
"root.json": {Data: []byte(`{"$ref":"child.json"}`)},
|
"root.json": {Data: []byte(`{"type":"string"}` + strings.Repeat(" ", schemaReadChunkSize*2))},
|
||||||
"child.json": {Data: []byte(`{"type":"string"}` + strings.Repeat(" ", schemaReadChunkSize*2))},
|
|
||||||
},
|
},
|
||||||
target: "child.json",
|
target: "root.json",
|
||||||
cancel: cancel,
|
cancel: cancel,
|
||||||
}
|
}
|
||||||
validator := NewFSValidator(source, ".").(ValidationPreparer)
|
validator := NewFSValidator(source, ".")
|
||||||
|
|
||||||
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
@@ -57,8 +56,8 @@ func TestValidationCancellationBetweenReferencedSchemaReadChunks(t *testing.T) {
|
|||||||
if reads := source.reads.Load(); reads != 1 {
|
if reads := source.reads.Load(); reads != 1 {
|
||||||
t.Fatalf("controlled child reads = %d, want 1", reads)
|
t.Fatalf("controlled child reads = %d, want 1", reads)
|
||||||
}
|
}
|
||||||
if closes := source.closes.Load(); closes != 1 {
|
if closes := source.closes.Load(); closes != 2 {
|
||||||
t.Fatalf("controlled child closes = %d, want 1", closes)
|
t.Fatalf("controlled root closes = %d, want stat and read handles closed", closes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,7 +112,7 @@ func TestCancellationDoesNotDetachBlockedSchemaRead(t *testing.T) {
|
|||||||
started: started,
|
started: started,
|
||||||
release: release,
|
release: release,
|
||||||
}
|
}
|
||||||
validator := NewFSValidator(source, ".").(ValidationPreparer)
|
validator := NewFSValidator(source, ".")
|
||||||
type outcome struct {
|
type outcome struct {
|
||||||
plan PreparedValidation
|
plan PreparedValidation
|
||||||
err error
|
err error
|
||||||
|
|||||||
91
internal/validate/schema_locations.go
Normal file
91
internal/validate/schema_locations.go
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
package validate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||||
|
)
|
||||||
|
|
||||||
|
// These schema-bearing positions mirror the inherited subschema tables in
|
||||||
|
// jsonschema/v6 v6.0.2 draft.go. Annotation objects are deliberately excluded.
|
||||||
|
func checkSchemaLocations(ctx context.Context, compiler *jsonschema.Compiler, resourceURL string, document any, pointer string, compiled *jsonschema.Schema) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
object, ok := document.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
version := compiled.DraftVersion
|
||||||
|
mapKeys := []string{"definitions", "properties", "patternProperties", "dependencies"}
|
||||||
|
singleKeys := []string{"not", "additionalProperties", "items", "additionalItems"}
|
||||||
|
arrayKeys := []string{"allOf", "anyOf", "oneOf", "items"}
|
||||||
|
if version >= 6 {
|
||||||
|
singleKeys = append(singleKeys, "propertyNames", "contains")
|
||||||
|
}
|
||||||
|
if version >= 7 {
|
||||||
|
singleKeys = append(singleKeys, "if", "then", "else")
|
||||||
|
}
|
||||||
|
if version >= 2019 {
|
||||||
|
mapKeys = append(mapKeys, "$defs", "dependentSchemas")
|
||||||
|
singleKeys = append(singleKeys, "unevaluatedProperties", "unevaluatedItems", "contentSchema")
|
||||||
|
}
|
||||||
|
if version >= 2020 {
|
||||||
|
arrayKeys = append(arrayKeys, "prefixItems")
|
||||||
|
}
|
||||||
|
visit := func(value any, segments ...string) error {
|
||||||
|
switch value.(type) {
|
||||||
|
case map[string]any, bool:
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
childPointer := pointer
|
||||||
|
for _, segment := range segments {
|
||||||
|
childPointer += "/" + strings.ReplaceAll(strings.ReplaceAll(segment, "~", "~0"), "/", "~1")
|
||||||
|
}
|
||||||
|
parsed, err := url.Parse(resourceURL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
parsed.Fragment = childPointer
|
||||||
|
child, err := compileJSONSchema(ctx, parsed.String(), compiler.Compile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("compile declared schema %q: %w", childPointer, err)
|
||||||
|
}
|
||||||
|
return checkSchemaLocations(ctx, compiler, resourceURL, value, childPointer, child)
|
||||||
|
}
|
||||||
|
for _, key := range mapKeys {
|
||||||
|
entries, ok := object[key].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for name, value := range entries {
|
||||||
|
if err := visit(value, key, name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range singleKeys {
|
||||||
|
if value, ok := object[key]; ok {
|
||||||
|
if err := visit(value, key); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range arrayKeys {
|
||||||
|
entries, ok := object[key].([]any)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i, value := range entries {
|
||||||
|
if err := visit(value, key, strconv.Itoa(i)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
60
internal/validate/schema_locations_test.go
Normal file
60
internal/validate/schema_locations_test.go
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
package validate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSchemaLocationsCheckUnusedDeclarations(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, schema string
|
||||||
|
wantError bool
|
||||||
|
}{
|
||||||
|
{"unused external", `{"$defs":{"unused":{"$ref":"other.json"}},"type":"string"}`, true},
|
||||||
|
{"annotation is data", `{"examples":[{"$ref":"other.json"}],"default":{"$ref":"other.json"},"type":"string"}`, false},
|
||||||
|
{"internal escaped key", `{"$defs":{"a/b~c":{"type":"string"}},"$ref":"#/$defs/a~1b~0c"}`, false},
|
||||||
|
{"URI escaped key", `{"$defs":{"a # %/雪":{"type":"string"}}}`, false},
|
||||||
|
{"internal anchor", `{"$defs":{"value":{"$anchor":"item","type":"string"}},"$ref":"#item"}`, false},
|
||||||
|
{"dynamic reference", `{"$dynamicAnchor":"node","type":"object","properties":{"child":{"$dynamicRef":"#node"}}}`, false},
|
||||||
|
{"recursive", `{"$id":"https://example.invalid/root","$defs":{"node":{"$anchor":"node","anyOf":[{"type":"null"},{"$ref":"#node"}]}},"$ref":"#/$defs/node"}`, false},
|
||||||
|
{"embedded old dialect annotation", `{"$defs":{"old":{"$id":"old","$schema":"http://json-schema.org/draft-07/schema#","$defs":{"unused":{"$ref":"other.json"}},"type":"string"}}}`, false},
|
||||||
|
{"embedded old dialect definition", `{"$defs":{"old":{"$id":"old","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"unused":{"$ref":"other.json"}}}}}`, true},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
doc, err := decodeJSONValue(context.Background(), []byte(test.schema))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = compileSchemaDocument(context.Background(), "promptkit-schema:///root.json", doc)
|
||||||
|
if test.wantError && (err == nil || !strings.Contains(err.Error(), "external schema resource")) {
|
||||||
|
t.Fatalf("expected external resource error, got %v", err)
|
||||||
|
}
|
||||||
|
if !test.wantError && err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSchemaLocationsPreserveRootValidation(t *testing.T) {
|
||||||
|
doc, err := decodeJSONValue(context.Background(), []byte(`{"$defs":{"number":{"type":"integer","minimum":2}},"$ref":"#/$defs/number"}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
schema, err := compileSchemaDocument(context.Background(), "promptkit-schema:///root.json", doc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := schema.Validate(3); err != nil {
|
||||||
|
t.Fatalf("valid instance: %v", err)
|
||||||
|
}
|
||||||
|
if err := schema.Validate("not a number"); err == nil {
|
||||||
|
t.Fatal("invalid instance was accepted")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := compileSchemaDocument(ctx, "promptkit-schema:///root.json", doc); err != context.Canceled {
|
||||||
|
t.Fatalf("canceled compilation = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
46
internal/validate/schema_portability_test.go
Normal file
46
internal/validate/schema_portability_test.go
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
package validate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSchemaPreparationDoesNotDependOnSourceName(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name, schema string
|
||||||
|
rejected bool
|
||||||
|
}{
|
||||||
|
{"source name", `{"$defs":{"v":{"type":"integer"}},"$ref":"schema.json#/$defs/v"}`, true},
|
||||||
|
{"unused source name", `{"$defs":{"v":{"$ref":"schema.json"}}}`, true},
|
||||||
|
{"fragment", `{"$defs":{"v":{"type":"integer"}},"$ref":"#/$defs/v"}`, false},
|
||||||
|
{"explicit ID", `{"$id":"https://example.invalid/schema.json","$defs":{"v":{"type":"integer"}},"$ref":"schema.json#/$defs/v"}`, false},
|
||||||
|
{"embedded ID", `{"$defs":{"v":{"$id":"child","type":"integer"}},"$ref":"child"}`, false},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "schema.json"), []byte(test.schema), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fsys := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(test.schema)}}
|
||||||
|
for i, validator := range []ValidationPreparer{NewStandardValidator(dir), NewFSValidator(fsys, ".")} {
|
||||||
|
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
||||||
|
_, err := validator.PrepareValidation(context.Background(), domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json"})
|
||||||
|
if test.rejected {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "source location") {
|
||||||
|
t.Fatalf("expected source-independent rejection, got %v", err)
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user