92 Commits

Author SHA1 Message Date
4f3212bfb6 Write release notes for v0.19.0 2026-11-06 01:20:09 +00:00
4faf43cc29 Remove completed audit and implementation roadmaps 2026-11-06 01:19:33 +00:00
c9663ec63c Protect retained metadata from custom JSON encoders 2026-11-06 01:15:55 +00:00
d8c98ac575 Record the responsibility simplification review 2026-11-06 01:01:14 +00:00
5ee602cc4b Record final validation and repair audit links 2026-11-05 20:19:06 +00:00
96c619f7d4 Update documentation for simplified responsibilities 2026-11-05 20:15:51 +00:00
b3b5ec3820 Verify ownership across execution and repair 2026-11-05 20:14:37 +00:00
5dd1418953 Simplify smoke report ownership 2026-11-05 20:12:40 +00:00
f5c900cb0a Share immutable prepared execution metadata 2026-11-05 20:11:26 +00:00
c5f3de13f3 Transfer owned provider settings during resolution 2026-11-05 20:09:17 +00:00
88725e79ab Detach captured backend and profile values 2026-11-05 20:07:55 +00:00
6c015a17fa Detach schemas without provider value limits 2026-11-05 20:06:37 +00:00
9463c9463a Migrate validator tests to prepared plans 2026-11-05 20:04:55 +00:00
1fbd175e53 Remove direct output validation 2026-11-05 20:01:28 +00:00
4b4ce43796 Prepare both smoke output validators 2026-11-05 20:00:33 +00:00
d78474a3bf Move renderer tests to captured templates 2026-11-05 19:59:34 +00:00
acaf2ccd8b Render only captured prompt templates 2026-11-05 19:57:55 +00:00
97e5739bfb Update tests for captured repository lookups 2026-11-05 19:57:13 +00:00
90f796aef7 Require captured repository lookups 2026-11-05 19:55:10 +00:00
0dec3dcec0 Build provider requests directly as payloads 2026-11-05 19:53:41 +00:00
448e64dc40 Use resolved targets for client routing 2026-11-05 19:52:48 +00:00
de46935a0e Plan the responsibility simplification sprint 2026-11-05 19:24:09 +00:00
c69703b2b6 Define the responsibility simplification roadmap 2026-11-05 19:14:48 +00:00
f660b6e0fc Record prepared validation ownership decision 2026-11-05 19:07:52 +00:00
5f56c65427 Record responsibility audit decisions and ownership guidance 2026-11-05 18:47:51 +00:00
cfa3a85308 Record the architectural responsibility audit 2026-11-05 18:10:01 +00:00
1ff6d778af Preserve operation ownership in terminal observations 2026-11-05 16:03:26 +00:00
3f93763c9c Record the completed audit remediation sprint 2026-11-05 16:00:39 +00:00
4aeb58f913 Avoid duplicating nested error causes 2026-11-05 16:00:39 +00:00
ac1e150168 Validate schemas without relying on source filenames 2026-11-05 15:59:30 +00:00
ff57dbf8b3 Preserve authentication cancellation at the response limit 2026-11-05 15:58:32 +00:00
beb18cc9d6 Clarify algorithmic cost of nested error projection 2026-11-05 15:49:32 +00:00
7203e5d6a1 Record observer integration audit finding 2026-11-05 15:40:18 +00:00
356563a6e5 Record audit remediation review findings 2026-11-05 14:59:41 +00:00
6673284312 Complete documentation reconciliation and validation 2026-11-05 14:37:12 +00:00
6f42e1e007 Retire the legacy capacity client wrapper 2026-11-05 14:35:10 +00:00
cee9bbc7b2 Remove redundant captured prompt fields 2026-11-05 14:34:05 +00:00
7db5572b62 Remove unused captured state and orphan helpers 2026-11-05 14:33:04 +00:00
a91bc1608b Test artifact hashes as opaque values 2026-11-05 14:31:14 +00:00
e7a3665443 Consolidate bundled catalog test expectations 2026-11-05 14:30:14 +00:00
59e8010623 Clarify credential inspection and protect request privacy 2026-11-05 14:28:45 +00:00
ada5377f32 Share generation attempt accounting 2026-11-05 14:27:28 +00:00
df208e132b Return generation attempt results as one value 2026-11-05 14:25:07 +00:00
8c0fb15727 Separate prompt and target capture 2026-11-05 14:24:19 +00:00
2e2fba4993 Separate final configuration record validation 2026-11-05 14:22:34 +00:00
68668cee56 Simplify diagnostic report ownership 2026-11-05 14:21:33 +00:00
f745b14b71 Move authentication probes into the outbound adapter 2026-11-05 14:20:04 +00:00
dbba1485a7 Select inventory namespaces before copying 2026-11-05 14:11:41 +00:00
7c435032b6 Bound repair feedback construction 2026-11-05 14:07:34 +00:00
e834e7d806 Add bounded diagnostic prefix encoding 2026-11-05 14:06:09 +00:00
a4901cbb0e Share requirement branch traversal 2026-11-05 14:04:58 +00:00
5dcfc2e953 Merge condition evidence without repeated copies 2026-11-05 14:03:45 +00:00
a02561d40f Centralize trusted JSON value cloning 2026-11-05 14:02:20 +00:00
6112ee8fb4 Merge provider defaults into one owned copy 2026-11-05 14:00:57 +00:00
0aceff8c08 Verify self-contained schema preparation and submission 2026-11-05 13:59:40 +00:00
cddb2b389c Check declared schema locations for external resources 2026-11-05 13:57:21 +00:00
e056cb1607 Share self-contained schema compilation 2026-11-05 13:56:03 +00:00
1e122f0473 Require validation preparation internally 2026-11-05 13:54:06 +00:00
2f0f5fa607 Detach prepared execution ownership from runners 2026-11-05 13:52:36 +00:00
7d500ee62f Clarify artifact reader ownership transfer 2026-11-05 13:51:29 +00:00
7be677c80b Bound authentication response recognition 2026-11-05 13:49:51 +00:00
1c075bd510 Identify built-in generation preparation failures 2026-11-05 13:48:16 +00:00
b8dc8a49a9 Retain successful response read failures 2026-11-05 13:46:56 +00:00
cd373ee46b Preserve owning categories in public errors 2026-11-05 13:45:00 +00:00
e881bbdcf4 Track the operation owning internal errors 2026-11-05 13:42:56 +00:00
106c808442 Use ConfigKit mapping values directly 2026-11-05 13:41:01 +00:00
4ec35f5ca6 Clarify local Go toolchain selection 2026-11-05 13:39:31 +00:00
6a1a32ae49 Plan the audit remediation sprint 2026-11-05 13:36:46 +00:00
e85ea2db35 Consolidate assessment recommendations into remediation roadmap 2026-11-05 13:15:09 +00:00
3ef2a9e18e Complete the remediation planning assessment 2026-11-05 12:48:29 +00:00
a85a6788b4 Record the first six remediation assessments 2026-11-05 12:38:10 +00:00
21fea46fb3 Plan the audit remediation assessment 2026-11-05 12:32:34 +00:00
62bf82a56b Add audit remediation roadmap 2026-11-05 04:37:26 +00:00
bd3dfa992c Record legacy capacity wrapper retirement decision 2026-11-05 04:31:14 +00:00
425f53a62e Record independent Go toolchain and consumer minimum policy 2026-11-05 04:28:41 +00:00
fcc94eb9e9 Record direct ConfigKit value handoff decision 2026-11-05 04:21:13 +00:00
95730a3c11 Record authentication response budget decision 2026-11-05 04:06:23 +00:00
c542b81666 Record self-contained schema decision and future bundling idea 2026-11-05 04:03:38 +00:00
e8e112186e Record ArtifactReader ownership decision 2026-11-05 03:48:04 +00:00
aa08d46ef6 Clarify audit remediation implementation cautions 2026-11-04 18:08:53 +00:00
668d09997a Consolidate audit findings and record open decisions 2026-11-04 18:05:43 +00:00
69203dcb12 Record dead code audit findings 2026-11-04 16:30:56 +00:00
3b760310a6 Record package boundary audit findings 2026-11-04 16:17:03 +00:00
b39460a507 Record behavioral equivalence audit findings 2026-11-04 16:06:17 +00:00
5177db0611 Record API and interface contract audit findings 2026-11-04 15:29:58 +00:00
31f1e6a89d Record test coverage and effectiveness audit findings 2026-11-04 15:11:58 +00:00
f7085d76f6 Record security audit findings 2026-11-04 14:50:24 +00:00
cf0bfd4d76 Record error handling audit findings 2026-11-04 14:12:57 +00:00
6b1323d1fe Record resource management audit findings 2026-11-04 14:02:50 +00:00
fb87fabc2e Record complexity and maintainability audit findings 2026-11-04 13:27:01 +00:00
5adc1c1e64 Record algorithmic efficiency audit findings 2026-11-04 13:18:32 +00:00
0d27252999 Record functional correctness audit findings 2026-11-04 13:12:28 +00:00
118 changed files with 3498 additions and 2788 deletions

View File

@@ -0,0 +1,43 @@
package promptkit_test
import (
"context"
"fmt"
"sync"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type independentArtifactReader struct{}
func (independentArtifactReader) Read(_ context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
return &promptkit.Artifact{Body: []byte(ref.Body), Hash: ref.Body}, nil
}
func TestConcurrentPreparationWithTransferredArtifacts(t *testing.T) {
input := configkit.Input{Source: configkit.Mapping("ownership", map[string]any{
"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.invalid/v1", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "inputs": []any{map[string]any{"name": "document", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": `{{input "document"}}`}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithArtifactReader(independentArtifactReader{}))
if err != nil {
t.Fatal(err)
}
var group sync.WaitGroup
for i := range 24 {
group.Go(func() {
value := fmt.Sprintf("body-%d", i)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"document": promptkit.Inline(value)}})
if err != nil {
t.Errorf("prepare: %v", err)
return
}
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != value || prepared.InputHashes["document"] != value {
t.Errorf("prepared = %+v", prepared)
}
})
}
group.Wait()
}

View File

@@ -78,7 +78,8 @@ type AuthenticationProbeConfig struct {
// It encodes fields without validating them; NewEngine validates the final // It encodes fields without validating them; NewEngine validates the final
// effective configuration after all selected source layers are combined. // effective configuration after all selected source layers are combined.
// Mapping captures a copy of supported input values at creation; unsupported // Mapping captures a copy of supported input values at creation; unsupported
// values fail when the source is loaded. // values fail when the source is loaded. ExtraParams must use ConfigKit Mapping
// input forms; callers must explicitly convert numbers or containers as needed.
func BackendSource(label string, backends []BackendConfig) configkit.Source { func BackendSource(label string, backends []BackendConfig) configkit.Source {
definitions := make([]any, 0, len(backends)) definitions := make([]any, 0, len(backends))
for _, config := range backends { for _, config := range backends {
@@ -96,7 +97,7 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
fields["queue_capacity"] = *config.QueueCapacity fields["queue_capacity"] = *config.QueueCapacity
} }
if config.ExtraParams != nil { if config.ExtraParams != nil {
fields["extra_params"] = providerMapForMapping(config.ExtraParams) fields["extra_params"] = config.ExtraParams
} }
if config.SmokeTests != nil { if config.SmokeTests != nil {
smokeTests := make(map[string]any) smokeTests := make(map[string]any)

View File

@@ -2,8 +2,8 @@ package promptkit_test
import ( import (
"context" "context"
"fmt" "reflect"
"strings" "slices"
"testing" "testing"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
@@ -58,11 +58,10 @@ func TestBundledCompositionReplacementAndInheritance(t *testing.T) {
if target := inspect("openrouter/profile"); target.BackendID != "openrouter/local" || target.Model != "collection-model" { if target := inspect("openrouter/profile"); target.BackendID != "openrouter/local" || target.Model != "collection-model" {
t.Fatalf("collection = %+v", target) t.Fatalf("collection = %+v", target)
} }
for _, tier := range []string{"low", "medium", "high", "xhigh", "max"} { target := inspect("auto-low")
target := inspect("auto-" + tier) wantPlugins := map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": "low"}}}
if target.BackendID != "other" || target.Model != "other-model" || !strings.Contains(fmt.Sprint(target.ExtraParams), "cost_tier:"+tier) || !strings.Contains(fmt.Sprint(target.ExtraParams), "id:auto-router") { if target.BackendID != "other" || target.Model != "other-model" || !reflect.DeepEqual(target.ExtraParams, wantPlugins) {
t.Errorf("tier %s = %+v", tier, target) t.Errorf("inherited tier = %+v", target)
}
} }
} }
@@ -70,11 +69,11 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
name, mode, id string name, mode, id string
wantErr bool wantErr bool
count int hasAuto bool
}{ }{
{"duplicate append", "append", "auto", true, 0}, {"duplicate append", "append", "auto", true, false},
{"distinct append", "append", "custom", false, 18}, {"distinct append", "append", "custom", false, true},
{"replace", "replace", "custom", false, 1}, {"replace", "replace", "custom", false, false},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{ input := configkit.Input{Source: configkit.Mapping("sequence", map[string]any{
@@ -92,7 +91,7 @@ func TestBundledCompositionSequenceModes(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
profiles, err := engine.ListProfiles(context.Background()) profiles, err := engine.ListProfiles(context.Background())
if err != nil || len(profiles) != test.count { if err != nil || !slices.Contains(profiles, "custom") || slices.Contains(profiles, "auto") != test.hasAuto {
t.Fatalf("profiles = %v, %v", profiles, err) t.Fatalf("profiles = %v, %v", profiles, err)
} }
}) })

View File

@@ -51,8 +51,7 @@ func TestBundledProfilePreparationAndExecution(t *testing.T) {
func TestBundledProfileHTTPPayloads(t *testing.T) { func TestBundledProfileHTTPPayloads(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "synthetic-key") t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
for _, test := range []struct{ id, model, tier string }{ for _, test := range []struct{ id, model, tier string }{
{"auto-low", "openrouter/auto", "low"}, {"auto-medium", "openrouter/auto", "medium"}, {"auto-high", "openrouter/auto", "high"}, {"auto-xhigh", "openrouter/auto", "xhigh"}, {"auto-max", "openrouter/auto", "max"}, {"auto-low", "openrouter/auto", "low"}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
{"auto", "openrouter/auto", ""}, {"free", "openrouter/free", ""}, {"gpt-luna-latest", "~openai/gpt-luna-latest", ""},
} { } {
t.Run(test.id, func(t *testing.T) { t.Run(test.id, func(t *testing.T) {
var body map[string]any var body map[string]any

View File

@@ -51,23 +51,16 @@ func TestBundledOpenRouterOptionsAndInventory(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
want := 0 if (len(profiles) > 0) != test.enabled || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
if test.enabled {
want = 17
}
if len(profiles) != want || len(backends) != boolCount(test.enabled) || len(prompts) != 0 || len(collections) != 0 {
t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections) t.Fatalf("inventories: %v %v %v %v", profiles, backends, prompts, collections)
} }
if test.enabled { if test.enabled {
wantProfiles := []string{"auto", "auto-high", "auto-low", "auto-max", "auto-medium", "auto-xhigh", "claude-fable-latest", "claude-haiku-latest", "claude-opus-latest", "claude-sonnet-latest", "free", "gemini-flash-latest", "gemini-pro-latest", "gpt-astra-latest", "gpt-luna-latest", "gpt-sol-latest", "gpt-terra-latest"} if !reflect.DeepEqual(backends, []string{"openrouter"}) {
if !reflect.DeepEqual(profiles, wantProfiles) || !reflect.DeepEqual(backends, []string{"openrouter"}) {
t.Fatalf("bundle inventory: %v %v", profiles, backends) t.Fatalf("bundle inventory: %v %v", profiles, backends)
} }
for id, model := range map[string]string{"auto": "openrouter/auto", "auto-max": "openrouter/auto", "gpt-luna-latest": "~openai/gpt-luna-latest", "free": "openrouter/free"} { profile, err := engine.InspectProfile(context.Background(), "auto")
profile, err := engine.InspectProfile(context.Background(), id) if err != nil || profile.EffectiveModelParams.BackendID != "openrouter" {
if err != nil || profile.EffectiveModelParams.Model != model || profile.EffectiveModelParams.BackendID != "openrouter" { t.Fatalf("bundle profile = %+v, %v", profile, err)
t.Fatalf("profile %s = %+v, %v", id, profile, err)
}
} }
view, err := engine.InspectConfiguration() view, err := engine.InspectConfiguration()
if err != nil { if err != nil {

View File

@@ -3,6 +3,7 @@ package promptkit_test
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"io" "io"
"net/http" "net/http"
"reflect" "reflect"
@@ -13,6 +14,45 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
type cancelingAuthenticationBody struct {
remaining int
cancel context.CancelFunc
closed bool
}
func (b *cancelingAuthenticationBody) Read(p []byte) (int, error) {
n := min(len(p), b.remaining)
for i := range p[:n] {
p[i] = 'x'
}
b.remaining -= n
if b.remaining == 0 {
b.cancel()
return n, context.Canceled
}
return n, nil
}
func (b *cancelingAuthenticationBody) Close() error { b.closed = true; return nil }
func TestAuthenticationReadCancellationWinsOverOverflow(t *testing.T) {
for _, size := range []int{1024, (1 << 20) + 1} {
ctx, cancel := context.WithCancel(context.Background())
body := &cancelingAuthenticationBody{remaining: size, cancel: cancel}
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: body}, nil
})}})
if err != nil {
cancel()
t.Fatal(err)
}
report, err := engine.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter"})
cancel()
if !errors.Is(err, context.Canceled) || report == nil || report.Completed || report.Failure != promptkit.ExecutionFailureCanceled || report.Facts.CredentialsAccepted != nil || report.Facts.ResponseReceived == nil || !*report.Facts.ResponseReceived || !body.closed {
t.Fatalf("size %d: report=%+v error=%v closed=%v", size, report, err, body.closed)
}
}
}
func TestBundledSmokeHTTPDefaults(t *testing.T) { func TestBundledSmokeHTTPDefaults(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "synthetic-key") t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
var urls []string var urls []string
@@ -125,3 +165,17 @@ func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
} }
} }
} }
func TestBundledSmokeAuthenticationResponseOverflow(t *testing.T) {
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(strings.Repeat("x", (1<<20)+1)))}, nil
})
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
if err != nil {
t.Fatal(err)
}
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "openrouter", APIKey: "private-key-marker"})
if err != nil || report == nil || !report.Completed || report.Reason != promptkit.SmokeTestReasonAuthenticationResponseLimitExceeded || report.Failure != promptkit.ExecutionFailureOther || report.Facts.CredentialsAccepted != nil {
t.Fatalf("overflow = %+v, %v", report, err)
}
}

View File

@@ -2,7 +2,6 @@ package promptkit
import ( import (
"fmt" "fmt"
"reflect"
"unicode/utf8" "unicode/utf8"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
@@ -395,88 +394,10 @@ func copyAnyMap(src map[string]any) map[string]any {
if src == nil { if src == nil {
return nil return nil
} }
out := make(map[string]any, len(src)) return jsonvalue.CloneTrusted(src).(map[string]any)
for k, v := range src {
out[k] = copyAny(v)
}
return out
} }
func copyAny(value any) any { func copyAny(value any) any { return jsonvalue.CloneTrusted(value) }
if value == nil {
return nil
}
switch v := value.(type) {
case map[string]any:
return copyAnyMap(v)
case []any:
out := make([]any, len(v))
for i, item := range v {
out[i] = copyAny(item)
}
return out
case []string:
return copyStringSlice(v)
case []byte:
return copyBytes(v)
default:
return copyReflectValue(reflect.ValueOf(value)).Interface()
}
}
func copyReflectValue(value reflect.Value) reflect.Value {
if !value.IsValid() {
return value
}
switch value.Kind() {
case reflect.Interface:
if value.IsNil() {
return reflect.Zero(value.Type())
}
copied := copyReflectValue(value.Elem())
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
return copied
}
out := reflect.New(value.Type()).Elem()
out.Set(copied)
return out
case reflect.Pointer:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.New(value.Type().Elem())
out.Elem().Set(copyReflectValue(value.Elem()))
return out
case reflect.Map:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeMapWithSize(value.Type(), value.Len())
iter := value.MapRange()
for iter.Next() {
out.SetMapIndex(copyReflectValue(iter.Key()), copyReflectValue(iter.Value()))
}
return out
case reflect.Slice:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(copyReflectValue(value.Index(i)))
}
return out
case reflect.Array:
out := reflect.New(value.Type()).Elem()
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(copyReflectValue(value.Index(i)))
}
return out
default:
return value
}
}
func copyStringSlice(src []string) []string { func copyStringSlice(src []string) []string {
if src == nil { if src == nil {

View File

@@ -139,8 +139,10 @@ and diffs expose resolved values, recorded normalization and provenance.
can apply further overrides, so the views are not promised to be identical. can apply further overrides, so the views are not promised to be identical.
Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding Remove `PromptVersion` and `PromptIdentity.Version` uses and the corresponding
`prompt_version` JSON members. Prompt hashes are opaque equality values and may `prompt_version` JSON members. Prompt definition hashes are opaque equality
change when the removed field no longer participates in hashing. values and may also change because redundant captured fields were removed.
Invalidate persisted caches or comparisons that span this change; hashes from
inspection and preparation of the same captured definition still agree.
## Identity Mapping ## Identity Mapping

View File

@@ -178,7 +178,12 @@ input := configkit.Input{Source: document}
``` ```
`ProfileSource` and `BackendSource` are smaller typed conveniences when null `ProfileSource` and `BackendSource` are smaller typed conveniences when null
is not needed. Backend and profile definitions are supplied by the caller; is not needed. Their provider maps use ConfigKit Mapping input forms: built-in
scalars, `[]any`, and `map[string]any` recursively. Convert `json.Number` or
typed containers explicitly to an appropriate supported value, choosing numeric
precision for your application. Mapping captures supported values immediately;
unsupported or cyclic values are reported when loaded. Runtime override values
follow their separate rules. Backend and profile definitions are supplied by the caller;
there is no registration-only backend namespace. there is no registration-only backend namespace.
When configuration or message files change, build and validate a new engine, When configuration or message files change, build and validate a new engine,
@@ -714,18 +719,19 @@ if err != nil {
} }
target := inspection.EffectiveModelParams target := inspection.EffectiveModelParams
if inspection.APIKeyRequired {
// Arrange a usable direct or configured environment credential.
}
if target.APIKeyEnv != "" { if target.APIKeyEnv != "" {
// This is a configured optional environment lookup source. // This is the configured environment lookup source, not its value.
} else if inspection.APIKeyRequired {
// Arrange a direct credential before later execution.
} }
``` ```
Use this configuration-time boundary when only the profile and its target need Use this configuration-time boundary when only the profile and its target need
checking. Use `Prepare` when the application also needs prompt, input, schema, checking. Use `Prepare` when the application also needs prompt, input, schema,
or rendering work; use prepared execution when that work must remain tied to a or rendering work; use prepared execution when that work must remain tied to a
later execution. A reported `APIKeyEnv` is a configured optional source, while later execution. A reported `APIKeyEnv` identifies a configured source, while
`APIKeyRequired` is the explicit local requirement. The `APIKeyRequired` is the explicit requirement; both may be present. The
[credential format reference](../formats.md#credentials) and the method's [credential format reference](../formats.md#credentials) and the method's
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error [GoDoc](../../engine.go) own the exact precedence, timing, result, and error
contracts. contracts.
@@ -879,6 +885,12 @@ secret-handling policies.
## Extension Interfaces ## Extension Interfaces
JSON Schema validation and native structured output use a self-contained root
document. Internal references are supported; references to other files or
network resources fail during preparation. `WithSchemaFile` selects one root
file, while `WithSchemaFS` and `SchemaDir` select a root source. Prepared
executions retain their captured schema even if the source later changes.
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
behavior does not fit the application. Their GoDoc defines concurrent use, behavior does not fit the application. Their GoDoc defines concurrent use,
context handling, ownership of copied values, nil responses, and preservation context handling, ownership of copied values, nil responses, and preservation
@@ -886,6 +898,11 @@ of collaborator errors. Implementations must honor cancellation, safely manage
copies they retain, avoid unsafe logging of content or credentials, and enforce copies they retain, avoid unsafe logging of content or credentials, and enforce
the application policy that motivated the injection. the application policy that motivated the injection.
A successful `ArtifactReader.Read` transfers the returned artifact and mutable
body storage to Promptkit. Readers that previously reused a returned buffer
must instead return independent storage and stop mutating it after return;
private scratch buffers may still be reused when the returned bytes are copied.
## Handle Errors ## Handle Errors
Use `errors.Is` with the Use `errors.Is` with the

View File

@@ -58,6 +58,14 @@ exist.
## Maintainer Validation ## Maintainer Validation
Use the newest Go installation already available locally for ordinary
development checks. Put its `bin` directory first on `PATH` and set
`GOTOOLCHAIN=local` to keep that toolchain selected. For repository-isolated
checks, set `GOWORK=off`; consumers may use their own workspaces. The selected
validation patch does not by itself change the module's consumer minimum. See
the [release toolchain policy](policy/release.md#consumer-minimum-and-release-toolchain)
and [release procedure](release.md#prepare-the-candidate) for release selection.
For ordinary work, select validation proportionate to the change. Do not repeat For ordinary work, select validation proportionate to the change. Do not repeat
successful checks for unchanged content merely to create a commit. During successful checks for unchanged content merely to create a commit. During
development, run relevant Go tests and vet, format changed Go files, and execute development, run relevant Go tests and vet, format changed Go files, and execute

View File

@@ -418,11 +418,17 @@ generic inheritance and layer mechanics, consult
## Schemas ## Schemas
Schemas are JSON documents selected by a prompt or request `schema_path`. Schemas are JSON documents selected by a prompt or request `schema_path`.
Directory and `fs.FS` schema paths resolve within the configured root; Directory and `fs.FS` schema paths resolve within the configured root. Each
referenced nested schemas resolve relative to the owning schema document. schema must be self-contained: in-document references, IDs, and anchors are
supported, but references requiring another file or network resource fail.
References must also work without the source filename as a base; use fragment
references or explicit schema IDs rather than relying on that filename.
`WithSchemaFile` exposes one schema under its base name. An unreadable, `WithSchemaFile` exposes one schema under its base name. An unreadable,
invalid, or unresolvable schema causes an operational validation error. invalid, or unresolvable schema causes an operational validation error.
Valid JSON that fails the schema instead yields a failed validation result. Valid JSON that fails the schema instead yields a failed validation result.
Schema documents are detached without applying the provider extra-parameter
copy limits; this does not change the schema decoder, compiler, or
self-contained-resource rules.
## Credentials ## Credentials

View File

@@ -16,14 +16,13 @@ Generation sends an HTTP `POST` with `Content-Type: application/json`.
Before the client is called, the engine resolves framework, backend, profile, Before the client is called, the engine resolves framework, backend, profile,
and request values into one execution target. Endpoint configuration is trimmed and request values into one execution target. Endpoint configuration is trimmed
and must be an absolute HTTP or HTTPS URL with a host and without user and must be an absolute HTTP or HTTPS URL with a host and without user
information, a query, or a fragment. A non-empty endpoint from the target information, a query, or a fragment. The target's endpoint is validated again
overrides the client's configured base URL. The final selected endpoint is before transport.
validated again before transport.
The completion URL is composed through parsed URL path operations. Nested base The completion URL is composed through parsed URL path operations. Nested base
paths are retained, repeated trailing slashes are normalized, and the result paths are retained, repeated trailing slashes are normalized, and the result
has exactly one appended `/chat/completions` suffix. Generation fails before has exactly one appended `/chat/completions` suffix. Generation fails before
sending when neither source supplies a valid endpoint. sending when the target does not supply a valid endpoint.
The target's backend ID is routing metadata for prepared values, results, and The target's backend ID is routing metadata for prepared values, results, and
injected clients. The built-in client does not derive the URL from that ID and injected clients. The built-in client does not derive the URL from that ID and
@@ -70,7 +69,9 @@ and numeric `usage`; unrelated fields and account limits are ignored. This is
only a minimal identifying subset of OpenRouter's documented only a minimal identifying subset of OpenRouter's documented
[current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits), [current-key response](https://github.com/OpenRouterTeam/docs/blob/main/api_reference/limits.mdx#checking-your-limits),
not validation of the full account document. Other statuses and unrecognized not validation of the full account document. Other statuses and unrecognized
200 responses do not establish acceptance. 200 responses do not establish acceptance. A 200 response exceeding the bounded
recognition allowance reports `authentication_response_limit_exceeded`, leaving
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
Diagnostic HTTP evidence describes the last request observed at Promptkit's Diagnostic HTTP evidence describes the last request observed at Promptkit's
RoundTripper boundary and any response associated with it. It includes the RoundTripper boundary and any response associated with it. It includes the
@@ -87,8 +88,7 @@ transport for HTTP-only diagnostics.
## Request Body ## Request Body
The request body always contains `model` and `messages`. The execution The request body always contains `model` and `messages`. The execution
target's model takes precedence over the client's configured model, and one target's model must be available.
must be available.
Each ordinary message contains its `role` and string `content`. A Each ordinary message contains its `role` and string `content`. A
cache-controlled message instead uses a text content block containing `type`, cache-controlled message instead uses a text content block containing `type`,
@@ -115,9 +115,10 @@ The client conditionally includes:
strict flag, and schema document. Plain JSON validation does not add an strict flag, and schema document. Plain JSON validation does not add an
object-only response constraint. object-only response constraint.
The engine resolves backend, profile, and request extra-parameter maps by Configured profile maps recursively overlay backend defaults, while a nonempty
whole-map replacement rather than key merging. The resulting effective map is runtime override replaces the complete configured map. The [format reference](../formats.md#defaults-and-overrides)
merged into the top-level body. Empty keys, values that cannot be encoded, and owns these resolution rules. The resulting effective map is merged into the
top-level body. Empty keys, values that cannot be encoded, and
collisions with these reserved fields are rejected before any provider call: collisions with these reserved fields are rejected before any provider call:
- `model` - `model`

View File

@@ -68,9 +68,8 @@ Preparation and validation never hold an active permit.
The manager's `Acquire` operation owns that active-permit acquisition and The manager's `Acquire` operation owns that active-permit acquisition and
returns an idempotent release function. The coordinator uses it around returns an idempotent release function. The coordinator uses it around
individual attempts. The earlier one-shot wrapper still delegates to the same individual attempts. There is one queue and one active limit per limited backend
operation for focused internal callers and tests, but engine assembly no pool.
longer uses it. There is one queue and one active limit per limited backend pool.
When all active permits are occupied, calls join a mutex-protected FIFO waiter When all active permits are occupied, calls join a mutex-protected FIFO waiter
list. Releasing a permit transfers it directly to the oldest remaining waiter list. Releasing a permit transfers it directly to the oldest remaining waiter
@@ -121,7 +120,7 @@ validation, bounded admission, idempotent release, context handling, and
unlimited admission. The unlimited admission. The
[client tests](../../internal/capacity/client_test.go) own peak enforcement, [client tests](../../internal/capacity/client_test.go) own peak enforcement,
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools, FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
unlimited calls, passthrough behavior, and panic release. unlimited calls, and panic release using test-only one-shot calls.
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
admission, lease lifetime, failure release, and shared initial/repair admission, lease lifetime, failure release, and shared initial/repair

View File

@@ -26,10 +26,9 @@ client uses its endpoint, credential metadata, generation fields, and extra
parameters. `BackendID` remains routing metadata parameters. `BackendID` remains routing metadata
for the generation boundary and is not mapped into the provider payload. for the generation boundary and is not mapped into the provider payload.
Construction trims and validates a nonempty configured base URL and clones any Construction clones any supplied `http.Client` so Promptkit can apply its
supplied `http.Client` so Promptkit can apply its timeout default without timeout default without mutating the caller's client. Generation preparation
mutating the caller's client. An empty configured base remains valid because a then:
resolved request target may supply the endpoint. Generation preparation then:
1. validates shared execution-setting invariants and the final selected base 1. validates shared execution-setting invariants and the final selected base
endpoint; endpoint;
@@ -42,6 +41,10 @@ resolved request target may supply the endpoint. Generation preparation then:
the complete-request failure remains a request failure; the complete-request failure remains a request failure;
6. resolves authentication and freezes the URL, body, and optional credential. 6. resolves authentication and freezes the URL, body, and optional credential.
Provider values are detached with trusted copying before encoding. Custom JSON
encoding methods and per-value encoding-failure diagnostics operate on those
private copies rather than retained prepared or diagnostic metadata.
Each execution of that prepared request creates a fresh HTTP request, body Each execution of that prepared request creates a fresh HTTP request, body
reader, and attempt timeout context. It closes the response body and returns reader, and attempt timeout context. It closes the response body and returns
the HTTP status, decoded response or error, reported token values and presence, the HTTP status, decoded response or error, reported token values and presence,
@@ -77,7 +80,8 @@ not retain headers, credentials, raw request or response bodies, or a redirect
trace. These are observations at Promptkit's RoundTripper boundary, not proof trace. These are observations at Promptkit's RoundTripper boundary, not proof
that bytes reached a peer or that a custom transport did no hidden dispatch. that bytes reached a peer or that a custom transport did no hidden dispatch.
Reachability and named authentication probes use the same configured built-in Reachability and named authentication probes use the same configured built-in
client but discard their response bodies after local recognition. The client. The OpenRouter adapter owns endpoint derivation, bounded account-response
recognition, and body closure; the smoke coordinator interprets its evidence. The
[integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics) [integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics)
owns their observable HTTP semantics. owns their observable HTTP semantics.
@@ -105,14 +109,13 @@ public ownership and redaction semantics belong to the
## Failure Categories ## Failure Categories
The package preserves distinct error identities for invalid client The package preserves distinct error identities for invalid generation
configuration, invalid generation requests, request execution failures, requests, request execution failures, non-success provider statuses, and
non-success provider statuses, and malformed successful responses. Provider malformed successful responses. Provider response bodies are never exposed in
response bodies are never exposed in raw form through non-success errors. raw form through non-success errors.
Invalid nonempty configured endpoints are configuration failures. A missing or A missing or invalid final selected endpoint is an invalid generation request
invalid final selected endpoint is an invalid generation request and is and is rejected before transport.
rejected before transport.
Authentication resolves a trimmed direct key before a trimmed configured Authentication resolves a trimmed direct key before a trimmed configured
environment value. Optional missing, empty, or whitespace-only sources do not environment value. Optional missing, empty, or whitespace-only sources do not

View File

@@ -10,6 +10,10 @@ lifecycle instrumentation spans the facade and runner boundaries.
Prepared execution starts only after its private payload is atomically claimed; Prepared execution starts only after its private payload is atomically claimed;
the facade receives that invocation's recorder privately and emits the terminal the facade receives that invocation's recorder privately and emits the terminal
event after public error mapping. Failed claims and preparation emit nothing. event after public error mapping. Failed claims and preparation emit nothing.
The public projection retains private operation-owner metadata separately from
inspectable child causes. Terminal classification uses that owner, preserving
context precedence and generation-specific transport/provider details rather
than mistaking a collaborator's nested lookup error for the parent's operation.
The facade masks inherited recorder and prepared-claim state when entering The facade masks inherited recorder and prepared-claim state when entering
preparation or execution. Delegated calls from an injected collaborator retain preparation or execution. Delegated calls from an injected collaborator retain

View File

@@ -23,7 +23,7 @@ contributor workflow and validation.
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) | | `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) | | `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) | | `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |

View File

@@ -42,10 +42,11 @@ Ordinary preparation selects the full prompt definition and construction-time
hash. It retains its early request-ID check before direct-session hash. It retains its early request-ID check before direct-session
normalization. Referenced message content and static template syntax were normalization. Referenced message content and static template syntax were
captured when the engine was built. captured when the engine was built.
Rendering uses those captured parses, cloning them per call to bind request
input helpers; it does not parse templates during execution.
`InspectPrompt` selects captured metadata through an optional repository `InspectPrompt` selects captured metadata directly from the repository. It
capability, falling back to the full definition and existing hash resolution returns owned metadata without
for internal repositories without it. It returns owned metadata without
rendering templates or touching profile, artifact, schema, renderer, validator, rendering templates or touching profile, artifact, schema, renderer, validator,
admission, or model collaborators. The root admission, or model collaborators. The root
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's [`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
@@ -57,8 +58,7 @@ snapshot by the selected prompt ID. The analyzer receives only
those identity fields rather than an execution definition. It analyzes the those identity fields rather than an execution definition. It analyzes the
snapshot with the supplied caller variables and normalized direct-session snapshot with the supplied caller variables and normalized direct-session
bypass; it does not resolve target readiness or enter the preparation bypass; it does not resolve target readiness or enter the preparation
pipeline. Missing snapshot capability is a prompt-load failure rather than a pipeline. An unavailable captured snapshot is a prompt-load failure. The root
source fallback. The root
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public [`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
operation's exact contract. operation's exact contract.
@@ -142,11 +142,10 @@ one execution-precedence and error-ordering implementation while allowing a
full backend pool to reject work before expensive schema, artifact, and full backend pool to reject work before expensive schema, artifact, and
rendering operations. `Prepare` discards the plan after returning its public rendering operations. `Prepare` discards the plan after returning its public
metadata. `Run` retains the plan through initial and repaired-output validation metadata. `Run` retains the plan through initial and repaired-output validation
and discards it when the operation ends. Prepared execution transfers the and discards it when the operation ends. Prepared execution retains one
completed preparation's owned message, input-hash, and copied-schema values immutable, redacted metadata snapshot shared by execution and internal details.
into its private payload, while deep-copying mutable provider extras and The facade detaches mutable public details. Claim completion or discard clears
creating an independent details snapshot. Claim completion or discard clears execution-only payload references without modifying the retained metadata.
the execution payload without removing the details snapshot.
The root facade's `Engine.ValidateRequest` uses the same complete preparation The root facade's `Engine.ValidateRequest` uses the same complete preparation
path as `Prepare` and discards the resulting internal prepared value without path as `Prepare` and discards the resulting internal prepared value without
@@ -176,7 +175,7 @@ encoding of that session, every role and content value, and cache-control
presence and values; its hexadecimal value is opaque. presence and values; its hexadecimal value is opaque.
The configuration snapshot is read-only after engine construction. Concurrent The configuration snapshot is read-only after engine construction. Concurrent
`Prepare` and `Run` calls copy their resolved targets and keep all invocation `Prepare` and `Run` calls use detached captured targets and keep invocation
state local. state local.
## Run Flow ## Run Flow

View File

@@ -85,6 +85,8 @@ returning and retains no source capability for public inspection.
Prompt materialization shares a lazy metadata snapshot across root and collection Prompt materialization shares a lazy metadata snapshot across root and collection
resources during construction, avoiding a full metadata copy per collection. resources during construction, avoiding a full metadata copy per collection.
Input metadata, message content and templates, and output contracts are captured
separately before the definition, inspection, and hash are assembled.
Normalization and defaults are recorded at individual resolved field paths, Normalization and defaults are recorded at individual resolved field paths,
including message roles and cache controls. List indexes identify fields for including message roles and cache controls. List indexes identify fields for
@@ -146,8 +148,8 @@ change rendering behavior.
ConfigKit resolves profile inheritance before the handoff. Within a collection, ConfigKit resolves profile inheritance before the handoff. Within a collection,
`extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference. `extends: {id: parent}` names a sibling local ID; it is not a cross-collection reference.
The configuration adapter validates final effective resource references and The configuration adapter validates final effective resource references and
field values, then builds target field values, then captures backend definitions and profile target snapshots
snapshots with backend defaults and presence/suppression state. A profile can with backend defaults and presence/suppression state. A profile can
remain partial until runtime overrides complete it. `InspectProfile` selects remain partial until runtime overrides complete it. `InspectProfile` selects
that captured target without prompt, artifact, schema, credential lookup, or that captured target without prompt, artifact, schema, credential lookup, or
provider work. Preparation applies runtime overrides and then checks target provider work. Preparation applies runtime overrides and then checks target
@@ -216,12 +218,14 @@ instead of first making a response-sized string copy.
Every preparation operation creates one operation-local validation plan. None, Every preparation operation creates one operation-local validation plan. None,
basic, and JSON modes retain the effective output contract without source basic, and JSON modes retain the effective output contract without source
access. JSON Schema mode loads the root document once, resolves and compiles access. JSON Schema mode loads the root document once, compiles its declared
each transitive reference, and retains the compiled validator. Schema compiler schema locations, and retains the compiled root validator. External resources
resources use canonical escaped file or private-scheme URLs; loaders decode are rejected without file or network loading; in-document references remain
their paths once and enforce the configured source boundary. The supported. Root resources use canonical escaped file or private-scheme URLs. The
provider-facing structured-output metadata uses the root document captured by provider-facing structured-output metadata uses the root document captured by
the same plan. the same plan. Its detached schema document is not revalidated against the
runtime provider-value copier's traversal limits. Smoke inference uses a
separate JSON-only prepared plan before checking the prepared schema plan.
Schema preparation and execution remain synchronous. Promptkit checks Schema preparation and execution remain synchronous. Promptkit checks
cancellation before and after source resolution, JSON decoding, compilation, cancellation before and after source resolution, JSON decoding, compilation,

80
docs/releases/v0.19.0.md Normal file
View File

@@ -0,0 +1,80 @@
# Promptkit v0.19.0
This release completes two rounds of audit remediation, correcting error and
diagnostic reporting and simplifying configuration, validation, and value
ownership. It includes intentional compatibility changes for custom artifact
readers, programmatic configuration sources, and schemas. The minimum supported
Go version remains 1.27.0; ConfigKit remains at v1.1.0.
## Compatibility And Upgrade
Consumers upgrading from v0.18.0 should review these changes:
- A successful `ArtifactReader.Read` transfers ownership of the returned
artifact and its mutable body storage to Promptkit. Custom readers must stop
mutating returned values through retained aliases. Readers that reuse scratch
buffers must return independent storage. Promptkit retains its defensive
body copy, but that copy does not make concurrent buffer reuse safe. See the
[extension guidance](../consumers/pkg-promptkit.md#extension-interfaces).
- Provider maps passed through `ProfileSource` and `BackendSource` now use
ConfigKit's native Mapping input forms directly: built-in scalars, `[]any`,
and `map[string]any` recursively. Convert `json.Number` and typed containers
explicitly to supported values, choosing numeric precision appropriate to
the application. The former recursive compatibility conversion is removed.
Runtime execution overrides retain their separate accepted-value behavior.
- Schemas used for local validation and native structured output must be
self-contained. Move externally referenced definitions into the root document
and use internal references or explicit schema IDs. References requiring
another file or network resource, or relying on the source filename as a base,
fail during preparation. Promptkit does not bundle external schemas. See the
[schema reference](../formats.md#schemas).
Prepared schemas no longer inherit the depth and node limits used when copying
runtime provider values. Schemas previously rejected solely by those incidental
limits may now be accepted; decoder/compiler checks and the self-contained
schema requirement still apply.
No configuration-format migration is required from v0.18.0. After addressing
any affected integrations:
```sh
go get go.maximumdirect.net/promptkit@v0.19.0
```
## Errors And Diagnostics
Public errors and terminal observations now preserve the category of the
operation that failed even when an injected collaborator returns an error from
another Promptkit operation. Underlying causes remain available through
`errors.Is` and `errors.As`, without repeatedly duplicating nested cause trees.
Built-in generation preserves response-body read failures separately from
malformed response content, and failures during request preparation are
distinguished from dispatched attempts.
OpenRouter authentication diagnostics bound response recognition to 1 MiB.
An oversized HTTP 200 response reports
`SmokeTestReasonAuthenticationResponseLimitExceeded`, leaving credential
acceptance unknown. Cancellation during the read remains an interruption.
Consumers that enumerate diagnostic reasons should handle the new reason;
see the [diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
## Internal Simplification And Ownership
Prompt lookup and rendering now consistently use captured configuration, and
smoke syntax and schema checks share the prepared output-validation lifecycle.
Unused routing defaults, alternate private workflows, and duplicate wire
representations have been removed.
Provider-setting resolution, inspection, prepared details, and smoke reports
avoid redundant validation and copying of already accepted values. Prepared
execution shares one private redacted metadata snapshot while preserving
independent caller-visible values. Provider values are detached before custom
JSON encoders run, protecting retained metadata from encoder-side mutation.
Requirement analysis, namespace-filtered listings, and repair-feedback
construction also avoid unnecessary traversal and temporary allocation. These
changes preserve existing request isolation and execution behavior without
introducing new performance guarantees.
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.19.0/docs/oss/attribution.md)

View File

@@ -38,8 +38,17 @@ consumers.
## Ideas ## Ideas
No unselected ideas are currently recorded here. Deliberately postponed ideas ### Schema Bundling
remain in the [deferred catalog](deferred.md).
Allow consumers to author reusable schemas across multiple files and package
them into a single self-contained schema for local validation and provider
submission. This could restore multi-file authoring convenience while retaining
the [self-contained schema requirement](../formats.md#schemas).
Bundling would need to preserve reference semantics, identifiers, anchors, and
recursion. A complete bundle would not guarantee that a provider supports every
schema feature. This is a possible future enhancement; implementation is not
currently approved.
## Entry Format ## Entry Format

View File

@@ -213,7 +213,8 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
// //
// Prompt schema_path values refer to the file's base name. path must name an // Prompt schema_path values refer to the file's base name. path must name an
// existing non-directory file when NewEngine applies the option. This option // existing non-directory file when NewEngine applies the option. This option
// replaces Config.SchemaDir and earlier schema-source options. // replaces Config.SchemaDir and earlier schema-source options. The selected
// schema must be self-contained; external resources are not loaded.
func WithSchemaFile(path string) Option { func WithSchemaFile(path string) Option {
return optionFunc(func(options *engineOptions) error { return optionFunc(func(options *engineOptions) error {
return options.internal.SetSchemaFile(path) return options.internal.SetSchemaFile(path)
@@ -311,8 +312,8 @@ func (e *Engine) InspectPrompt(
// override. BackendID is empty for an endpoint-only profile. // override. BackendID is empty for an endpoint-only profile.
// //
// APIKeyEnv in the returned target is an environment-variable name, never its // APIKeyEnv in the returned target is an environment-variable name, never its
// value. APIKeyRequired instead reports a direct credential requirement and is // value. APIKeyRequired reports that a credential is required; an explicit
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives // profile environment source can also be present. InspectProfile neither derives
// an ID from a prompt default_profile nor checks credential availability, so an // an ID from a prompt default_profile nor checks credential availability, so an
// absent or blank named environment variable is not an error. // absent or blank named environment variable is not an error.
// //

View File

@@ -13,10 +13,44 @@ import (
"go.maximumdirect.net/promptkit/internal/usecase" "go.maximumdirect.net/promptkit/internal/usecase"
) )
// publicOperationError keeps classification separate from inspectable causes.
// Cache wording so formatting nested operations does not repeatedly rebuild it.
type publicOperationError struct {
owner error
cause error
message string
}
func (e *publicOperationError) Error() string { return e.message }
func (e *publicOperationError) Unwrap() error { return e.cause }
func (e *publicOperationError) Is(target error) bool { return errors.Is(e.owner, target) }
func mapPublicError(err error) error { func mapPublicError(err error) error {
if err == nil { if err == nil {
return nil return nil
} }
var operation *usecase.OperationError
if errors.As(err, &operation) && operation != nil {
owner := publicErrorFor(operation.Category)
if owner != nil {
// Project the underlying specialized error separately, retaining both
// the outer operation and the original cause for Is/As callers.
mapped := mapPublicError(operation.Cause)
if errors.Is(operation.Cause, llm.ErrTransport) {
mapped = errors.Join(ErrTransport, mapped)
}
if errors.Is(operation.Cause, llm.ErrMalformedResponse) {
mapped = errors.Join(ErrMalformedResponse, mapped)
}
// The projection already retains ordinary causes; adding the original
// again would duplicate its entire subtree at each enclosing operation.
return &publicOperationError{owner: owner, cause: mapped, message: fmt.Sprintf("%v: %v", owner, mapped)}
}
}
return mapUnownedPublicError(err)
}
func mapUnownedPublicError(err error) error {
var internalCapacityError *usecase.CapacityError var internalCapacityError *usecase.CapacityError
if errors.As(err, &internalCapacityError) && internalCapacityError != nil && if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
strings.TrimSpace(internalCapacityError.BackendID) != "" { strings.TrimSpace(internalCapacityError.BackendID) != "" {

View File

@@ -80,3 +80,12 @@ func TestMapPublicErrorDoesNotTreatRequestEncodingAsTransport(t *testing.T) {
t.Fatalf("request encoding category = %v", err) t.Fatalf("request encoding category = %v", err)
} }
} }
func TestMapPublicErrorRetainsOwnerWithSpecializedCapacity(t *testing.T) {
internal := &usecase.CapacityError{BackendID: "limited"}
err := mapPublicError(&usecase.OperationError{Category: usecase.ErrLLMGenerate, Cause: internal})
var capacity *CapacityError
if !errors.Is(err, ErrLLMGenerate) || !errors.As(err, &capacity) || capacity.BackendID != "limited" || !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("mapped owned capacity = %v", err)
}
}

View File

@@ -3,8 +3,6 @@ package artifact
import ( import (
"bytes" "bytes"
"context" "context"
"crypto/sha256"
"fmt"
"mime" "mime"
"os" "os"
"path/filepath" "path/filepath"
@@ -153,9 +151,12 @@ func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef,
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) { if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want)) b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
} }
wantHash := fmt.Sprintf("%x", sha256.Sum256(want)) inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(want)})
if artifact.Hash != wantHash { if err != nil {
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash) b.Fatal(err)
}
if inline.Hash == "" || artifact.Hash != inline.Hash {
b.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
} }
} }

View File

@@ -3,9 +3,7 @@ package artifact
import ( import (
"bytes" "bytes"
"context" "context"
"crypto/sha256"
"errors" "errors"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -284,9 +282,12 @@ func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" { if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType) t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
} }
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body)) inline, err := NewCompositeReader().Read(context.Background(), domain.ArtifactRef{Type: domain.ArtifactRefInline, Body: string(tc.body)})
if artifact.Hash != wantHash { if err != nil {
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash) t.Fatal(err)
}
if inline.Hash == "" || artifact.Hash != inline.Hash {
t.Fatalf("file and inline hashes differ: %q/%q", artifact.Hash, inline.Hash)
} }
}) })
} }

View File

@@ -27,17 +27,13 @@ type Registry struct {
// NewEffectiveRegistry stores already-validated effective backend definitions // NewEffectiveRegistry stores already-validated effective backend definitions
// under their exact configuration keys. Configuration validation and // under their exact configuration keys. Configuration validation and
// normalization precede this ownership boundary. // normalization precede this ownership boundary.
func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, error) { func NewEffectiveRegistry(definitions map[string]domain.Backend) *Registry {
registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))} registry := &Registry{backends: make(map[string]domain.Backend, len(definitions))}
for id, definition := range definitions { for id, definition := range definitions {
definition.ID = id definition.ID = id
cloned, err := domain.CloneBackend(definition) registry.backends[id] = domain.CloneBackend(definition)
if err != nil {
return nil, fmt.Errorf("backend %q: %w", id, err)
}
registry.backends[id] = cloned
} }
return registry, nil return registry
} }
// GetBackend returns a defensive copy of the backend registered with id. // GetBackend returns a defensive copy of the backend registered with id.
@@ -49,11 +45,7 @@ func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if !ok { if !ok {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id) return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
} }
cloned, err := domain.CloneBackend(definition) return domain.CloneBackend(definition), nil
if err != nil {
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
}
return cloned, nil
} }
// CapacityPolicies returns a copy of the normalized policies for limited // CapacityPolicies returns a copy of the normalized policies for limited

View File

@@ -22,16 +22,13 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
Kind: &authenticationKind, Endpoint: &authenticationEndpoint, Kind: &authenticationKind, Endpoint: &authenticationEndpoint,
}, },
} }
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{ registry := backend.NewEffectiveRegistry(map[string]domain.Backend{
" local.v1 ": { " local.v1 ": {
Endpoint: "https://backend.example/v1", ExtraParams: params, Endpoint: "https://backend.example/v1", ExtraParams: params,
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true, ConcurrencyLimit: 2, QueueCapacity: 3,
SmokeTests: smokeTests, SmokeTests: smokeTests,
}, },
}) })
if err != nil {
t.Fatal(err)
}
params["nested"].(map[string]any)["enabled"] = false params["nested"].(map[string]any)["enabled"] = false
defaultProfile = "mutated-profile" defaultProfile = "mutated-profile"
reachabilityEndpoint = "https://mutated.example/ready" reachabilityEndpoint = "https://mutated.example/ready"
@@ -79,10 +76,7 @@ func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
} }
func TestEffectiveRegistryMissingBackend(t *testing.T) { func TestEffectiveRegistryMissingBackend(t *testing.T) {
registry, err := backend.NewEffectiveRegistry(nil) registry := backend.NewEffectiveRegistry(nil)
if err != nil {
t.Fatal(err)
}
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) { if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("missing backend error = %v", err) t.Fatalf("missing backend error = %v", err)
} }

View File

@@ -1,40 +0,0 @@
package capacity
import (
"context"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/llm"
)
type client struct {
manager *Manager
next llm.Client
}
// NewClient wraps next with configured active-generation limits. A nil manager
// leaves next unchanged.
func NewClient(manager *Manager, next llm.Client) llm.Client {
if manager == nil {
return next
}
return &client{
manager: manager,
next: next,
}
}
func (c *client) Generate(
ctx context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
if c.manager.getPool(req.Target.BackendID) == nil {
return c.next.Generate(ctx, req)
}
release, err := c.manager.Acquire(ctx, req.Target.BackendID)
if err != nil {
return nil, err
}
defer release()
return c.next.Generate(ctx, req)
}

View File

@@ -3,7 +3,6 @@ package capacity
import ( import (
"context" "context"
"errors" "errors"
"reflect"
"runtime" "runtime"
"sync" "sync"
"sync/atomic" "sync/atomic"
@@ -24,6 +23,18 @@ type clientFunc func(
domain.GenerateRequest, domain.GenerateRequest,
) (*domain.GenerateResponse, error) ) (*domain.GenerateResponse, error)
// newTestClient exercises one acquisition around a controlled call.
func newTestClient(manager *Manager, next llm.Client) llm.Client {
return clientFunc(func(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
release, err := manager.Acquire(ctx, req.Target.BackendID)
if err != nil {
return nil, err
}
defer release()
return next.Generate(ctx, req)
})
}
func (f clientFunc) Generate( func (f clientFunc) Generate(
ctx context.Context, ctx context.Context,
req domain.GenerateRequest, req domain.GenerateRequest,
@@ -167,7 +178,7 @@ func TestClientLimitsPeakConcurrencyAndServesWaitersFIFO(t *testing.T) {
"second": secondRelease, "second": secondRelease,
"third": thirdRelease, "third": thirdRelease,
}) })
client := NewClient(manager, next) client := newTestClient(manager, next)
first := generateAsync(client, context.Background(), "limited", "first") first := generateAsync(client, context.Background(), "limited", "first")
if got := receiveStarted(t, next.started); got != "first" { if got := receiveStarted(t, next.started); got != "first" {
@@ -212,7 +223,7 @@ func TestClientPeakConcurrencyDoesNotExceedConfiguredLimit(t *testing.T) {
releases[string(rune('a'+i))] = gate releases[string(rune('a'+i))] = gate
} }
next := newBlockingClient(releases) next := newBlockingClient(releases)
client := NewClient(manager, next) client := newTestClient(manager, next)
results := make([]<-chan generateResult, 0, len(releases)) results := make([]<-chan generateResult, 0, len(releases))
for id := range releases { for id := range releases {
@@ -257,7 +268,7 @@ func TestClientRemovesCanceledWaiters(t *testing.T) {
"three": make(chan struct{}), "three": make(chan struct{}),
} }
next := newBlockingClient(releases) next := newBlockingClient(releases)
client := NewClient(manager, next) client := newTestClient(manager, next)
holder := generateAsync(client, context.Background(), "limited", "holder") holder := generateAsync(client, context.Background(), "limited", "holder")
if got := receiveStarted(t, next.started); got != "holder" { if got := receiveStarted(t, next.started); got != "holder" {
@@ -321,7 +332,7 @@ func TestClientGrantCancellationRaceDoesNotLeakPermit(t *testing.T) {
} }
return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil return &domain.GenerateResponse{Content: req.Prompt.SessionID}, nil
}) })
client := NewClient(manager, next) client := newTestClient(manager, next)
holder := generateAsync(client, context.Background(), "limited", "holder") holder := generateAsync(client, context.Background(), "limited", "holder")
waitForActiveCount(t, manager, "limited", 1) waitForActiveCount(t, manager, "limited", 1)
@@ -394,7 +405,7 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
"alpha": alphaRelease, "alpha": alphaRelease,
"beta": betaRelease, "beta": betaRelease,
}) })
client := NewClient(manager, next) client := newTestClient(manager, next)
alpha := generateAsync(client, context.Background(), "alpha", "alpha") alpha := generateAsync(client, context.Background(), "alpha", "alpha")
beta := generateAsync(client, context.Background(), "beta", "beta") beta := generateAsync(client, context.Background(), "beta", "beta")
@@ -423,63 +434,6 @@ func TestClientUsesIndependentPoolsAndUnlimitedFastPaths(t *testing.T) {
t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err) t.Fatalf("unlimited backend %q response=(%#v, %v)", backendID, response, err)
} }
} }
if got := NewClient(nil, next); got != next {
t.Fatal("nil manager did not return the wrapped client unchanged")
}
}
func TestClientPreservesRequestsResponsesAndErrors(t *testing.T) {
manager := newTestManager(t, map[string]domain.BackendCapacityPolicy{
"limited": {ConcurrencyLimit: 1},
})
request := domain.GenerateRequest{
Prompt: domain.RenderedPrompt{
SessionID: "session",
Messages: []domain.RenderedMessage{
{Role: "user", Content: "content"},
},
},
Target: domain.ExecutionTarget{
BackendID: "limited",
Model: "model",
ExtraParams: map[string]any{"key": "value"},
},
}
response := &domain.GenerateResponse{
Content: "output",
Usage: domain.TokenUsage{TotalTokens: 7},
}
collaboratorErr := errors.New("collaborator failure")
tests := []struct {
name string
response *domain.GenerateResponse
err error
}{
{name: "successful response", response: response},
{name: "nil response"},
{name: "collaborator error", response: response, err: collaboratorErr},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var captured domain.GenerateRequest
next := clientFunc(func(
_ context.Context,
req domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
captured = req
return tc.response, tc.err
})
gotResponse, gotErr := NewClient(manager, next).Generate(context.Background(), request)
if !reflect.DeepEqual(captured, request) {
t.Fatalf("request changed: %#v", captured)
}
if gotResponse != tc.response || gotErr != tc.err {
t.Fatalf("response=(%p, %v), want (%p, %v)",
gotResponse, gotErr, tc.response, tc.err)
}
})
}
} }
func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) { func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
@@ -496,7 +450,7 @@ func TestClientReleasesPermitDuringPanicUnwinding(t *testing.T) {
} }
return &domain.GenerateResponse{Content: "recovered"}, nil return &domain.GenerateResponse{Content: "recovered"}, nil
}) })
client := NewClient(manager, next) client := newTestClient(manager, next)
request := domain.GenerateRequest{ request := domain.GenerateRequest{
Target: domain.ExecutionTarget{BackendID: "limited"}, Target: domain.ExecutionTarget{BackendID: "limited"},
} }

View File

@@ -3,10 +3,12 @@ package configuration
import ( import (
"context" "context"
"fmt" "fmt"
"reflect"
"strings" "strings"
"testing" "testing"
"go.maximumdirect.net/configkit" "go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit/internal/defaults"
) )
func TestBundledOpenRouterUnderlay(t *testing.T) { func TestBundledOpenRouterUnderlay(t *testing.T) {
@@ -15,9 +17,12 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
backend, err := state.GetBackend("openrouter") backend, err := state.GetBackend("openrouter")
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" { if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
t.Fatalf("backend = %+v, %v", backend, err) t.Fatalf("backend = %+v, %v", backend, err)
} }
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
t.Fatalf("bundle inventories: backends=%v prompts=%v collections=%v", state.backendIDs, state.promptIDs, state.collections)
}
models := map[string]string{ models := map[string]string{
"gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest", "gpt-astra-latest": "~openai/gpt-astra-latest", "gpt-sol-latest": "~openai/gpt-sol-latest", "gpt-terra-latest": "~openai/gpt-terra-latest", "gpt-luna-latest": "~openai/gpt-luna-latest",
"claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest", "claude-fable-latest": "~anthropic/claude-fable-latest", "claude-opus-latest": "~anthropic/claude-opus-latest", "claude-sonnet-latest": "~anthropic/claude-sonnet-latest", "claude-haiku-latest": "~anthropic/claude-haiku-latest",
@@ -31,14 +36,16 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
} }
for id, model := range models { for id, model := range models {
profile, err := state.GetProfile(context.Background(), id) profile, err := state.GetProfile(context.Background(), id)
if err != nil || profile.ResolvedTarget.Model != model || profile.ResolvedTarget.BackendID != "openrouter" { if err != nil {
t.Fatalf("profile %s = %+v, %v", id, profile, err) t.Fatalf("profile %s = %+v, %v", id, profile, err)
} }
want := defaults.ExecutionTargetDefault()
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
if tier, ok := strings.CutPrefix(id, "auto-"); ok { if tier, ok := strings.CutPrefix(id, "auto-"); ok {
plugins := fmt.Sprint(profile.ResolvedTarget.ExtraParams) want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
if !strings.Contains(plugins, "cost_tier:"+tier) || !strings.Contains(plugins, "id:auto-router") { }
t.Fatalf("profile %s plugins = %+v", id, profile.ResolvedTarget.ExtraParams) if !reflect.DeepEqual(*profile.ResolvedTarget, want) {
} t.Fatalf("profile %s target = %+v, want %+v", id, *profile.ResolvedTarget, want)
} }
} }
if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") { if !strings.Contains(fmt.Sprint(state.result.Sources()), "Promptkit bundled OpenRouter") {

View File

@@ -2,36 +2,66 @@ package configuration
import ( import (
"context" "context"
"sort"
"strings"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
// IdentityFilter selects all identities, one namespace, or unqualified IDs.
type IdentityFilter struct {
All bool
Namespace string
}
func selectIdentities[T any](ctx context.Context, entries []T, filter IdentityFilter, id func(T) string) ([]T, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if filter.All {
result := append([]T(nil), entries...)
if err := ctx.Err(); err != nil {
return nil, err
}
return result, nil
}
if filter.Namespace != "" {
prefix := filter.Namespace + "/"
start := sort.Search(len(entries), func(i int) bool { return id(entries[i]) >= prefix })
end := start + sort.Search(len(entries)-start, func(i int) bool { return !strings.HasPrefix(id(entries[start+i]), prefix) })
result := append([]T(nil), entries[start:end]...)
if err := ctx.Err(); err != nil {
return nil, err
}
return result, nil
}
var result []T
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return nil, err
}
if !strings.Contains(id(entry), "/") {
result = append(result, entry)
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return result, nil
}
// CollectionIdentity is the captured metadata for one final collection entry. // CollectionIdentity is the captured metadata for one final collection entry.
type CollectionIdentity struct { type CollectionIdentity struct {
Namespace string Namespace string
ReleaseVersion *string ReleaseVersion *string
} }
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) { func (state *loaded) ListProfiles(ctx context.Context, filter IdentityFilter) ([]string, error) {
if err := ctx.Err(); err != nil { return selectIdentities(ctx, state.profileIDs, filter, func(id string) string { return id })
return nil, err
}
profiles := append([]string(nil), state.profileIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return profiles, nil
} }
func (state *loaded) ListBackends(ctx context.Context) ([]string, error) { func (state *loaded) ListBackends(ctx context.Context, filter IdentityFilter) ([]string, error) {
if err := ctx.Err(); err != nil { return selectIdentities(ctx, state.backendIDs, filter, func(id string) string { return id })
return nil, err
}
backends := append([]string(nil), state.backendIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return backends, nil
} }
func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) { func (state *loaded) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
@@ -72,13 +102,6 @@ func (state *loaded) ListCollections(ctx context.Context) ([]CollectionIdentity,
return collections, nil return collections, nil
} }
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) { func (state *loaded) ListPrompts(ctx context.Context, filter IdentityFilter) ([]domain.PromptIdentity, error) {
if err := ctx.Err(); err != nil { return selectIdentities(ctx, state.promptIDs, filter, func(entry domain.PromptIdentity) string { return entry.ID })
return nil, err
}
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return prompts, nil
} }

View File

@@ -23,7 +23,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
backends, err := state.ListBackends(context.Background()) backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -32,7 +32,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends) t.Fatalf("backend IDs = %#v, want %#v", backends, wantBackends)
} }
profiles, err := state.ListProfiles(context.Background()) profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -41,7 +41,7 @@ func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles) t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
} }
prompts, err := state.ListPrompts(context.Background()) prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -65,36 +65,36 @@ func TestLoadedListsAreIndependent(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
profiles, err := state.ListProfiles(context.Background()) profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
profiles = append(profiles, "caller profile") profiles = append(profiles, "caller profile")
profiles[0] = "changed profile" profiles[0] = "changed profile"
backends, err := state.ListBackends(context.Background()) backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
backends[0] = "changed backend" backends[0] = "changed backend"
prompts, err := state.ListPrompts(context.Background()) prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
prompts[0].ID = "changed prompt" prompts[0].ID = "changed prompt"
profiles, err = state.ListProfiles(context.Background()) profiles, err = state.ListProfiles(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !reflect.DeepEqual(profiles, []string{"profile"}) { if !reflect.DeepEqual(profiles, []string{"profile"}) {
t.Fatalf("caller mutation changed profile inventory: %#v", profiles) t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
} }
backends, err = state.ListBackends(context.Background()) backends, err = state.ListBackends(context.Background(), IdentityFilter{All: true})
if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) { if err != nil || !reflect.DeepEqual(backends, []string{"backend"}) {
t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err) t.Fatalf("caller mutation changed backend inventory: %#v, %v", backends, err)
} }
prompts, err = state.ListPrompts(context.Background()) prompts, err = state.ListPrompts(context.Background(), IdentityFilter{All: true})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -104,20 +104,66 @@ func TestLoadedListsAreIndependent(t *testing.T) {
} }
} }
func TestLoadedListsSelectNamespaces(t *testing.T) {
ids := []string{"a", "a-b/one", "a/one", "a/two", "b", "z/one"}
state := &loaded{profileIDs: append([]string(nil), ids...), backendIDs: append([]string(nil), ids...)}
for _, id := range ids {
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id})
}
cases := []struct {
name string
filter IdentityFilter
want []string
}{
{"all", IdentityFilter{All: true}, ids},
{"unqualified", IdentityFilter{}, []string{"a", "b"}},
{"namespace", IdentityFilter{Namespace: "a"}, []string{"a/one", "a/two"}},
{"prefix sharing", IdentityFilter{Namespace: "a-b"}, []string{"a-b/one"}},
{"missing", IdentityFilter{Namespace: "absent"}, nil},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
profiles, err := state.ListProfiles(context.Background(), tc.filter)
if err != nil || !reflect.DeepEqual(profiles, tc.want) {
t.Fatalf("profiles = %v, %v; want %v", profiles, err, tc.want)
}
backends, err := state.ListBackends(context.Background(), tc.filter)
if err != nil || !reflect.DeepEqual(backends, tc.want) {
t.Fatalf("backends = %v, %v; want %v", backends, err, tc.want)
}
prompts, err := state.ListPrompts(context.Background(), tc.filter)
if err != nil || len(prompts) != len(tc.want) {
t.Fatalf("prompts = %v, %v; want %v", prompts, err, tc.want)
}
for i := range prompts {
if prompts[i].ID != tc.want[i] {
t.Fatalf("prompt ID = %q; want %q", prompts[i].ID, tc.want[i])
}
}
if len(profiles) > 0 {
profiles[0], backends[0], prompts[0].ID = "changed", "changed", "changed"
}
})
}
if !reflect.DeepEqual(state.profileIDs, ids) || !reflect.DeepEqual(state.backendIDs, ids) || state.promptIDs[0].ID != ids[0] {
t.Fatal("caller mutation changed captured identities")
}
}
func TestLoadedListsEmptyInventory(t *testing.T) { func TestLoadedListsEmptyInventory(t *testing.T) {
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil) state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
profiles, err := state.ListProfiles(context.Background()) profiles, err := state.ListProfiles(context.Background(), IdentityFilter{All: true})
if err != nil || len(profiles) != 0 { if err != nil || len(profiles) != 0 {
t.Fatalf("empty profile listing = %#v, %v", profiles, err) t.Fatalf("empty profile listing = %#v, %v", profiles, err)
} }
backends, err := state.ListBackends(context.Background()) backends, err := state.ListBackends(context.Background(), IdentityFilter{All: true})
if err != nil || len(backends) != 0 { if err != nil || len(backends) != 0 {
t.Fatalf("empty backend listing = %#v, %v", backends, err) t.Fatalf("empty backend listing = %#v, %v", backends, err)
} }
prompts, err := state.ListPrompts(context.Background()) prompts, err := state.ListPrompts(context.Background(), IdentityFilter{All: true})
if err != nil || len(prompts) != 0 { if err != nil || len(prompts) != 0 {
t.Fatalf("empty prompt listing = %#v, %v", prompts, err) t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
} }
@@ -134,17 +180,19 @@ func TestLoadedListsHonorCanceledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
profiles, err := state.ListProfiles(ctx) for _, filter := range []IdentityFilter{{All: true}, {}, {Namespace: "example"}} {
if !errors.Is(err, context.Canceled) || len(profiles) != 0 { profiles, err := state.ListProfiles(ctx, filter)
t.Fatalf("canceled profile listing = %#v, %v", profiles, err) if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
} t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
backends, err := state.ListBackends(ctx) }
if !errors.Is(err, context.Canceled) || len(backends) != 0 { backends, err := state.ListBackends(ctx, filter)
t.Fatalf("canceled backend listing = %#v, %v", backends, err) if !errors.Is(err, context.Canceled) || len(backends) != 0 {
} t.Fatalf("canceled backend listing = %#v, %v", backends, err)
prompts, err := state.ListPrompts(ctx) }
if !errors.Is(err, context.Canceled) || len(prompts) != 0 { prompts, err := state.ListPrompts(ctx, filter)
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err) if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
}
} }
} }

View File

@@ -119,7 +119,6 @@ const (
type loaded struct { type loaded struct {
result *configkit.Result[document] result *configkit.Result[document]
backends map[string]map[string]fieldState
profiles map[string]map[string]fieldState profiles map[string]map[string]fieldState
promptMetadata map[string][]promptInputPresence promptMetadata map[string][]promptInputPresence
prompts map[string]*promptEntry prompts map[string]*promptEntry
@@ -212,7 +211,6 @@ func indexedProfiles(config *document) map[string]profileConfig {
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
var backendFields = []string{"endpoint", "api_key_env", "concurrency_limit", "queue_capacity", "extra_params", "smoke_tests"}
var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"} var profileFields = []string{"backend", "endpoint", "model", "temperature", "top_p", "max_tokens", "timeout_seconds", "service_tier", "reasoning_effort", "api_key_env", "api_key_required", "extra_params"}
// Load captures one immutable effective configuration for internal consumers. // Load captures one immutable effective configuration for internal consumers.
@@ -237,7 +235,6 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
Sensitive: secretProviderPath, Sensitive: secretProviderPath,
Resolve: func(config *document, resolution *configkit.Resolution) error { Resolve: func(config *document, resolution *configkit.Resolution) error {
for _, resources := range resourceSets(config) { for _, resources := range resourceSets(config) {
captureFields(resources.backends, func(v backendConfig) string { return v.ID }, resources.bases.backends, resources.namespace, backendFields, resolution, state.backends)
captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles) captureFields(resources.profiles, func(v profileConfig) string { return v.ID }, resources.bases.profiles, resources.namespace, profileFields, resolution, state.profiles)
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata) capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
} }
@@ -257,7 +254,6 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
// load performs one ConfigKit lifecycle over the selected source tiers. // load performs one ConfigKit lifecycle over the selected source tiers.
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) { func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
state := &loaded{ state := &loaded{
backends: make(map[string]map[string]fieldState),
profiles: make(map[string]map[string]fieldState), profiles: make(map[string]map[string]fieldState),
promptMetadata: make(map[string][]promptInputPresence), promptMetadata: make(map[string][]promptInputPresence),
} }
@@ -419,101 +415,110 @@ func validateEffective(config *document, validation *configkit.Validation) {
} }
for _, resources := range sets { for _, resources := range sets {
for index, backend := range resources.backends { for index, backend := range resources.backends {
base := resources.bases.backends.Index(index) validateBackendRecord(backend, resources.bases.backends.Index(index), validation)
validateBackendEndpoints(backend, base, validation)
if !domain.IsValidLocalResourceKey(backend.ID) {
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
}
if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required")
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
}
if backend.QueueCapacity != nil {
if *backend.QueueCapacity < 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
}
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
}
}
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
if smokeTests := backend.SmokeTests; smokeTests != nil {
smokePath := base.Key("smoke_tests")
if smokeTests.DefaultProfile != nil {
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
}
}
if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil {
validation.Error(kindPath, "authentication kind is required")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
}
}
}
} }
for index, profile := range resources.profiles { for index, profile := range resources.profiles {
base := resources.bases.profiles.Index(index) validateProfileRecord(profile, resources.bases.profiles.Index(index), backendIDs, validation)
if profile.Endpoint != nil && *profile.Endpoint != "" {
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
validation.Error(base.Key("endpoint"), err.Error())
}
}
if !domain.IsValidLocalResourceKey(profile.ID) {
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
}
if profile.Backend != nil && *profile.Backend != "" {
if !domain.IsValidResourceID(*profile.Backend) {
validation.Error(base.Key("backend"), "backend reference is malformed")
} else if _, exists := backendIDs[*profile.Backend]; !exists {
validation.Error(base.Key("backend"), "backend is not defined")
}
}
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
validation.Error(base.Key("model"), "model must not be blank")
}
if profile.Temperature != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
validation.Error(base.Key("temperature"), err.Error())
}
}
if profile.TopP != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
validation.Error(base.Key("top_p"), err.Error())
}
}
if profile.MaxTokens != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
validation.Error(base.Key("max_tokens"), err.Error())
}
}
if profile.TimeoutSeconds != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
validation.Error(base.Key("timeout_seconds"), err.Error())
}
}
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
} }
for index, prompt := range resources.prompts { for index, prompt := range resources.prompts {
base := resources.bases.prompts.Index(index) validatePromptRecord(prompt, resources.bases.prompts.Index(index), validation)
if !domain.IsValidLocalResourceKey(prompt.ID) { }
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash") }
} }
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
validation.Error(base.Key("default_profile"), "default profile reference is malformed") func validateBackendRecord(backend backendConfig, base configkit.Path, validation *configkit.Validation) {
validateBackendEndpoints(backend, base, validation)
if !domain.IsValidLocalResourceKey(backend.ID) {
validation.Error(base.Key("id"), "backend ID must be a nonblank local key without a slash")
}
if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required")
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
}
if backend.ConcurrencyLimit != nil && *backend.ConcurrencyLimit < 0 {
validation.Error(base.Key("concurrency_limit"), "concurrency limit must not be negative")
}
if backend.QueueCapacity != nil {
if *backend.QueueCapacity < 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity must not be negative")
}
if backend.ConcurrencyLimit == nil || *backend.ConcurrencyLimit <= 0 {
validation.Error(base.Key("queue_capacity"), "queue capacity requires a positive concurrency limit")
} else if *backend.QueueCapacity > int(^uint(0)>>1)-*backend.ConcurrencyLimit {
validation.Error(base.Key("queue_capacity"), "total capacity overflows int")
}
}
validateExtraParams(backend.ExtraParams, base.Key("extra_params"), true, validation)
if smokeTests := backend.SmokeTests; smokeTests != nil {
smokePath := base.Key("smoke_tests")
if smokeTests.DefaultProfile != nil {
if strings.TrimSpace(*smokeTests.DefaultProfile) == "" {
validation.Error(smokePath.Key("default_profile"), "default profile must not be blank")
} else if !domain.IsValidResourceID(*smokeTests.DefaultProfile) {
validation.Error(smokePath.Key("default_profile"), "default profile reference is malformed")
} }
} }
if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil {
validation.Error(kindPath, "authentication kind is required")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
}
}
}
}
func validateProfileRecord(profile profileConfig, base configkit.Path, backendIDs map[string]struct{}, validation *configkit.Validation) {
if profile.Endpoint != nil && *profile.Endpoint != "" {
if _, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*profile.Endpoint); err != nil {
validation.Error(base.Key("endpoint"), err.Error())
}
}
if !domain.IsValidLocalResourceKey(profile.ID) {
validation.Error(base.Key("id"), "profile ID must be a nonblank local key without a slash")
}
if profile.Backend != nil && *profile.Backend != "" {
if !domain.IsValidResourceID(*profile.Backend) {
validation.Error(base.Key("backend"), "backend reference is malformed")
} else if _, exists := backendIDs[*profile.Backend]; !exists {
validation.Error(base.Key("backend"), "backend is not defined")
}
}
if profile.Model != nil && *profile.Model != "" && strings.TrimSpace(*profile.Model) == "" {
validation.Error(base.Key("model"), "model must not be blank")
}
if profile.Temperature != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{Temperature: *profile.Temperature}); err != nil {
validation.Error(base.Key("temperature"), err.Error())
}
}
if profile.TopP != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TopP: *profile.TopP}); err != nil {
validation.Error(base.Key("top_p"), err.Error())
}
}
if profile.MaxTokens != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{MaxTokens: *profile.MaxTokens}); err != nil {
validation.Error(base.Key("max_tokens"), err.Error())
}
}
if profile.TimeoutSeconds != nil {
if err := domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{TimeoutSeconds: *profile.TimeoutSeconds}); err != nil {
validation.Error(base.Key("timeout_seconds"), err.Error())
}
}
validateExtraParams(profile.ExtraParams, base.Key("extra_params"), false, validation)
}
func validatePromptRecord(prompt promptConfig, base configkit.Path, validation *configkit.Validation) {
if !domain.IsValidLocalResourceKey(prompt.ID) {
validation.Error(base.Key("id"), "prompt ID must be a nonblank local key without a slash")
}
if prompt.DefaultProfile != nil && *prompt.DefaultProfile != "" && !domain.IsValidResourceID(*prompt.DefaultProfile) {
validation.Error(base.Key("default_profile"), "default profile reference is malformed")
} }
} }

View File

@@ -496,7 +496,7 @@ func TestLoadComposesLiteralKeysAndEffectivePresence(t *testing.T) {
t.Fatalf("recursive overlay failed: %+v", provider) t.Fatalf("recursive overlay failed: %+v", provider)
} }
backend := loaded.result.Config.Backends[0] backend := loaded.result.Config.Backends[0]
if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 || loaded.backends["local.v1"]["queue_capacity"] != fieldValue { if backend.QueueCapacity == nil || *backend.QueueCapacity != 0 {
t.Fatalf("explicit zero queue capacity was lost: %+v", backend) t.Fatalf("explicit zero queue capacity was lost: %+v", backend)
} }
if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil { if _, err := loaded.registry.GetBackend("base/local.v1"); err != nil {
@@ -582,9 +582,6 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 { composed.APIKeyEnv == nil || *composed.APIKeyEnv != "TOKEN_NAME" || composed.QueueCapacity == nil || *composed.QueueCapacity != 1024 {
t.Fatalf("resolved backend was not normalized: %+v", composed) t.Fatalf("resolved backend was not normalized: %+v", composed)
} }
if loaded.backends["local"]["queue_capacity"] != fieldNull {
t.Fatalf("explicit null was not retained: %+v", loaded.backends["local"])
}
field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint")) field, found := loaded.result.Metadata().Lookup(configkit.Keys("backends").Index(0).Key("endpoint"))
if !found || field.Origin.Operation != configkit.ChangeNormalization { if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("endpoint origin was not preserved: %+v", field) t.Fatalf("endpoint origin was not preserved: %+v", field)
@@ -763,7 +760,7 @@ func TestLoadDefaultsAbsentQueueCapacity(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
backend, err := loaded.GetBackend("local") backend, err := loaded.GetBackend("local")
if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 || loaded.backends["local"]["queue_capacity"] != fieldAbsent { if err != nil || backend.APIKeyEnv != "" || backend.QueueCapacity != 1024 || loaded.result.Config.Backends[0].QueueCapacity == nil || *loaded.result.Config.Backends[0].QueueCapacity != 1024 {
t.Fatalf("absent queue capacity did not receive its default: %+v", backend) t.Fatalf("absent queue capacity did not receive its default: %+v", backend)
} }
} }

View File

@@ -72,104 +72,21 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
if raw.SessionID != nil { if raw.SessionID != nil {
definition.SessionID = *raw.SessionID definition.SessionID = *raw.SessionID
} }
seenInputs := make(map[string]bool, len(raw.Inputs)) inputs, inputMetadata, err := state.capturePromptInputs(id, raw.Inputs, base)
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs)) if err != nil {
entry := &promptEntry{ return nil, err
messages: make([]*template.Template, 0, len(raw.Messages)),
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
} }
for index, input := range raw.Inputs { messages, compiledMessages, err := state.capturePromptMessages(raw.Messages, base, lookupOrigin)
path := base.Key("inputs").Index(index) if err != nil {
if input.Name == "" { return nil, err
return nil, fmt.Errorf("%s: input name is required", path.Key("name"))
}
if seenInputs[input.Name] {
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
}
seenInputs[input.Name] = true
contentType := ""
if input.ContentType != nil {
contentType = *input.ContentType
}
description := ""
if input.Description != nil {
description = *input.Description
}
definition.Inputs = append(definition.Inputs, domain.PromptInput{
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
})
presence := state.promptMetadata[id][index]
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
if presence.contentType == fieldValue {
metadata.ContentType = stringPointer(contentType)
}
if presence.description == fieldValue {
metadata.Description = stringPointer(description)
}
entry.requirementInputs = append(entry.requirementInputs, metadata)
} }
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages)) contract, err := capturePromptOutput(raw.Output, base)
for index, message := range raw.Messages { if err != nil {
path := base.Key("messages").Index(index) return nil, err
if message.Role == nil {
return nil, fmt.Errorf("%s: role is required", path.Key("role"))
}
if _, err := domain.NormalizeMessageRole(*message.Role); err != nil {
return nil, fmt.Errorf("%s: %w", path.Key("role"), err)
}
if (message.Content != nil) == (message.ContentFile != nil) {
return nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
}
content := ""
if message.Content != nil {
content = *message.Content
} else {
var err error
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
if err != nil {
return nil, err
}
}
var control *domain.CacheControl
if message.CacheControl != nil {
if message.CacheControl.Type == nil {
return nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
}
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
if message.CacheControl.TTL != nil {
control.TTL = *message.CacheControl.TTL
}
if _, err := domain.NormalizeCacheControl(control); err != nil {
return nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
}
}
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
if err != nil {
return nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
}
entry.messages = append(entry.messages, compiled)
definition.Templates = append(definition.Templates, domain.PromptMessageTemplate{
Role: *message.Role, Content: content, CacheControl: control,
})
} }
if raw.Output == nil { definition.Inputs = inputs
return nil, fmt.Errorf("%s: output is required", base.Key("output")) definition.Templates = messages
} entry := &promptEntry{requirementInputs: inputMetadata, messages: compiledMessages}
if raw.Output.Format == nil || raw.Output.ValidationMode == nil {
return nil, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
}
contract := domain.OutputContract{
Format: domain.OutputFormat(*raw.Output.Format),
ValidationMode: domain.ValidationMode(*raw.Output.ValidationMode),
RepairAttempts: raw.Output.RepairAttempts,
}
if raw.Output.SchemaPath != nil {
contract.SchemaPath = *raw.Output.SchemaPath
}
if err := domain.ValidateOutputContract(contract); err != nil {
return nil, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
}
definition.OutputFormat = contract.Format
definition.Validation = contract definition.Validation = contract
if raw.SessionID != nil { if raw.SessionID != nil {
var err error var err error
@@ -192,6 +109,113 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
return entry, nil return entry, nil
} }
func (state *loaded) capturePromptInputs(id string, rawInputs []promptInputConfig, base configkit.Path) ([]domain.PromptInput, []prompt.InputMetadata, error) {
seenInputs := make(map[string]bool, len(rawInputs))
inputs := make([]domain.PromptInput, 0, len(rawInputs))
metadata := make([]prompt.InputMetadata, 0, len(rawInputs))
for index, input := range rawInputs {
path := base.Key("inputs").Index(index)
if input.Name == "" {
return nil, nil, fmt.Errorf("%s: input name is required", path.Key("name"))
}
if seenInputs[input.Name] {
return nil, nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
}
seenInputs[input.Name] = true
contentType := ""
if input.ContentType != nil {
contentType = *input.ContentType
}
description := ""
if input.Description != nil {
description = *input.Description
}
inputs = append(inputs, domain.PromptInput{
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
})
presence := state.promptMetadata[id][index]
item := prompt.InputMetadata{Name: input.Name, Required: input.Required}
if presence.contentType == fieldValue {
item.ContentType = stringPointer(contentType)
}
if presence.description == fieldValue {
item.Description = stringPointer(description)
}
metadata = append(metadata, item)
}
return inputs, metadata, nil
}
func (state *loaded) capturePromptMessages(rawMessages []promptMessageConfig, base configkit.Path, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) ([]domain.PromptMessageTemplate, []*template.Template, error) {
templates := make([]domain.PromptMessageTemplate, 0, len(rawMessages))
compiledMessages := make([]*template.Template, 0, len(rawMessages))
for index, message := range rawMessages {
path := base.Key("messages").Index(index)
if message.Role == nil {
return nil, nil, fmt.Errorf("%s: role is required", path.Key("role"))
}
if _, err := domain.NormalizeMessageRole(*message.Role); err != nil {
return nil, nil, fmt.Errorf("%s: %w", path.Key("role"), err)
}
if (message.Content != nil) == (message.ContentFile != nil) {
return nil, nil, fmt.Errorf("%s: exactly one of content or content_file is required", path)
}
content := ""
if message.Content != nil {
content = *message.Content
} else {
var err error
content, err = state.readContent(path.Key("content_file"), *message.ContentFile, lookupOrigin)
if err != nil {
return nil, nil, err
}
}
var control *domain.CacheControl
if message.CacheControl != nil {
if message.CacheControl.Type == nil {
return nil, nil, fmt.Errorf("%s: type is required", path.Key("cache_control").Key("type"))
}
control = &domain.CacheControl{Type: domain.CacheControlType(*message.CacheControl.Type)}
if message.CacheControl.TTL != nil {
control.TTL = *message.CacheControl.TTL
}
if _, err := domain.NormalizeCacheControl(control); err != nil {
return nil, nil, fmt.Errorf("%s: %w", path.Key("cache_control"), err)
}
}
compiled, err := prompt.CompileTemplate(fmt.Sprintf("msg_%d", index), content)
if err != nil {
return nil, nil, fmt.Errorf("%s: invalid template syntax: %w", path, err)
}
compiledMessages = append(compiledMessages, compiled)
templates = append(templates, domain.PromptMessageTemplate{
Role: *message.Role, Content: content, CacheControl: control,
})
}
return templates, compiledMessages, nil
}
func capturePromptOutput(output *promptOutputConfig, base configkit.Path) (domain.OutputContract, error) {
if output == nil {
return domain.OutputContract{}, fmt.Errorf("%s: output is required", base.Key("output"))
}
if output.Format == nil || output.ValidationMode == nil {
return domain.OutputContract{}, fmt.Errorf("%s: format and validation_mode are required", base.Key("output"))
}
contract := domain.OutputContract{
Format: domain.OutputFormat(*output.Format),
ValidationMode: domain.ValidationMode(*output.ValidationMode),
RepairAttempts: output.RepairAttempts,
}
if output.SchemaPath != nil {
contract.SchemaPath = *output.SchemaPath
}
if err := domain.ValidateOutputContract(contract); err != nil {
return domain.OutputContract{}, fmt.Errorf("%s: invalid output contract: %w", base.Key("output"), err)
}
return contract, nil
}
func stringPointer(value string) *string { func stringPointer(value string) *string {
copy := value copy := value
return &copy return &copy
@@ -305,12 +329,6 @@ func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domai
return ownPromptInspection(entry.inspection), nil return ownPromptInspection(entry.inspection), nil
} }
// GetPromptDefinition implements the runner's prompt lookup boundary.
func (state *loaded) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) {
definition, _, err := state.GetPromptSnapshot(ctx, id)
return definition, err
}
// GetCompiledTemplates returns construction-time parses for one prompt ID. // GetCompiledTemplates returns construction-time parses for one prompt ID.
func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) { func (state *loaded) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
if state == nil { if state == nil {

View File

@@ -17,109 +17,115 @@ func (state *loaded) buildExecutionProfiles() error {
definitions := make(map[string]domain.Backend, len(backendConfigs)) definitions := make(map[string]domain.Backend, len(backendConfigs))
for _, id := range sortedKeys(backendConfigs) { for _, id := range sortedKeys(backendConfigs) {
config := backendConfigs[id] config := backendConfigs[id]
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams} definitions[id] = captureBackend(id, config)
if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil {
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
}
if config.SmokeTests.Authentication != nil {
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
Kind: config.SmokeTests.Authentication.Kind,
Endpoint: config.SmokeTests.Authentication.Endpoint,
}
}
definition.SmokeTests = smokeTests
}
if config.Endpoint != nil {
definition.Endpoint = *config.Endpoint
}
if config.APIKeyEnv != nil {
definition.APIKeyEnv = *config.APIKeyEnv
}
if config.ConcurrencyLimit != nil {
definition.ConcurrencyLimit = *config.ConcurrencyLimit
}
if config.QueueCapacity != nil {
definition.QueueCapacity = *config.QueueCapacity
definition.QueueCapacitySet = definition.ConcurrencyLimit > 0
}
definitions[id] = definition
} }
registry, err := backend.NewEffectiveRegistry(definitions) state.registry = backend.NewEffectiveRegistry(definitions)
if err != nil {
return err
}
state.registry = registry
state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs)) state.indexedProfiles = make(map[string]domain.ExecutionProfile, len(profileConfigs))
for _, id := range sortedKeys(profileConfigs) { for _, id := range sortedKeys(profileConfigs) {
config := profileConfigs[id] config := profileConfigs[id]
fields := state.profiles[id] profile, err := captureExecutionProfile(id, config, state.profiles[id], state.registry)
target := defaults.ExecutionTargetDefault() if err != nil {
if config.Backend != nil && *config.Backend != "" { return err
selected, err := registry.GetBackend(*config.Backend)
if err != nil {
return fmt.Errorf("profile %q backend: %w", id, err)
}
target.BackendID = selected.ID
target.Endpoint = selected.Endpoint
target.APIKeyEnv = selected.APIKeyEnv
target.ExtraParams = selected.ExtraParams
} }
presence := domain.ExecutionTargetPresence{}
if fields["endpoint"] != fieldAbsent {
target.Endpoint = valueOrZero(config.Endpoint)
}
if fields["model"] != fieldAbsent {
target.Model = valueOrZero(config.Model)
}
if fields["temperature"] != fieldAbsent {
target.Temperature = valueOrZero(config.Temperature)
presence.Temperature = fields["temperature"] == fieldValue
}
if fields["top_p"] != fieldAbsent {
target.TopP = valueOrZero(config.TopP)
presence.TopP = fields["top_p"] == fieldValue
}
if fields["max_tokens"] != fieldAbsent {
target.MaxTokens = valueOrZero(config.MaxTokens)
presence.MaxTokens = fields["max_tokens"] == fieldValue
}
if fields["timeout_seconds"] == fieldValue {
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
presence.TimeoutSeconds = true
}
if fields["service_tier"] != fieldAbsent {
target.ServiceTier = valueOrZero(config.ServiceTier)
}
if fields["reasoning_effort"] != fieldAbsent {
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
}
if fields["api_key_env"] != fieldAbsent {
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
}
if fields["api_key_required"] != fieldAbsent {
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
}
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
target.APIKeyEnv = ""
}
switch fields["extra_params"] {
case fieldNull:
target.ExtraParams = nil
case fieldValue:
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
if err != nil {
return fmt.Errorf("profile %q extra parameters: %w", id, err)
}
target.ExtraParams = merged
}
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
state.indexedProfiles[id] = profile state.indexedProfiles[id] = profile
} }
return nil return nil
} }
func captureBackend(id string, config backendConfig) domain.Backend {
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil {
smokeTests.Reachability = &domain.ReachabilityProbeConfig{Endpoint: config.SmokeTests.Reachability.Endpoint}
}
if config.SmokeTests.Authentication != nil {
smokeTests.Authentication = &domain.AuthenticationProbeConfig{
Kind: config.SmokeTests.Authentication.Kind,
Endpoint: config.SmokeTests.Authentication.Endpoint,
}
}
definition.SmokeTests = smokeTests
}
if config.Endpoint != nil {
definition.Endpoint = *config.Endpoint
}
if config.APIKeyEnv != nil {
definition.APIKeyEnv = *config.APIKeyEnv
}
if config.ConcurrencyLimit != nil {
definition.ConcurrencyLimit = *config.ConcurrencyLimit
}
if config.QueueCapacity != nil {
definition.QueueCapacity = *config.QueueCapacity
}
return definition
}
func captureExecutionProfile(id string, config profileConfig, fields map[string]fieldState, registry *backend.Registry) (domain.ExecutionProfile, error) {
target := defaults.ExecutionTargetDefault()
if config.Backend != nil && *config.Backend != "" {
selected, err := registry.GetBackend(*config.Backend)
if err != nil {
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w", id, err)
}
target.BackendID = selected.ID
target.Endpoint = selected.Endpoint
target.APIKeyEnv = selected.APIKeyEnv
target.ExtraParams = selected.ExtraParams
}
presence := domain.ExecutionTargetPresence{}
if fields["endpoint"] != fieldAbsent {
target.Endpoint = valueOrZero(config.Endpoint)
}
if fields["model"] != fieldAbsent {
target.Model = valueOrZero(config.Model)
}
if fields["temperature"] != fieldAbsent {
target.Temperature = valueOrZero(config.Temperature)
presence.Temperature = fields["temperature"] == fieldValue
}
if fields["top_p"] != fieldAbsent {
target.TopP = valueOrZero(config.TopP)
presence.TopP = fields["top_p"] == fieldValue
}
if fields["max_tokens"] != fieldAbsent {
target.MaxTokens = valueOrZero(config.MaxTokens)
presence.MaxTokens = fields["max_tokens"] == fieldValue
}
if fields["timeout_seconds"] == fieldValue {
target.TimeoutSeconds = valueOrZero(config.TimeoutSeconds)
presence.TimeoutSeconds = true
}
if fields["service_tier"] != fieldAbsent {
target.ServiceTier = valueOrZero(config.ServiceTier)
}
if fields["reasoning_effort"] != fieldAbsent {
target.ReasoningEffort = valueOrZero(config.ReasoningEffort)
}
if fields["api_key_env"] != fieldAbsent {
target.APIKeyEnv = valueOrZero(config.APIKeyEnv)
}
if fields["api_key_required"] != fieldAbsent {
target.APIKeyRequired = valueOrZero(config.APIKeyRequired)
}
if target.APIKeyRequired && fields["api_key_env"] == fieldAbsent {
target.APIKeyEnv = ""
}
switch fields["extra_params"] {
case fieldNull:
target.ExtraParams = nil
case fieldValue:
merged, err := overlayProviderMap(target.ExtraParams, config.ExtraParams)
if err != nil {
return domain.ExecutionProfile{}, fmt.Errorf("profile %q extra parameters: %w", id, err)
}
target.ExtraParams = merged
}
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
return profile, nil
}
func valueOrZero[T any](value *T) T { func valueOrZero[T any](value *T) T {
if value == nil { if value == nil {
var zero T var zero T
@@ -138,24 +144,33 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
if merged == nil { if merged == nil {
merged = make(map[string]any) merged = make(map[string]any)
} }
if err := overlayOwnedProviderMap(merged, upper); err != nil {
return nil, err
}
return merged, nil
}
func overlayOwnedProviderMap(merged, upper map[string]any) error {
for key, value := range upper { for key, value := range upper {
if child, ok := value.(map[string]any); ok { if child, ok := value.(map[string]any); ok {
if existing, ok := merged[key].(map[string]any); ok { if existing, ok := merged[key].(map[string]any); ok {
combined, err := overlayProviderMap(existing, child) if existing == nil {
if err != nil { existing = make(map[string]any)
return nil, err merged[key] = existing
}
if err := overlayOwnedProviderMap(existing, child); err != nil {
return err
} }
merged[key] = combined
continue continue
} }
} }
copied, err := jsonvalue.Copy(value) copied, err := jsonvalue.Copy(value)
if err != nil { if err != nil {
return nil, err return err
} }
merged[key] = copied merged[key] = copied
} }
return merged, nil return nil
} }
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) { func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
@@ -180,11 +195,7 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
return nil, err return nil, err
} }
target := *profile.ResolvedTarget target := *profile.ResolvedTarget
params, err := jsonvalue.CopyMap(target.ExtraParams) target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
if err != nil {
return nil, err
}
target.ExtraParams = params
profile.ResolvedTarget = &target profile.ResolvedTarget = &target
return &profile, nil return &profile, nil
} }

View File

@@ -42,6 +42,28 @@ func TestLoadIndexesIdentifiedTargets(t *testing.T) {
} }
} }
func TestOverlayProviderMapOwnsInputsAndReplacesValues(t *testing.T) {
lower := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"old"}, "null": "old"}, "disjoint": []any{"original"}}
upper := map[string]any{"nested": map[string]any{"replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}}
got, err := overlayProviderMap(lower, upper)
if err != nil {
t.Fatal(err)
}
want := map[string]any{"nested": map[string]any{"keep": "lower", "replace": []any{"new"}, "null": nil, "added": map[string]any{"value": "upper"}}, "disjoint": []any{"original"}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("overlay = %#v", got)
}
lower["disjoint"].([]any)[0] = "changed"
upper["nested"].(map[string]any)["replace"].([]any)[0] = "changed"
if !reflect.DeepEqual(got, want) {
t.Fatalf("overlay shares input storage: %#v", got)
}
got["nested"].(map[string]any)["added"].(map[string]any)["value"] = "changed"
if upper["nested"].(map[string]any)["added"].(map[string]any)["value"] != "upper" {
t.Fatal("result mutation reached upper")
}
}
func TestRequestCompletesLoadedPartialProfile(t *testing.T) { func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
state, err := load(mapping("partial profile", map[string]any{ state, err := load(mapping("partial profile", map[string]any{
"profiles": []any{map[string]any{"id": "partial"}}, "profiles": []any{map[string]any{"id": "partial"}},
@@ -53,7 +75,7 @@ func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
runner := usecase.NewRunner(state, state, nil, prompt.NewGoRenderer(), nil, nil, nil) runner := usecase.NewRunner(state, state, nil, prompt.NewCompiledRenderer(state), nil, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "example", ProfileID: "partial", PromptID: "example", ProfileID: "partial",
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"}, Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},
@@ -97,7 +119,7 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
selected, err := state.registry.GetBackend("second") selected, err := state.registry.GetBackend("second")
if err != nil || selected.QueueCapacity != 0 || !selected.QueueCapacitySet { if err != nil || selected.QueueCapacity != 0 || selected.ConcurrencyLimit != 2 {
t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err) t.Fatalf("backend capacity was not preserved: %+v, %v", selected, err)
} }
child, err := state.GetProfile(context.Background(), "child") child, err := state.GetProfile(context.Background(), "child")
@@ -197,10 +219,7 @@ func TestLoadedZeroControlsReachBuiltInRequest(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{}) client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}},
Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence, Target: *profile.ResolvedTarget, TargetPresence: profile.TargetPresence,

View File

@@ -145,7 +145,6 @@ type PromptDefinition struct {
SessionID string `yaml:"session_id" json:"session_id,omitempty"` SessionID string `yaml:"session_id" json:"session_id,omitempty"`
Inputs []PromptInput `yaml:"inputs"` Inputs []PromptInput `yaml:"inputs"`
Templates []PromptMessageTemplate `yaml:"templates"` Templates []PromptMessageTemplate `yaml:"templates"`
OutputFormat OutputFormat `yaml:"output_format"`
Validation OutputContract `yaml:"validation"` Validation OutputContract `yaml:"validation"`
} }
@@ -170,7 +169,6 @@ type PromptInput struct {
type PromptMessageTemplate struct { type PromptMessageTemplate struct {
Role string `yaml:"role"` Role string `yaml:"role"`
Content string `yaml:"content"` Content string `yaml:"content"`
ContentFile string `yaml:"content_file"`
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"` CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
} }
@@ -184,7 +182,6 @@ type Backend struct {
SmokeTests *BackendSmokeTestConfig SmokeTests *BackendSmokeTestConfig
ConcurrencyLimit int ConcurrencyLimit int
QueueCapacity int QueueCapacity int
QueueCapacitySet bool
} }
// BackendCapacityPolicy describes normalized run and generation capacity for // BackendCapacityPolicy describes normalized run and generation capacity for

View File

@@ -31,14 +31,6 @@ func qualifyResourceID(id, namespace string) (string, bool) {
return namespace + "/" + id, true return namespace + "/" + id, true
} }
func resourceNamespaceFromID(id string) (string, bool) {
if !validResourceID(id) || !strings.Contains(id, "/") {
return "", false
}
namespace, _, _ := strings.Cut(id, "/")
return namespace, true
}
// IsValidResourceID reports whether id is an unqualified resource ID or one // IsValidResourceID reports whether id is an unqualified resource ID or one
// valid namespace/local-ID pair. // valid namespace/local-ID pair.
func IsValidResourceID(id string) bool { return validResourceID(id) } func IsValidResourceID(id string) bool { return validResourceID(id) }

View File

@@ -86,25 +86,3 @@ func TestQualifyResourceID(t *testing.T) {
}) })
} }
} }
func TestResourceNamespaceFromID(t *testing.T) {
tests := []struct {
name string
id string
namespace string
wantOK bool
}{
{name: "qualified", id: "team/item", namespace: "team", wantOK: true},
{name: "same-prefix namespace stays distinct", id: "team-extra/item", namespace: "team-extra", wantOK: true},
{name: "unqualified", id: "item", wantOK: false},
{name: "malformed", id: "team/sub/item", wantOK: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, ok := resourceNamespaceFromID(test.id)
if got != test.namespace || ok != test.wantOK {
t.Fatalf("resourceNamespaceFromID(%q) = (%q, %t), want (%q, %t)", test.id, got, ok, test.namespace, test.wantOK)
}
})
}
}

View File

@@ -18,5 +18,5 @@ func NormalizeSessionID(raw string) (string, error) {
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength { if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength) return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
} }
return normalized, nil return strings.Clone(normalized), nil
} }

View File

@@ -112,24 +112,21 @@ type SmokeTestAttempt struct {
type SmokeTestReason string type SmokeTestReason string
const ( const (
SmokeTestReasonNone SmokeTestReason = "" SmokeTestReasonNone SmokeTestReason = ""
SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable" SmokeTestReasonAuthenticationNotApplicable SmokeTestReason = "authentication_not_applicable"
SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured" SmokeTestReasonProbeNotConfigured SmokeTestReason = "probe_not_configured"
SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable" SmokeTestReasonHTTPDiagnosticsUnavailable SmokeTestReason = "http_diagnostics_unavailable"
SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing" SmokeTestReasonRequiredCredentialMissing SmokeTestReason = "required_credential_missing"
SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized" SmokeTestReasonAuthenticationResponseUnrecognized SmokeTestReason = "authentication_response_unrecognized"
SmokeTestReasonAuthenticationResponseLimitExceeded SmokeTestReason = "authentication_response_limit_exceeded"
) )
// CloneBackend returns a copy of a backend, including its mutable settings. // CloneBackend returns a copy of a backend, including its mutable settings.
func CloneBackend(backend Backend) (Backend, error) { func CloneBackend(backend Backend) Backend {
cloned := backend cloned := backend
var err error cloned.ExtraParams = jsonvalue.CloneTrusted(backend.ExtraParams).(map[string]any)
cloned.ExtraParams, err = jsonvalue.CopyMap(backend.ExtraParams)
if err != nil {
return Backend{}, err
}
cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests) cloned.SmokeTests = CloneBackendSmokeTestConfig(backend.SmokeTests)
return cloned, nil return cloned
} }
// CloneBackendSmokeTestConfig returns an independent copy of optional metadata. // CloneBackendSmokeTestConfig returns an independent copy of optional metadata.
@@ -153,105 +150,6 @@ func CloneBackendSmokeTestConfig(config *BackendSmokeTestConfig) *BackendSmokeTe
return &cloned return &cloned
} }
// CloneSmokeTestRequest returns a copy that owns its optional settings.
func CloneSmokeTestRequest(request SmokeTestRequest) (SmokeTestRequest, error) {
cloned := request
cloned.TransientRetries = cloneInt(request.TransientRetries)
if request.Execution == nil {
return cloned, nil
}
execution := *request.Execution
execution.Temperature = cloneFloat64(request.Execution.Temperature)
execution.MaxTokens = cloneInt(request.Execution.MaxTokens)
execution.TopP = cloneFloat64(request.Execution.TopP)
execution.TimeoutSeconds = cloneInt(request.Execution.TimeoutSeconds)
execution.ReasoningEffort = cloneString(request.Execution.ReasoningEffort)
var err error
execution.ExtraParams, err = jsonvalue.CopyMap(request.Execution.ExtraParams)
if err != nil {
return SmokeTestRequest{}, err
}
cloned.Execution = &execution
return cloned, nil
}
// CloneSmokeTestReport returns a copy with independent optional facts and maps.
func CloneSmokeTestReport(report SmokeTestReport) (SmokeTestReport, error) {
cloned := report
cloned.Applicable = cloneBool(report.Applicable)
cloned.Facts = cloneSmokeTestFacts(report.Facts)
cloned.Timing = cloneSmokeTestTiming(report.Timing)
var err error
cloned.Target.EffectiveSettings.ExtraParams, err = jsonvalue.CopyMap(report.Target.EffectiveSettings.ExtraParams)
if err != nil {
return SmokeTestReport{}, err
}
if report.Attempts != nil {
cloned.Attempts = make([]SmokeTestAttempt, len(report.Attempts))
for index, attempt := range report.Attempts {
cloned.Attempts[index] = attempt
cloned.Attempts[index].HTTPStatus = cloneInt(attempt.HTTPStatus)
cloned.Attempts[index].Facts = cloneSmokeTestFacts(attempt.Facts)
cloned.Attempts[index].Timing = cloneSmokeTestTiming(attempt.Timing)
}
}
return cloned, nil
}
func cloneSmokeTestFacts(facts SmokeTestFacts) SmokeTestFacts {
facts.CredentialsSupplied = cloneBool(facts.CredentialsSupplied)
facts.EndpointReachable = cloneBool(facts.EndpointReachable)
facts.CredentialsAccepted = cloneBool(facts.CredentialsAccepted)
facts.ResponseReceived = cloneBool(facts.ResponseReceived)
facts.GenerationCompleted = cloneBool(facts.GenerationCompleted)
facts.TextReturned = cloneBool(facts.TextReturned)
facts.ExpectedTextMatched = cloneBool(facts.ExpectedTextMatched)
facts.StructuredOutputRequested = cloneBool(facts.StructuredOutputRequested)
facts.ValidJSON = cloneBool(facts.ValidJSON)
facts.SchemaMatched = cloneBool(facts.SchemaMatched)
return facts
}
func cloneSmokeTestTiming(timing SmokeTestTiming) SmokeTestTiming {
timing.CapacityWait = cloneDuration(timing.CapacityWait)
timing.RequestDuration = cloneDuration(timing.RequestDuration)
timing.RetryDelay = cloneDuration(timing.RetryDelay)
timing.EvaluationDuration = cloneDuration(timing.EvaluationDuration)
return timing
}
func cloneBool(value *bool) *bool {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneDuration(value *time.Duration) *time.Duration {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneFloat64(value *float64) *float64 {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneInt(value *int) *int {
if value == nil {
return nil
}
cloned := *value
return &cloned
}
func cloneString(value *string) *string { func cloneString(value *string) *string {
if value == nil { if value == nil {
return nil return nil

View File

@@ -1,9 +1,6 @@
package domain package domain
import ( import "testing"
"testing"
"time"
)
func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) { func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
profile := "profile" profile := "profile"
@@ -22,10 +19,7 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
}, },
} }
cloned, err := CloneBackend(backend) cloned := CloneBackend(backend)
if err != nil {
t.Fatal(err)
}
*cloned.SmokeTests.DefaultProfile = "changed" *cloned.SmokeTests.DefaultProfile = "changed"
*cloned.SmokeTests.Reachability.Endpoint = "changed" *cloned.SmokeTests.Reachability.Endpoint = "changed"
*cloned.SmokeTests.Authentication.Kind = "changed" *cloned.SmokeTests.Authentication.Kind = "changed"
@@ -41,72 +35,12 @@ func TestCloneBackendCopiesDiagnosticMetadataAndProviderValues(t *testing.T) {
} }
} }
func TestCloneSmokeTestRequestOwnsOptionalSettings(t *testing.T) { func TestCloneBackendPreservesProviderMapPresence(t *testing.T) {
temperature := 0.25 if cloned := CloneBackend(Backend{}); cloned.ExtraParams != nil {
retries := 2 t.Fatalf("nil provider map became %#v", cloned.ExtraParams)
reasoning := "low"
request := SmokeTestRequest{
APIKey: "secret",
TransientRetries: &retries,
Execution: &ExecutionTargetOverride{
Temperature: &temperature,
ReasoningEffort: &reasoning,
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
},
} }
cloned := CloneBackend(Backend{ExtraParams: map[string]any{"empty": map[string]any{}, "null": nil}})
cloned, err := CloneSmokeTestRequest(request) if cloned.ExtraParams == nil || cloned.ExtraParams["empty"].(map[string]any) == nil || cloned.ExtraParams["null"] != nil {
if err != nil { t.Fatalf("provider map presence changed: %#v", cloned.ExtraParams)
t.Fatal(err)
}
*cloned.TransientRetries = 3
*cloned.Execution.Temperature = 0.5
*cloned.Execution.ReasoningEffort = "high"
cloned.Execution.ExtraParams["nested"].(map[string]any)["value"] = "changed"
if retries != 2 || temperature != 0.25 || reasoning != "low" {
t.Fatal("cloned request shared optional scalar pointers")
}
if got := request.Execution.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
t.Fatalf("source provider parameter changed through clone: %v", got)
}
}
func TestCloneSmokeTestReportOwnsMutableValues(t *testing.T) {
applicable := true
reachable := true
status := 200
wait := 3 * time.Millisecond
report := SmokeTestReport{
Applicable: &applicable,
Target: SmokeTestTarget{
EffectiveSettings: ExecutionTarget{
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
},
},
Facts: SmokeTestFacts{EndpointReachable: &reachable},
Attempts: []SmokeTestAttempt{{
HTTPStatus: &status,
Facts: SmokeTestFacts{EndpointReachable: &reachable},
Timing: SmokeTestTiming{CapacityWait: &wait},
}},
}
cloned, err := CloneSmokeTestReport(report)
if err != nil {
t.Fatal(err)
}
*cloned.Applicable = false
*cloned.Facts.EndpointReachable = false
*cloned.Attempts[0].HTTPStatus = 503
*cloned.Attempts[0].Facts.EndpointReachable = false
*cloned.Attempts[0].Timing.CapacityWait = time.Second
cloned.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] = "changed"
if !applicable || !reachable || status != 200 || wait != 3*time.Millisecond {
t.Fatal("cloned report shared optional scalar pointers")
}
if got := report.Target.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"]; got != "original" {
t.Fatalf("source provider parameter changed through clone: %v", got)
} }
} }

View File

@@ -43,7 +43,7 @@ type ConfigurationSources struct {
type Options struct { type Options struct {
llmClient llm.Client llmClient llm.Client
artifactReader artifact.Reader artifactReader artifact.Reader
validator validate.Validator validator validate.ValidationPreparer
validatorSource bool validatorSource bool
artifactSource bool artifactSource bool
disableBundledOpenRouter bool disableBundledOpenRouter bool
@@ -73,7 +73,7 @@ func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
// SetSchemaFile selects a single schema file. // SetSchemaFile selects a single schema file.
func (o *Options) SetSchemaFile(path string) error { func (o *Options) SetSchemaFile(path string) error {
fsys, root, _, err := fileSource(path) fsys, root, err := fileSource(path)
if err != nil { if err != nil {
return err return err
} }
@@ -98,9 +98,9 @@ type resourceNamespaces struct {
} }
type identityInventory interface { type identityInventory interface {
ListBackends(context.Context) ([]string, error) ListBackends(context.Context, configuration.IdentityFilter) ([]string, error)
ListProfiles(context.Context) ([]string, error) ListProfiles(context.Context, configuration.IdentityFilter) ([]string, error)
ListPrompts(context.Context) ([]domain.PromptIdentity, error) ListPrompts(context.Context, configuration.IdentityFilter) ([]domain.PromptIdentity, error)
ListNamespaces(context.Context, []string) ([]string, error) ListNamespaces(context.Context, []string) ([]string, error)
ListCollections(context.Context) ([]configuration.CollectionIdentity, error) ListCollections(context.Context) ([]configuration.CollectionIdentity, error)
} }
@@ -157,10 +157,7 @@ func New(cfg Config, options Options) (*Engine, error) {
var client llm.Client var client llm.Client
var diagnosticHTTPClient *llm.OpenAICompatibleClient var diagnosticHTTPClient *llm.OpenAICompatibleClient
if options.llmClient == nil { if options.llmClient == nil {
builtIn, newErr := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient}) builtIn := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient})
if newErr != nil {
return nil, newErr
}
diagnosticHTTPClient = builtIn diagnosticHTTPClient = builtIn
client = generation.NewBuiltIn(manager, builtIn) client = generation.NewBuiltIn(manager, builtIn)
} else { } else {
@@ -208,23 +205,23 @@ func validateSourceFS(fsys fs.FS, root string) error {
return nil return nil
} }
func fileSource(name string) (fs.FS, string, string, error) { func fileSource(name string) (fs.FS, string, error) {
if strings.TrimSpace(name) == "" { if strings.TrimSpace(name) == "" {
return nil, "", "", fmt.Errorf("source path is required") return nil, "", fmt.Errorf("source path is required")
} }
directory := filepath.Dir(name) directory := filepath.Dir(name)
base := filepath.Base(name) base := filepath.Base(name)
if base == "." || base == string(filepath.Separator) { if base == "." || base == string(filepath.Separator) {
return nil, "", "", fmt.Errorf("source path must name a file") return nil, "", fmt.Errorf("source path must name a file")
} }
info, err := os.Stat(name) info, err := os.Stat(name)
if err != nil { if err != nil {
return nil, "", "", fmt.Errorf("failed to access source file %q: %w", name, err) return nil, "", fmt.Errorf("failed to access source file %q: %w", name, err)
} }
if info.IsDir() { if info.IsDir() {
return nil, "", "", fmt.Errorf("source path %q must be a file", name) return nil, "", fmt.Errorf("source path %q must be a file", name)
} }
return os.DirFS(directory), filepath.ToSlash(base), directory, nil return os.DirFS(directory), filepath.ToSlash(base), nil
} }
func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) { func (e *Engine) InspectPrompt(ctx context.Context, id string) (*domain.PromptInspection, error) {
@@ -268,24 +265,11 @@ func (e *Engine) ListProfiles(ctx context.Context, filters ...ListingFilter) ([]
} }
func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) { func (e *Engine) ListPrompts(ctx context.Context, filters ...ListingFilter) ([]domain.PromptIdentity, error) {
mode, namespace, err := resolveListingFilter(filters, e.namespaces.prompt) filter, err := inventoryFilter(filters, e.namespaces.prompt)
if err != nil { if err != nil {
return nil, err return nil, err
} }
prompts, err := e.listing.ListPrompts(ctx) return e.listing.ListPrompts(ctx, filter)
if err != nil {
return nil, err
}
var filtered []domain.PromptIdentity
for _, prompt := range prompts {
if matchesListingNamespace(prompt.ID, mode, namespace) {
filtered = append(filtered, prompt)
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return filtered, nil
} }
func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) { func (e *Engine) ListNamespaces(ctx context.Context, kinds []string) ([]string, error) {
@@ -307,26 +291,21 @@ func filterIdentityList(
ctx context.Context, ctx context.Context,
filters []ListingFilter, filters []ListingFilter,
scope string, scope string,
list func(context.Context) ([]string, error), list func(context.Context, configuration.IdentityFilter) ([]string, error),
) ([]string, error) { ) ([]string, error) {
filter, err := inventoryFilter(filters, scope)
if err != nil {
return nil, err
}
return list(ctx, filter)
}
func inventoryFilter(filters []ListingFilter, scope string) (configuration.IdentityFilter, error) {
mode, namespace, err := resolveListingFilter(filters, scope) mode, namespace, err := resolveListingFilter(filters, scope)
if err != nil { if err != nil {
return nil, err return configuration.IdentityFilter{}, err
} }
ids, err := list(ctx) return configuration.IdentityFilter{All: mode == ListingAll, Namespace: namespace}, nil
if err != nil {
return nil, err
}
var filtered []string
for _, id := range ids {
if matchesListingNamespace(id, mode, namespace) {
filtered = append(filtered, id)
}
}
if err := ctx.Err(); err != nil {
return nil, err
}
return filtered, nil
} }
func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) { func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterMode, string, error) {
@@ -355,19 +334,6 @@ func resolveListingFilter(filters []ListingFilter, scope string) (ListingFilterM
} }
} }
func matchesListingNamespace(id string, mode ListingFilterMode, namespace string) bool {
switch mode {
case ListingNamed:
idNamespace, _, qualified := strings.Cut(id, "/")
return qualified && idNamespace == namespace
case ListingUnqualified:
return !strings.Contains(id, "/")
case ListingAll:
return true
default:
return false
}
}
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) { func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
request, err := e.qualifyRunRequest(request) request, err := e.qualifyRunRequest(request)
if err != nil { if err != nil {

View File

@@ -67,15 +67,6 @@ func FindFSYAMLFiles(ctx context.Context, fsys fs.FS, root string) ([]string, er
return files, err return files, err
} }
// RelativePath computes a clean relative path from root to path.
func RelativePath(root string, filePath string) string {
rel, err := filepath.Rel(root, filePath)
if err != nil {
return filepath.Clean(filePath)
}
return filepath.Clean(rel)
}
// CleanFSRoot normalizes a root path for use with fs.FS while preserving // CleanFSRoot normalizes a root path for use with fs.FS while preserving
// nonblank leading and trailing whitespace. // nonblank leading and trailing whitespace.
func CleanFSRoot(root string) string { func CleanFSRoot(root string) string {

View File

@@ -81,16 +81,6 @@ func TestFindFSYAMLFilesHonorsContextCancellation(t *testing.T) {
} }
} }
func TestRelativePathNested(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "nested", "profiles", "local.yaml")
got := RelativePath(root, path)
want := filepath.Join("nested", "profiles", "local.yaml")
if got != want {
t.Fatalf("expected relative path %q, got %q", want, got)
}
}
func TestCleanFSRoot(t *testing.T) { func TestCleanFSRoot(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@@ -85,6 +85,12 @@ func (c *Coordinator) GenerateWithEvidence(
return c.generate(ctx, req, true) return c.generate(ctx, req, true)
} }
// PreparationError identifies a built-in request preparation failure.
type PreparationError struct{ Cause error }
func (e *PreparationError) Error() string { return e.Cause.Error() }
func (e *PreparationError) Unwrap() error { return e.Cause }
func (c *Coordinator) generate( func (c *Coordinator) generate(
ctx context.Context, ctx context.Context,
req domain.GenerateRequest, req domain.GenerateRequest,
@@ -95,7 +101,7 @@ func (c *Coordinator) generate(
} }
prepared, err := c.builtIn.PrepareGeneration(req) prepared, err := c.builtIn.PrepareGeneration(req)
if err != nil { if err != nil {
return nil, err, GenerationEvidence{} return nil, &PreparationError{Cause: err}, GenerationEvidence{PreparationFailed: true}
} }
evidence := GenerationEvidence{} evidence := GenerationEvidence{}
@@ -120,66 +126,37 @@ func (c *Coordinator) generate(
return finish(nil, err) return finish(nil, err)
} }
ordinal := attempts + 1 ordinal := attempts + 1
outcome, dispatched, capacityWait, generationDuration, startedAt := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence) result := c.execute(ctx, req.Target.BackendID, prepared, captureEvidence)
if dispatched { if result.Dispatched {
attempts++ attempts++
} }
if captureEvidence { recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
attempt := AttemptEvidence{ if result.Outcome.Err == nil {
StartedAt: startedAt.UTC(),
Dispatched: dispatched,
ResponseReceived: outcome.Evidence.ResponseReceived,
HTTP: cloneHTTPEvidence(outcome.Evidence),
CapacityWait: capacityWait,
RequestDuration: generationDuration,
Failure: ClassifyFailure(outcome.Err),
ResponseDecoded: outcome.ResponseDecoded,
GenerationCompleted: outcome.GenerationCompleted,
}
if outcome.StructuredOutputRequested != nil {
requested := *outcome.StructuredOutputRequested
attempt.StructuredOutputRequested = &requested
}
if dispatched {
usage.Add(outcome.Usage, outcome.UsagePresence)
var attemptUsage observation.UsageAccumulator
attemptUsage.Add(outcome.Usage, outcome.UsagePresence)
attempt.Usage = attemptUsage.Observed()
}
evidence.Attempts = append(evidence.Attempts, attempt)
}
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: dispatched, CapacityWait: capacityWait, GenerationDuration: generationDuration,
Outcome: attemptOutcome(outcome.Err), Failure: ClassifyFailure(outcome.Err),
StatusCode: outcome.StatusCode, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
}
if outcome.Err == nil {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
if captureEvidence { if captureEvidence {
return finish(outcome.Response, err) return finish(result.Outcome.Response, err)
} }
return finish(nil, err) return finish(nil, err)
} }
return finish(outcome.Response, nil) return finish(result.Outcome.Response, nil)
} }
if !dispatched || !eligibleStatus(outcome) { if !result.Dispatched || !eligibleStatus(result.Outcome) {
if captureEvidence { if captureEvidence {
return finish(outcome.Response, outcome.Err) return finish(result.Outcome.Response, result.Outcome.Err)
} }
return finish(nil, outcome.Err) return finish(nil, result.Outcome.Err)
} }
if retriesUsed == req.TransientRetries { if retriesUsed == req.TransientRetries {
if retriesUsed == 0 { if retriesUsed == 0 {
return finish(nil, outcome.Err) return finish(nil, result.Outcome.Err)
} }
return finish(nil, &ExhaustedError{attempts: attempts, cause: outcome.Err}) return finish(nil, &ExhaustedError{attempts: attempts, cause: result.Outcome.Err})
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return finish(nil, err) return finish(nil, err)
} }
retryOrdinal := retriesUsed + 1 retryOrdinal := retriesUsed + 1
delay := retryDelay(c.backoff(retryOrdinal), outcome.RetryNotBefore, c.now()) delay := retryDelay(c.backoff(retryOrdinal), result.Outcome.RetryNotBefore, c.now())
waitStart := c.now() waitStart := c.now()
waitErr := c.wait(ctx, delay) waitErr := c.wait(ctx, delay)
waitDuration := elapsed(waitStart, c.now()) waitDuration := elapsed(waitStart, c.now())
@@ -200,127 +177,133 @@ func (c *Coordinator) generate(
} }
} }
func (c *Coordinator) generateInjected( func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateRequest, captureEvidence bool) (*domain.GenerateResponse, error, GenerationEvidence) {
ctx context.Context,
req domain.GenerateRequest,
captureEvidence bool,
) (*domain.GenerateResponse, error, GenerationEvidence) {
evidence := GenerationEvidence{} evidence := GenerationEvidence{}
var usage observation.UsageAccumulator var usage observation.UsageAccumulator
finish := func(response *domain.GenerateResponse, err error) (*domain.GenerateResponse, error, GenerationEvidence) {
if captureEvidence {
evidence.Usage = usage.Observed()
}
return response, err, evidence
}
recorder := observation.FromContext(ctx)
generation := req.Generation generation := req.Generation
if generation == 0 { if generation == 0 {
generation = 1 generation = 1
} }
start := c.now() result := c.executeInjected(ctx, req)
release, err := c.manager.Acquire(ctx, req.Target.BackendID) recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
capacityWait := c.capacityWait(req.Target.BackendID, start)
if err != nil {
if captureEvidence {
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
StartedAt: start.UTC(),
CapacityWait: capacityWait,
Failure: ClassifyFailure(err),
})
}
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
}
return finish(nil, err)
}
defer release()
if err := ctx.Err(); err != nil {
release()
if captureEvidence {
evidence.Attempts = append(evidence.Attempts, AttemptEvidence{
StartedAt: start.UTC(),
CapacityWait: capacityWait,
Failure: ClassifyFailure(err),
})
}
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: 1, CapacityWait: capacityWait, Outcome: domain.ExecutionOutcomeError, Failure: ClassifyFailure(err)})
}
return finish(nil, err)
}
dispatchStart := c.now()
response, err := c.injected.Generate(ctx, req)
duration := elapsed(dispatchStart, c.now())
if err == nil && response == nil {
err = llm.ErrMalformedResponse
}
if captureEvidence { if captureEvidence {
attempt := AttemptEvidence{ evidence.Usage = usage.Observed()
StartedAt: start.UTC(),
Dispatched: true,
ResponseReceived: response != nil,
RequestDuration: duration,
Failure: ClassifyFailure(err),
GenerationCompleted: err == nil && response != nil,
}
requested := req.StructuredOutput != nil
attempt.StructuredOutputRequested = &requested
var reportedUsage domain.TokenUsage
var reportedPresence *domain.TokenUsagePresence
if err == nil && response != nil {
reportedUsage = response.Usage
reportedPresence = response.UsagePresence
}
usage.Add(reportedUsage, reportedPresence)
var attemptUsage observation.UsageAccumulator
attemptUsage.Add(reportedUsage, reportedPresence)
attempt.Usage = attemptUsage.Observed()
evidence.Attempts = append(evidence.Attempts, attempt)
} }
release() if result.Outcome.Err != nil {
if recorder != nil { return nil, result.Outcome.Err, evidence
data := observation.AttemptData{Generation: generation, Attempt: 1, Dispatched: true, CapacityWait: capacityWait, GenerationDuration: duration,
Outcome: attemptOutcome(err), Failure: ClassifyFailure(err)}
if err == nil {
data.Usage = response.Usage
data.UsagePresence = response.UsagePresence
}
recorder.Attempt(data)
}
if err != nil {
return finish(nil, err)
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
if captureEvidence { if captureEvidence {
return finish(response, err) return result.Outcome.Response, err, evidence
} }
return finish(nil, err) return nil, err, evidence
} }
return finish(response, nil) return result.Outcome.Response, nil, evidence
} }
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) (outcome llm.AttemptOutcome, dispatched bool, capacityWait, generationDuration time.Duration, startedAt time.Time) { func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRequest) attemptResult {
start := c.now() result := attemptResult{StartedAt: c.now()}
startedAt = start release, err := c.manager.Acquire(ctx, req.Target.BackendID)
release, err := c.manager.Acquire(ctx, backendID) result.CapacityWait = c.capacityWait(req.Target.BackendID, result.StartedAt)
capacityWait = c.capacityWait(backendID, start)
if err != nil { if err != nil {
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt result.Outcome.Err = err
return result
} }
defer release() defer release()
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return llm.AttemptOutcome{Err: err}, false, capacityWait, 0, startedAt result.Outcome.Err = err
return result
}
start := c.now()
result.Dispatched = true
requested := req.StructuredOutput != nil
result.Outcome.StructuredOutputRequested = &requested
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
result.RequestDuration = elapsed(start, c.now())
if result.Outcome.Err == nil && result.Outcome.Response == nil {
result.Outcome.Err = llm.ErrMalformedResponse
}
if result.Outcome.Err == nil {
result.Outcome.Usage = result.Outcome.Response.Usage
result.Outcome.UsagePresence = result.Outcome.Response.UsagePresence
}
return result
}
type attemptResult struct {
Outcome llm.AttemptOutcome
Dispatched bool
CapacityWait time.Duration
RequestDuration time.Duration
StartedAt time.Time
}
// recordAttempt runs after the attempt has released its permit, including when
// the recorder calls application code. Injected clients expose no HTTP facts.
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
outcome := result.Outcome
failure := ClassifyFailure(outcome.Err)
if capture {
attempt := AttemptEvidence{
StartedAt: result.StartedAt.UTC(), Dispatched: result.Dispatched,
CapacityWait: result.CapacityWait, RequestDuration: result.RequestDuration,
Failure: failure,
}
if injected {
attempt.ResponseReceived = outcome.Response != nil
attempt.GenerationCompleted = outcome.Err == nil && outcome.Response != nil
if result.Dispatched {
attempt.CapacityWait = 0
}
} else {
attempt.ResponseReceived = outcome.Evidence.ResponseReceived
attempt.HTTP = cloneHTTPEvidence(outcome.Evidence)
attempt.ResponseDecoded = outcome.ResponseDecoded
attempt.GenerationCompleted = outcome.GenerationCompleted
}
if outcome.StructuredOutputRequested != nil {
requested := *outcome.StructuredOutputRequested
attempt.StructuredOutputRequested = &requested
}
if result.Dispatched {
usage.Add(outcome.Usage, outcome.UsagePresence)
var single observation.UsageAccumulator
single.Add(outcome.Usage, outcome.UsagePresence)
attempt.Usage = single.Observed()
}
evidence.Attempts = append(evidence.Attempts, attempt)
}
if recorder != nil {
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
}
}
func (c *Coordinator) execute(ctx context.Context, backendID string, prepared *llm.PreparedGeneration, captureEvidence bool) attemptResult {
result := attemptResult{StartedAt: c.now()}
release, err := c.manager.Acquire(ctx, backendID)
result.CapacityWait = c.capacityWait(backendID, result.StartedAt)
if err != nil {
result.Outcome.Err = err
return result
}
defer release()
if err := ctx.Err(); err != nil {
result.Outcome.Err = err
return result
} }
dispatchStart := c.now() dispatchStart := c.now()
if captureEvidence { if captureEvidence {
outcome = prepared.ExecuteWithEvidence(ctx) result.Outcome = prepared.ExecuteWithEvidence(ctx)
dispatched = outcome.Evidence.DispatchInvoked result.Dispatched = result.Outcome.Evidence.DispatchInvoked
} else { } else {
outcome = prepared.Execute(ctx) result.Outcome = prepared.Execute(ctx)
dispatched = true result.Dispatched = true
} }
return outcome, dispatched, capacityWait, elapsed(dispatchStart, c.now()), startedAt result.RequestDuration = elapsed(dispatchStart, c.now())
return result
} }
func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration { func (c *Coordinator) capacityWait(backendID string, start time.Time) time.Duration {

View File

@@ -40,10 +40,7 @@ func testCoordinator(t *testing.T, transport transportFunc) *Coordinator {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}}) client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: transport}})
if err != nil {
t.Fatal(err)
}
coordinator := NewBuiltIn(manager, client) coordinator := NewBuiltIn(manager, client)
coordinator.jitter = func(time.Duration) time.Duration { return 0 } coordinator.jitter = func(time.Duration) time.Duration { return 0 }
coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() } coordinator.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }

View File

@@ -10,6 +10,8 @@ import (
// GenerationEvidence contains invocation-local request evidence without raw // GenerationEvidence contains invocation-local request evidence without raw
// provider bodies, headers, credentials, or generated content. // provider bodies, headers, credentials, or generated content.
type GenerationEvidence struct { type GenerationEvidence struct {
// PreparationFailed identifies a built-in request rejected before attempts.
PreparationFailed bool
// InitialEndpoint is the frozen built-in request URL, if prepared. // InitialEndpoint is the frozen built-in request URL, if prepared.
// Injected generation leaves it unknown. // Injected generation leaves it unknown.
InitialEndpoint string InitialEndpoint string

View File

@@ -144,6 +144,32 @@ func (c fixedClient) Generate(context.Context, domain.GenerateRequest) (*domain.
return c.response, c.err return c.response, c.err
} }
func TestInjectedObserverRunsAfterPermitRelease(t *testing.T) {
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP request"); return nil, nil })
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{}})
called := false
recorder, err := observation.New(func(event domain.ExecutionEvent) {
if event.Kind != domain.ExecutionAttempt {
return
}
called = true
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
release, err := coordinator.manager.Acquire(ctx, "limited")
if err != nil {
t.Errorf("observer could not acquire permit: %v", err)
return
}
release()
})
if err != nil {
t.Fatal(err)
}
if _, err := coordinator.Generate(observation.WithRecorder(context.Background(), recorder), testRequest(0)); err != nil || !called {
t.Fatalf("generation error = %v, observer called = %v", err, called)
}
}
func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) { func TestObservedInjectedUsageAndRepairGeneration(t *testing.T) {
manager, err := capacity.NewManager(nil) manager, err := capacity.NewManager(nil)
if err != nil { if err != nil {

View File

@@ -0,0 +1,97 @@
package jsonvalue
import "reflect"
func cloneTrustedMap(src map[string]any) map[string]any {
if src == nil {
return nil
}
out := make(map[string]any, len(src))
for k, v := range src {
out[k] = CloneTrusted(v)
}
return out
}
// CloneTrusted detaches a previously accepted value without revalidating it.
func CloneTrusted(value any) any {
if value == nil {
return nil
}
switch v := value.(type) {
case map[string]any:
return cloneTrustedMap(v)
case []any:
out := make([]any, len(v))
for i, item := range v {
out[i] = CloneTrusted(item)
}
return out
case []string:
if v == nil {
return []string(nil)
}
return append([]string{}, v...)
case []byte:
if v == nil {
return []byte(nil)
}
return append([]byte{}, v...)
default:
return cloneTrustedReflect(reflect.ValueOf(value)).Interface()
}
}
func cloneTrustedReflect(value reflect.Value) reflect.Value {
if !value.IsValid() {
return value
}
switch value.Kind() {
case reflect.Interface:
if value.IsNil() {
return reflect.Zero(value.Type())
}
copied := cloneTrustedReflect(value.Elem())
if copied.IsValid() && copied.Type().AssignableTo(value.Type()) {
return copied
}
out := reflect.New(value.Type()).Elem()
out.Set(copied)
return out
case reflect.Pointer:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.New(value.Type().Elem())
out.Elem().Set(cloneTrustedReflect(value.Elem()))
return out
case reflect.Map:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeMapWithSize(value.Type(), value.Len())
iter := value.MapRange()
for iter.Next() {
out.SetMapIndex(cloneTrustedReflect(iter.Key()), cloneTrustedReflect(iter.Value()))
}
return out
case reflect.Slice:
if value.IsNil() {
return reflect.Zero(value.Type())
}
out := reflect.MakeSlice(value.Type(), value.Len(), value.Cap())
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
}
return out
case reflect.Array:
out := reflect.New(value.Type()).Elem()
for i := 0; i < value.Len(); i++ {
out.Index(i).Set(cloneTrustedReflect(value.Index(i)))
}
return out
default:
return value
}
}

View File

@@ -0,0 +1,27 @@
package jsonvalue
import (
"encoding/json"
"reflect"
"testing"
)
type trustedNamed string
func TestCloneTrustedPreservesTypesAndDetaches(t *testing.T) {
named := trustedNamed("value")
original := map[string]any{"": map[string]any{"number": json.Number("9007199254740993")}, "typed": []string{"original"}, "pointer": &named, "nilmap": map[string]any(nil), "nilslice": []string(nil), "empty": []any{}}
copy := CloneTrusted(original).(map[string]any)
if !reflect.DeepEqual(copy, original) {
t.Fatalf("clone = %#v", copy)
}
copy[""].(map[string]any)["number"] = json.Number("1")
copy["typed"].([]string)[0] = "changed"
*copy["pointer"].(*trustedNamed) = "changed"
if original[""].(map[string]any)["number"] != json.Number("9007199254740993") || original["typed"].([]string)[0] != "original" || named != "value" {
t.Fatal("clone shares mutable storage")
}
if copy["nilmap"].(map[string]any) != nil || copy["nilslice"].([]string) != nil || copy["empty"].([]any) == nil {
t.Fatal("nil/empty shape changed")
}
}

View File

@@ -25,10 +25,7 @@ func TestGetWithEvidenceCapturesResponseAndOmitsURLUserInfo(t *testing.T) {
})) }))
defer server.Close() defer server.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: time.Second})
if err != nil {
t.Fatalf("construct client: %v", err)
}
endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health" endpoint := strings.Replace(server.URL, "http://", "http://user:secret@", 1) + "/health"
response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0) response, evidence, err := client.GetWithEvidence(context.Background(), endpoint, 0)
if err != nil { if err != nil {
@@ -88,7 +85,7 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment) t.Setenv("PROMPTKIT_DIAGNOSTIC_KEY", tc.environment)
var authorization string var authorization string
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
authorization = request.Header.Get("Authorization") authorization = request.Header.Get("Authorization")
return &http.Response{ return &http.Response{
@@ -98,9 +95,6 @@ func TestGetWithAuthenticationEvidenceResolvesOptionalCredentials(t *testing.T)
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
response, evidence, err := client.GetWithAuthenticationEvidence( response, evidence, err := client.GetWithAuthenticationEvidence(
context.Background(), "https://example.invalid/key", 0, tc.target, context.Background(), "https://example.invalid/key", 0, tc.target,
@@ -269,10 +263,7 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
transport := http.DefaultTransport transport := http.DefaultTransport
supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar} supplied := &http.Client{Transport: transport, Timeout: 7 * time.Second, CheckRedirect: redirect, Jar: jar}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: supplied})
if err != nil {
t.Fatalf("construct client: %v", err)
}
const calls = 8 const calls = 8
evidences := make([]HTTPExecutionEvidence, calls) evidences := make([]HTTPExecutionEvidence, calls)
@@ -307,15 +298,12 @@ func TestGetWithEvidencePreservesClientAndIsolatesConcurrentCalls(t *testing.T)
} }
func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) { func TestGetWithEvidenceAppliesTargetTimeout(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{Timeout: 5 * time.Second, HTTPClient: &http.Client{
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
<-request.Context().Done() <-request.Context().Done()
return nil, request.Context().Err() return nil, request.Context().Err()
}), }),
}}) }})
if err != nil {
t.Fatalf("construct client: %v", err)
}
response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1) response, evidence, err := client.GetWithEvidence(context.Background(), "https://diagnostic.example/slow", 1)
if response != nil || !errors.Is(err, context.DeadlineExceeded) { if response != nil || !errors.Is(err, context.DeadlineExceeded) {

View File

@@ -17,10 +17,10 @@ import (
"go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
) )
var ( var (
ErrInvalidConfig = errors.New("invalid llm client configuration")
ErrInvalidRequest = errors.New("invalid generate request") ErrInvalidRequest = errors.New("invalid generate request")
ErrRequestFailed = errors.New("llm request failed") ErrRequestFailed = errors.New("llm request failed")
ErrTransport = errors.New("llm transport failed") ErrTransport = errors.New("llm transport failed")
@@ -43,28 +43,15 @@ func (e *requestFailedError) Unwrap() []error {
} }
type OpenAICompatibleConfig struct { type OpenAICompatibleConfig struct {
BaseURL string
Model string
Timeout time.Duration Timeout time.Duration
HTTPClient *http.Client HTTPClient *http.Client
} }
type OpenAICompatibleClient struct { type OpenAICompatibleClient struct {
baseURL string httpClient *http.Client
defaultModel string
httpClient *http.Client
} }
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) { func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) *OpenAICompatibleClient {
baseURL := ""
if strings.TrimSpace(cfg.BaseURL) != "" {
var err error
baseURL, err = domain.NormalizeOpenAICompatibleBaseEndpoint(cfg.BaseURL)
if err != nil {
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
}
}
timeout := cfg.Timeout timeout := cfg.Timeout
if timeout <= 0 { if timeout <= 0 {
timeout = defaults.LLMRequestTimeoutDefault timeout = defaults.LLMRequestTimeoutDefault
@@ -82,10 +69,8 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
} }
return &OpenAICompatibleClient{ return &OpenAICompatibleClient{
baseURL: baseURL, httpClient: client,
defaultModel: cfg.Model, }
httpClient: client,
}, nil
} }
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
@@ -132,11 +117,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
} }
selectedEndpoint := req.Target.Endpoint endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(req.Target.Endpoint)
if strings.TrimSpace(selectedEndpoint) == "" {
selectedEndpoint = c.baseURL
}
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(selectedEndpoint)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: invalid endpoint: %v", ErrInvalidRequest, err)
} }
@@ -145,12 +126,10 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: invalid endpoint path: %v", ErrInvalidRequest, err)
} }
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel) // Custom JSON encoders may mutate their receivers. Both payload encoding
if err != nil { // and encoding-failure diagnostics must use values detached from metadata.
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) req.Target.ExtraParams = jsonvalue.CloneTrusted(req.Target.ExtraParams).(map[string]any)
} wirePayload, err := openAIChatRequestPayload(req)
wirePayload, err := openAIChatRequestPayload(wireReq)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
} }
@@ -179,7 +158,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
return &PreparedGeneration{ return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey, endpoint: endpoint, payload: payload, apiKey: apiKey,
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient, timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
structuredOutputRequested: wireReq.ResponseFormat != nil, structuredOutputRequested: wirePayload["response_format"] != nil,
}, nil }, nil
} }
@@ -370,40 +349,69 @@ func tokenCount(value *int) int {
return *value return *value
} }
type responseErrorReader struct {
reader io.Reader
err error
}
func (r *responseErrorReader) Read(p []byte) (int, error) {
n, err := r.reader.Read(p)
if err != nil && !errors.Is(err, io.EOF) {
r.err = err
}
return n, err
}
func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) { func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
observed := &responseErrorReader{reader: body}
limited := &io.LimitedReader{ limited := &io.LimitedReader{
R: body, R: observed,
N: maxOpenAIChatResponseBytes + 1, N: maxOpenAIChatResponseBytes + 1,
} }
decoder := json.NewDecoder(limited) decoder := json.NewDecoder(limited)
var response openAIChatResponse var response openAIChatResponse
if err := decoder.Decode(&response); err != nil { if err := decoder.Decode(&response); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
return openAIChatResponse{}, &requestFailedError{cause: err} return openAIChatResponse{}, &requestFailedError{cause: observed.err}
} }
if limited.N == 0 { if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError() return openAIChatResponse{}, openAIChatResponseTooLargeError()
} }
if observed.err != nil {
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
}
return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse) return openAIChatResponse{}, fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
} }
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
}
if limited.N == 0 { if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError() return openAIChatResponse{}, openAIChatResponseTooLargeError()
} }
var trailing any var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
return openAIChatResponse{}, &requestFailedError{cause: err} return openAIChatResponse{}, &requestFailedError{cause: observed.err}
} }
if limited.N == 0 { if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError() return openAIChatResponse{}, openAIChatResponseTooLargeError()
} }
if observed.err != nil {
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
}
return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse) return openAIChatResponse{}, fmt.Errorf("%w: response contains trailing data", ErrMalformedResponse)
} }
if errors.Is(observed.err, context.Canceled) || errors.Is(observed.err, context.DeadlineExceeded) {
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
}
if limited.N == 0 { if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError() return openAIChatResponse{}, openAIChatResponseTooLargeError()
} }
if observed.err != nil {
return openAIChatResponse{}, &requestFailedError{cause: observed.err}
}
return response, nil return response, nil
} }
@@ -416,99 +424,52 @@ func openAIChatResponseTooLargeError() error {
) )
} }
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) { func openAIChatRequestPayload(req domain.GenerateRequest) (map[string]any, error) {
model := strings.TrimSpace(req.Target.Model) model := strings.TrimSpace(req.Target.Model)
if model == "" { if model == "" {
model = strings.TrimSpace(defaultModel) return nil, errors.New("model is required")
}
if model == "" {
return openAIChatRequest{}, errors.New("model is required")
} }
wireReq := openAIChatRequest{
Model: model,
}
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID) sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
if err != nil { if err != nil {
return openAIChatRequest{}, err return nil, err
} }
wireReq.SessionID = sessionID
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages)) messages := make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
for _, msg := range req.Prompt.Messages { for _, msg := range req.Prompt.Messages {
wireReq.Messages = append(wireReq.Messages, openAIChatRequestMessageFromRenderedMessage(msg)) messages = append(messages, openAIChatRequestMessageFromRenderedMessage(msg))
} }
out := map[string]any{"model": model, "messages": messages}
if sessionID != "" {
out["session_id"] = sessionID
}
if req.Target.Temperature != 0 || req.TargetPresence.Temperature { if req.Target.Temperature != 0 || req.TargetPresence.Temperature {
wireReq.Temperature = &req.Target.Temperature out["temperature"] = req.Target.Temperature
} }
if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens { if req.Target.MaxTokens != 0 || req.TargetPresence.MaxTokens {
wireReq.MaxTokens = &req.Target.MaxTokens out["max_tokens"] = req.Target.MaxTokens
} }
if req.Target.TopP != 0 || req.TargetPresence.TopP { if req.Target.TopP != 0 || req.TargetPresence.TopP {
wireReq.TopP = &req.Target.TopP out["top_p"] = req.Target.TopP
} }
if strings.TrimSpace(req.Target.ServiceTier) != "" { if strings.TrimSpace(req.Target.ServiceTier) != "" {
wireReq.ServiceTier = req.Target.ServiceTier out["service_tier"] = req.Target.ServiceTier
} }
if strings.TrimSpace(req.Target.ReasoningEffort) != "" { if strings.TrimSpace(req.Target.ReasoningEffort) != "" {
wireReq.ReasoningEffort = req.Target.ReasoningEffort out["reasoning_effort"] = req.Target.ReasoningEffort
}
if len(req.Target.ExtraParams) > 0 {
wireReq.ExtraParams = req.Target.ExtraParams
} }
if req.StructuredOutput != nil { if req.StructuredOutput != nil {
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput) responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
if err != nil { if err != nil {
return openAIChatRequest{}, err return nil, err
}
if responseFormat != nil {
out["response_format"] = responseFormat
} }
wireReq.ResponseFormat = responseFormat
} }
return wireReq, nil for key, value := range req.Target.ExtraParams {
}
type openAIChatRequest struct {
Model string `json:"model"`
SessionID string `json:"session_id,omitempty"`
Messages []openAIChatRequestMessage `json:"messages"`
Temperature *float64 `json:"temperature,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
ExtraParams map[string]any `json:"-"`
}
func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
out := map[string]any{
"model": req.Model,
"messages": req.Messages,
}
if req.SessionID != "" {
out["session_id"] = req.SessionID
}
if req.Temperature != nil {
out["temperature"] = *req.Temperature
}
if req.MaxTokens != nil {
out["max_tokens"] = *req.MaxTokens
}
if req.TopP != nil {
out["top_p"] = *req.TopP
}
if req.ServiceTier != "" {
out["service_tier"] = req.ServiceTier
}
if req.ReasoningEffort != "" {
out["reasoning_effort"] = req.ReasoningEffort
}
if req.ResponseFormat != nil {
out["response_format"] = req.ResponseFormat
}
for key, value := range req.ExtraParams {
if key == "" { if key == "" {
return nil, errors.New("extra_params key must not be empty") return nil, errors.New("extra_params key must not be empty")
} }

View File

@@ -123,26 +123,19 @@ func (p *recordingProvider) lastRequest(t *testing.T) recordedProviderRequest {
return p.requests[len(p.requests)-1] return p.requests[len(p.requests)-1]
} }
func newProviderClient(t *testing.T, provider *recordingProvider, config OpenAICompatibleConfig) *OpenAICompatibleClient {
t.Helper()
if config.BaseURL == "" {
config.BaseURL = provider.endpoint("/v1")
}
client, err := NewOpenAICompatibleClient(config)
if err != nil {
t.Fatalf("construct client: %v", err)
}
return client
}
func ordinaryGenerateRequest() domain.GenerateRequest { func ordinaryGenerateRequest() domain.GenerateRequest {
return domain.GenerateRequest{ return domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model"}, Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "model"},
} }
} }
func (p *recordingProvider) generateRequest() domain.GenerateRequest {
req := ordinaryGenerateRequest()
req.Target.Endpoint = p.endpoint("/v1")
return req
}
type deadlineCapturingTransport struct { type deadlineCapturingTransport struct {
deadline time.Time deadline time.Time
hasDeadline bool hasDeadline bool
@@ -253,35 +246,13 @@ func assertDeadlineNear(t *testing.T, deadline, before, after time.Time, duratio
} }
} }
func TestNewOpenAICompatibleClientRejectsInvalidBaseURL(t *testing.T) {
for _, endpoint := range []string{
"://invalid",
"/v1",
"https:///v1",
"ftp://provider.example/v1",
"https://user@provider.example/v1",
"https://provider.example/v1?mode=chat",
"https://provider.example/v1#chat",
} {
t.Run(endpoint, func(t *testing.T) {
_, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: endpoint})
if !errors.Is(err, ErrInvalidConfig) {
t.Fatalf("expected ErrInvalidConfig, got %v", err)
}
})
}
}
func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) { func TestNewOpenAICompatibleClientDoesNotMutateSuppliedZeroTimeoutClient(t *testing.T) {
transport := http.DefaultTransport transport := http.DefaultTransport
supplied := &http.Client{Transport: transport} supplied := &http.Client{Transport: transport}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
HTTPClient: supplied, HTTPClient: supplied,
}) })
if err != nil {
t.Fatalf("unexpected constructor error: %v", err)
}
if supplied.Timeout != 0 { if supplied.Timeout != 0 {
t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout) t.Fatalf("expected supplied client timeout to remain zero, got %v", supplied.Timeout)
@@ -305,13 +276,10 @@ func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *t
Transport: transport, Transport: transport,
} }
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
Timeout: 2 * time.Second, Timeout: 2 * time.Second,
HTTPClient: supplied, HTTPClient: supplied,
}) })
if err != nil {
t.Fatalf("unexpected constructor error: %v", err)
}
if supplied.Timeout != suppliedTimeout { if supplied.Timeout != suppliedTimeout {
t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout) t.Fatalf("expected supplied client timeout to remain %v, got %v", suppliedTimeout, supplied.Timeout)
@@ -335,13 +303,10 @@ func TestNewOpenAICompatibleClientTreatsSuppliedNegativeTimeoutAsUnset(t *testin
} }
configuredTimeout := 23 * time.Second configuredTimeout := 23 * time.Second
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
Timeout: configuredTimeout, Timeout: configuredTimeout,
HTTPClient: supplied, HTTPClient: supplied,
}) })
if err != nil {
t.Fatalf("unexpected constructor error: %v", err)
}
if supplied.Timeout != -time.Second { if supplied.Timeout != -time.Second {
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout) t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
@@ -370,7 +335,6 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
{name: "optional response fields", run: checkOptionalResponseFieldOmission}, {name: "optional response fields", run: checkOptionalResponseFieldOmission},
{name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping}, {name: "reasoning and extra parameters", run: checkReasoningAndExtraParameterMapping},
{name: "optional request field presence", run: checkRequestFieldPresence}, {name: "optional request field presence", run: checkRequestFieldPresence},
{name: "configured model fallback", run: checkConfiguredModelFallback},
} }
for _, tc := range tests { for _, tc := range tests {
@@ -380,14 +344,14 @@ func TestOpenAICompatibleClientRequestMapping(t *testing.T) {
func checkCanonicalRoleAndExactContentMapping(t *testing.T) { func checkCanonicalRoleAndExactContentMapping(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
const content = " exact content\nwith whitespace " const content = " exact content\nwith whitespace "
_, err := client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
Role: domain.RoleDeveloper, Role: domain.RoleDeveloper,
Content: content, Content: content,
}}}, }}},
Target: domain.ExecutionTarget{Model: "model"}, Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
}) })
if err != nil { if err != nil {
t.Fatalf("Generate() error = %v", err) t.Fatalf("Generate() error = %v", err)
@@ -413,7 +377,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33} "usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
}`) }`)
client := newProviderClient(t, provider, OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
Timeout: 2 * time.Second, Timeout: 2 * time.Second,
}) })
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key") t.Setenv("PROMPTKIT_TEST_API_KEY", "secret-key")
@@ -424,6 +388,7 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
{Role: "user", Content: "Say hello"}, {Role: "user", Content: "Say hello"},
}}, }},
Target: domain.ExecutionTarget{ Target: domain.ExecutionTarget{
Endpoint: provider.endpoint("/v1"),
Model: "gpt-test", Model: "gpt-test",
Temperature: 0.4, Temperature: 0.4,
MaxTokens: 123, MaxTokens: 123,
@@ -594,9 +559,11 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
tc.configureEnv(t) tc.configureEnv(t)
} }
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
request := ordinaryGenerateRequest() request := provider.generateRequest()
request.Target = tc.target request.Target = tc.target
request.Target.Endpoint = provider.endpoint("/v1")
request.Target.Model = "model"
_, err := client.Generate(context.Background(), request) _, err := client.Generate(context.Background(), request)
if tc.wantError != nil { if tc.wantError != nil {
@@ -625,7 +592,7 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
func checkCacheControlledMessageMapping(t *testing.T) { func checkCacheControlledMessageMapping(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
_, err := client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
@@ -639,7 +606,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
}, },
{Role: "user", Content: "Dynamic request."}, {Role: "user", Content: "Dynamic request."},
}}, }},
Target: domain.ExecutionTarget{Model: "model"}, Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -691,7 +658,7 @@ func checkCacheControlledMessageMapping(t *testing.T) {
func checkEmptyCacheControlTTLOmission(t *testing.T) { func checkEmptyCacheControlTTLOmission(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
_, err := client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
@@ -703,7 +670,7 @@ func checkEmptyCacheControlTTLOmission(t *testing.T) {
}, },
}, },
}}, }},
Target: domain.ExecutionTarget{Model: "model"}, Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -745,8 +712,8 @@ func checkSessionIDMapping(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
request := ordinaryGenerateRequest() request := provider.generateRequest()
request.Prompt.SessionID = tc.sessionID request.Prompt.SessionID = tc.sessionID
if _, err := client.Generate(context.Background(), request); err != nil { if _, err := client.Generate(context.Background(), request); err != nil {
@@ -766,19 +733,14 @@ func checkSessionIDMapping(t *testing.T) {
} }
func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) { func TestOpenAICompatibleClientRejectsTooLongSessionID(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
BaseURL: "http://example.com/v1",
Model: "model",
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{ Prompt: domain.RenderedPrompt{
SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1), SessionID: strings.Repeat("x", domain.SessionIDMaxLength+1),
Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}, Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}},
}, },
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model"},
}) })
if err == nil { if err == nil {
t.Fatal("expected invalid request error") t.Fatal("expected invalid request error")
@@ -828,9 +790,9 @@ func checkContentPresence(t *testing.T) {
"cache_write_tokens": 5 "cache_write_tokens": 5
} }
}`) }`)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
response, err := client.Generate(context.Background(), ordinaryGenerateRequest()) response, err := client.Generate(context.Background(), provider.generateRequest())
if err != nil { if err != nil {
t.Fatalf("generate: %v", err) t.Fatalf("generate: %v", err)
} }
@@ -865,10 +827,11 @@ func checkCacheUsageMapping(t *testing.T) {
"cache_write_tokens": 60 "cache_write_tokens": 60
} }
}`) }`)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
resp, err := client.Generate(context.Background(), domain.GenerateRequest{ resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model"},
}) })
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
@@ -901,8 +864,8 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`) provider.respond(http.StatusOK, `{`+tc.usage+`"choices":[{"message":{"content":"ok"}}]}`)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
resp, err := client.Generate(context.Background(), ordinaryGenerateRequest()) resp, err := client.Generate(context.Background(), provider.generateRequest())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -913,17 +876,17 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
} }
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`) provider.respond(http.StatusOK, `{"usage":{"prompt_tokens":"seven"},"choices":[{"message":{"content":"ok"}}]}`)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
if resp, err := client.Generate(context.Background(), ordinaryGenerateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) { if resp, err := client.Generate(context.Background(), provider.generateRequest()); resp != nil || !errors.Is(err, ErrMalformedResponse) {
t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err) t.Fatalf("invalid numeric usage response=%+v, error=%v", resp, err)
} }
} }
func checkOptionalResponseFieldOmission(t *testing.T) { func checkOptionalResponseFieldOmission(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
_, err := client.Generate(context.Background(), ordinaryGenerateRequest()) _, err := client.Generate(context.Background(), provider.generateRequest())
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
} }
@@ -938,11 +901,12 @@ func checkOptionalResponseFieldOmission(t *testing.T) {
func checkReasoningAndExtraParameterMapping(t *testing.T) { func checkReasoningAndExtraParameterMapping(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
_, err := client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{ Target: domain.ExecutionTarget{
Endpoint: provider.endpoint("/v1"),
Model: "model", Model: "model",
ReasoningEffort: "high", ReasoningEffort: "high",
ExtraParams: map[string]any{ ExtraParams: map[string]any{
@@ -1020,8 +984,8 @@ func checkRequestFieldPresence(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
request := ordinaryGenerateRequest() request := provider.generateRequest()
request.TargetPresence = tc.presence request.TargetPresence = tc.presence
if _, err := client.Generate(context.Background(), request); err != nil { if _, err := client.Generate(context.Background(), request); err != nil {
@@ -1073,21 +1037,17 @@ func checkOmittedGenerationTimeout(t *testing.T) {
transport := &deadlineCapturingTransport{} transport := &deadlineCapturingTransport{}
clientTimeout := 5 * time.Second clientTimeout := 5 * time.Second
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Timeout: clientTimeout, Timeout: clientTimeout,
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Transport: transport, Transport: transport,
}, },
}) })
if err != nil {
t.Fatal(err)
}
before := time.Now() before := time.Now()
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0}, Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "model", TimeoutSeconds: 0},
}) })
after := time.Now() after := time.Now()
if err == nil { if err == nil {
@@ -1146,11 +1106,11 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
_, err := client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", ExtraParams: tc.extraParams}, Target: domain.ExecutionTarget{Endpoint: provider.endpoint("/v1"), Model: "model", ExtraParams: tc.extraParams},
}) })
if err == nil { if err == nil {
t.Fatal("expected invalid request error") t.Fatal("expected invalid request error")
@@ -1192,8 +1152,8 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
request := ordinaryGenerateRequest() request := provider.generateRequest()
request.Target.ExtraParams = tc.extraParams request.Target.ExtraParams = tc.extraParams
request.StructuredOutput = &domain.StructuredOutputSpec{ request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema, Type: domain.StructuredOutputJSONSchema,
@@ -1218,30 +1178,12 @@ func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
} }
} }
func checkConfiguredModelFallback(t *testing.T) {
provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})
_, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
var model string
if !provider.lastRequest(t).decodeField(t, "model", &model) || model != "default-model" {
t.Fatalf("model = %q, want default-model", model)
}
}
func TestOpenAICompatibleClientEndpointComposition(t *testing.T) { func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
run func(*testing.T) run func(*testing.T)
}{ }{
{name: "request endpoint overrides configured endpoint", run: checkEndpointOverride}, {name: "request endpoint", run: checkSelectedEndpoint},
{name: "request endpoint works without configured endpoint", run: checkEmptyConfiguredEndpoint},
{name: "completion URL preserves valid base paths", run: checkCompletionURLComposition}, {name: "completion URL preserves valid base paths", run: checkCompletionURLComposition},
{name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection}, {name: "invalid selected endpoint fails before transport", run: checkInvalidSelectedEndpointRejection},
{name: "an endpoint is required", run: checkRequiredEndpoint}, {name: "an endpoint is required", run: checkRequiredEndpoint},
@@ -1252,35 +1194,6 @@ func TestOpenAICompatibleClientEndpointComposition(t *testing.T) {
} }
} }
func checkEndpointOverride(t *testing.T) {
configuredProvider := newRecordingProvider(t)
configuredProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"default"}}]}`)
selectedProvider := newRecordingProvider(t)
selectedProvider.respond(http.StatusOK, `{"choices":[{"message":{"content":"override"}}]}`)
client := newProviderClient(t, configuredProvider, OpenAICompatibleConfig{Model: "m"})
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: selectedProvider.endpoint("/v1")},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if resp.Content != "override" {
t.Fatalf("expected override response, got %q", resp.Content)
}
if got := configuredProvider.requestCount(); got != 0 {
t.Fatalf("configured provider calls = %d, want 0", got)
}
if got := selectedProvider.requestCount(); got != 1 {
t.Fatalf("selected provider calls = %d, want 1", got)
}
if got := selectedProvider.lastRequest(t).url; got != "/v1/chat/completions" {
t.Fatalf("selected URL = %q, want /v1/chat/completions", got)
}
}
func checkCommonResponseFailures(t *testing.T) { func checkCommonResponseFailures(t *testing.T) {
const sensitiveBody = `provider-secret-fragment request_payload_details` const sensitiveBody = `provider-secret-fragment request_payload_details`
tests := []struct { tests := []struct {
@@ -1310,9 +1223,9 @@ func checkCommonResponseFailures(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
provider.respond(tc.statusCode, tc.body) provider.respond(tc.statusCode, tc.body)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "m"}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
response, err := client.Generate(context.Background(), ordinaryGenerateRequest()) response, err := client.Generate(context.Background(), provider.generateRequest())
if response != nil { if response != nil {
t.Fatalf("expected no partial response, got %+v", response) t.Fatalf("expected no partial response, got %+v", response)
} }
@@ -1357,9 +1270,7 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
body := sizedSuccessResponseBody(tc.size) body := sizedSuccessResponseBody(tc.size)
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
return &http.Response{ return &http.Response{
StatusCode: http.StatusOK, StatusCode: http.StatusOK,
@@ -1370,12 +1281,10 @@ func checkSuccessfulResponseByteBoundary(t *testing.T) {
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatalf("construct client: %v", err)
}
response, err := client.Generate(context.Background(), domain.GenerateRequest{ response, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
}) })
if tc.wantErr { if tc.wantErr {
if response != nil { if response != nil {
@@ -1425,9 +1334,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
strings.NewReader(prefix), strings.NewReader(prefix),
continuation, continuation,
)} )}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
return &http.Response{ return &http.Response{
StatusCode: http.StatusOK, StatusCode: http.StatusOK,
@@ -1438,9 +1345,6 @@ func checkContinuingOversizedResponse(t *testing.T) {
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatalf("construct client: %v", err)
}
type outcome struct { type outcome struct {
response *domain.GenerateResponse response *domain.GenerateResponse
@@ -1450,6 +1354,7 @@ func checkContinuingOversizedResponse(t *testing.T) {
go func() { go func() {
response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{ response, generateErr := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
}) })
done <- outcome{response: response, err: generateErr} done <- outcome{response: response, err: generateErr}
}() }()
@@ -1497,9 +1402,7 @@ func checkSingleResponseDocument(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
body := &countingReadCloser{reader: strings.NewReader(tc.body)} body := &countingReadCloser{reader: strings.NewReader(tc.body)}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
return &http.Response{ return &http.Response{
StatusCode: http.StatusOK, StatusCode: http.StatusOK,
@@ -1510,12 +1413,10 @@ func checkSingleResponseDocument(t *testing.T) {
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatalf("construct client: %v", err)
}
response, err := client.Generate(context.Background(), domain.GenerateRequest{ response, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1", Model: "m"},
}) })
if tc.wantErr { if tc.wantErr {
if response != nil { if response != nil {
@@ -1541,22 +1442,19 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded} transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
generationTimeout := 2 * time.Second generationTimeout := 2 * time.Second
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "m",
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Timeout: 10 * time.Second, Timeout: 10 * time.Second,
Transport: transport, Transport: transport,
}, },
}) })
if err != nil {
t.Fatal(err)
}
before := time.Now() before := time.Now()
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{ Target: domain.ExecutionTarget{
Endpoint: "http://example.com/v1",
Model: "m",
TimeoutSeconds: int(generationTimeout / time.Second), TimeoutSeconds: int(generationTimeout / time.Second),
}, },
}) })
@@ -1578,25 +1476,22 @@ func checkGenerationTimeoutPrecedence(t *testing.T) {
func checkCallerDeadlinePrecedence(t *testing.T) { func checkCallerDeadlinePrecedence(t *testing.T) {
transport := &deadlineCapturingTransport{err: context.DeadlineExceeded} transport := &deadlineCapturingTransport{err: context.DeadlineExceeded}
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "m",
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Timeout: 10 * time.Second, Timeout: 10 * time.Second,
Transport: transport, Transport: transport,
}, },
}) })
if err != nil {
t.Fatal(err)
}
callerDeadline := time.Now().Add(time.Second) callerDeadline := time.Now().Add(time.Second)
ctx, cancel := context.WithDeadline(context.Background(), callerDeadline) ctx, cancel := context.WithDeadline(context.Background(), callerDeadline)
defer cancel() defer cancel()
_, err = client.Generate(ctx, domain.GenerateRequest{ _, err := client.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{ Target: domain.ExecutionTarget{
Endpoint: "http://example.com/v1",
Model: "m",
TimeoutSeconds: 2, TimeoutSeconds: 2,
}, },
}) })
@@ -1618,22 +1513,18 @@ func checkCallerDeadlinePrecedence(t *testing.T) {
} }
func checkCallerCancellationIdentity(t *testing.T) { func checkCallerCancellationIdentity(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "m",
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Transport: contextErrorTransport{}, Transport: contextErrorTransport{},
}, },
}) })
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
_, err = client.Generate(ctx, domain.GenerateRequest{ _, err := client.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
}) })
if err == nil { if err == nil {
t.Fatal("expected cancellation error") t.Fatal("expected cancellation error")
@@ -1647,21 +1538,17 @@ func checkCallerCancellationIdentity(t *testing.T) {
} }
func checkExpiredCallerDeadlineIdentity(t *testing.T) { func checkExpiredCallerDeadlineIdentity(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "m",
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Transport: contextErrorTransport{}, Transport: contextErrorTransport{},
}, },
}) })
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0)) ctx, cancel := context.WithDeadline(context.Background(), time.Unix(1, 0))
defer cancel() defer cancel()
_, err = client.Generate(ctx, domain.GenerateRequest{ _, err := client.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
}) })
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("expected request failure and caller deadline identities, got %v", err) t.Fatalf("expected request failure and caller deadline identities, got %v", err)
@@ -1669,20 +1556,16 @@ func checkExpiredCallerDeadlineIdentity(t *testing.T) {
} }
func checkWholeRequestTimeoutIdentity(t *testing.T) { func checkWholeRequestTimeoutIdentity(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1",
Model: "m",
HTTPClient: &http.Client{ HTTPClient: &http.Client{
Timeout: time.Millisecond, Timeout: time.Millisecond,
Transport: waitingContextTransport{}, Transport: waitingContextTransport{},
}, },
}) })
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "http://example.com/v1", Model: "m"},
}) })
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("expected request failure and client timeout identities, got %v", err) t.Fatalf("expected request failure and client timeout identities, got %v", err)
@@ -1696,20 +1579,15 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
apiKey = "sensitive-api-key" apiKey = "sensitive-api-key"
content = "sensitive prompt content" content = "sensitive prompt content"
) )
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: endpoint,
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportCause return nil, transportCause
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: content}}},
Target: domain.ExecutionTarget{APIKey: apiKey}, Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m", APIKey: apiKey},
}) })
if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) { if !errors.Is(err, ErrRequestFailed) || !errors.Is(err, transportCause) {
t.Fatalf("expected request failure and transport cause identities, got %v", err) t.Fatalf("expected request failure and transport cause identities, got %v", err)
@@ -1726,13 +1604,7 @@ func checkTransportFailureIdentityAndRedaction(t *testing.T) {
} }
func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) { func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
BaseURL: "http://example.com/v1",
Model: "m",
})
if err != nil {
t.Fatal(err)
}
type testCase struct { type testCase struct {
name string name string
@@ -1765,11 +1637,9 @@ func TestOpenAICompatibleClientRejectsInvalidExecutionSettings(t *testing.T) {
} }
} }
func checkEmptyConfiguredEndpoint(t *testing.T) { func checkSelectedEndpoint(t *testing.T) {
var selectedURL string var selectedURL string
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "",
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
selectedURL = req.URL.String() selectedURL = req.URL.String()
return &http.Response{ return &http.Response{
@@ -1782,13 +1652,10 @@ func checkEmptyConfiguredEndpoint(t *testing.T) {
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatalf("expected empty configured base URL to be allowed, got %v", err)
}
response, err := client.Generate(context.Background(), domain.GenerateRequest{ response, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1"}, Target: domain.ExecutionTarget{Endpoint: "http://request-endpoint.example/v1", Model: "m"},
}) })
if err != nil { if err != nil {
t.Fatalf("generate with request endpoint: %v", err) t.Fatalf("generate with request endpoint: %v", err)
@@ -1817,9 +1684,7 @@ func checkCompletionURLComposition(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
var selectedURL string var selectedURL string
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: tc.baseURL,
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
selectedURL = req.URL.String() selectedURL = req.URL.String()
return &http.Response{ return &http.Response{
@@ -1830,12 +1695,10 @@ func checkCompletionURLComposition(t *testing.T) {
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatalf("construct client: %v", err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: tc.baseURL, Model: "m"},
}) })
if err != nil { if err != nil {
t.Fatalf("generate: %v", err) t.Fatalf("generate: %v", err)
@@ -1860,21 +1723,16 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
for _, endpoint := range invalidEndpoints { for _, endpoint := range invalidEndpoints {
t.Run(endpoint, func(t *testing.T) { t.Run(endpoint, func(t *testing.T) {
transportCalls := 0 transportCalls := 0
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://configured.example/v1",
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
transportCalls++ transportCalls++
return nil, errors.New("transport must not be called") return nil, errors.New("transport must not be called")
})}, })},
}) })
if err != nil {
t.Fatalf("construct client: %v", err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: endpoint}, Target: domain.ExecutionTarget{Endpoint: endpoint, Model: "m"},
}) })
if !errors.Is(err, ErrInvalidRequest) { if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected ErrInvalidRequest, got %v", err)
@@ -1890,17 +1748,11 @@ func checkInvalidSelectedEndpointRejection(t *testing.T) {
} }
func checkRequiredEndpoint(t *testing.T) { func checkRequiredEndpoint(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
BaseURL: "",
Model: "m",
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{ _, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{}, Target: domain.ExecutionTarget{Model: "m"},
}) })
if err == nil { if err == nil {
t.Fatal("expected endpoint-required error") t.Fatal("expected endpoint-required error")
@@ -1909,3 +1761,13 @@ func checkRequiredEndpoint(t *testing.T) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected ErrInvalidRequest, got %v", err)
} }
} }
func TestOpenAICompatibleClientRequiresModel(t *testing.T) {
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
_, err := client.PrepareGeneration(domain.GenerateRequest{
Target: domain.ExecutionTarget{Endpoint: "https://provider.example/v1"},
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("missing model error = %v, want invalid request", err)
}
}

View File

@@ -0,0 +1,108 @@
package llm
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"net/url"
"go.maximumdirect.net/promptkit/internal/domain"
)
const maxAuthenticationResponseBytes = 1 << 20
// AuthenticationEvidence describes the observable result of a key probe.
type AuthenticationEvidence struct {
HTTP HTTPExecutionEvidence
Accepted *bool
Overflow bool
Unrecognized bool
}
// OpenRouterKeyEndpoint derives the default key endpoint from a base endpoint.
func OpenRouterKeyEndpoint(base string) (string, error) {
return url.JoinPath(base, "key")
}
// ProbeOpenRouterKey executes and interprets an OpenRouter key request.
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
parsed, err := url.Parse(endpoint)
if err != nil {
return AuthenticationEvidence{}, err
}
parsed.User = nil
response, evidence, requestErr := c.GetWithAuthenticationEvidence(ctx, parsed.String(), timeoutSeconds, target)
result := AuthenticationEvidence{HTTP: evidence}
if response != nil && response.Body != nil {
defer response.Body.Close()
}
if !evidence.ResponseReceived && response == nil {
return result, requestErr
}
status := evidence.HTTPStatus
if status == nil && response != nil {
status = &response.StatusCode
}
if status == nil {
result.Unrecognized = true
return result, nil
}
switch *status {
case http.StatusUnauthorized:
accepted := false
result.Accepted = &accepted
case http.StatusOK:
if response == nil || response.Body == nil {
result.Unrecognized = true
return result, nil
}
contents, err := io.ReadAll(io.LimitReader(response.Body, maxAuthenticationResponseBytes+1))
if err != nil && ctx.Err() != nil {
return result, ctx.Err()
}
if len(contents) > maxAuthenticationResponseBytes {
result.Overflow = true
return result, nil
}
if err != nil {
return result, err
}
if recognizedOpenRouterKeyResponse(contents) {
accepted := true
result.Accepted = &accepted
} else {
result.Unrecognized = true
}
}
return result, nil
}
func recognizedOpenRouterKeyResponse(contents []byte) bool {
var envelope map[string]json.RawMessage
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
return false
}
var data map[string]json.RawMessage
if err := json.Unmarshal(envelope["data"], &data); err != nil || data == nil {
return false
}
label := bytes.TrimSpace(data["label"])
var labelValue string
if len(label) == 0 || label[0] != '"' || json.Unmarshal(label, &labelValue) != nil {
return false
}
freeTier := bytes.TrimSpace(data["is_free_tier"])
if !bytes.Equal(freeTier, []byte("true")) && !bytes.Equal(freeTier, []byte("false")) {
return false
}
decoder := json.NewDecoder(bytes.NewReader(data["usage"]))
decoder.UseNumber()
var number any
if decoder.Decode(&number) != nil {
return false
}
_, ok := number.(json.Number)
return ok
}

View File

@@ -0,0 +1,134 @@
package llm
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestProbeOpenRouterKey(t *testing.T) {
valid := `{"data":{"label":"x","is_free_tier":false,"usage":0}}`
for _, tc := range []struct {
name string
status int
body string
key string
accepted *bool
unrecognized, overflow bool
}{
{name: "recognized", status: 200, body: valid, key: "secret", accepted: boolValue(true)},
{name: "exact limit", status: 200, body: valid + strings.Repeat(" ", maxAuthenticationResponseBytes-len(valid)), accepted: boolValue(true)},
{name: "unknown", status: 200, body: `{"data":{"label":"x","is_free_tier":false,"usage":"0"}}`, unrecognized: true},
{name: "trailing value", status: 200, body: valid + ` {}`, unrecognized: true},
{name: "overflow", status: 200, body: strings.Repeat("x", maxAuthenticationResponseBytes+1), overflow: true},
{name: "unauthorized anonymous", status: 401, accepted: boolValue(false)},
{name: "other status", status: 503},
} {
t.Run(tc.name, func(t *testing.T) {
var authorization string
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.User != nil {
t.Fatal("request retained URL credentials")
}
authorization = req.Header.Get("Authorization")
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
})}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
if err != nil {
t.Fatal(err)
}
if got.HTTP.Endpoint != "https://example.invalid/key" || got.HTTP.HTTPStatus == nil || *got.HTTP.HTTPStatus != tc.status || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow {
t.Fatalf("evidence = %+v", got)
}
if (got.Accepted == nil) != (tc.accepted == nil) || got.Accepted != nil && *got.Accepted != *tc.accepted {
t.Fatalf("acceptance = %v", got.Accepted)
}
if (authorization != "") != (tc.key != "") || got.HTTP.CredentialsSupplied != (tc.key != "") {
t.Fatalf("credential evidence = %+v", got.HTTP)
}
})
}
}
func boolValue(value bool) *bool { return &value }
type authenticationFailingBody struct{ err error }
func (b authenticationFailingBody) Read([]byte) (int, error) { return 0, b.err }
func (b authenticationFailingBody) Close() error { return nil }
func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
want := errors.New("interrupted")
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
})}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
t.Fatalf("evidence = %+v, error = %v", got, err)
}
}
type observedAuthenticationBody struct {
reader io.Reader
read int
closed bool
}
func (b *observedAuthenticationBody) Read(p []byte) (int, error) {
n, err := b.reader.Read(p)
b.read += n
return n, err
}
func (b *observedAuthenticationBody) Close() error {
b.closed = true
return nil
}
func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
for _, tc := range []struct {
name string
status int
wantRead int
}{
{"overflow", http.StatusOK, maxAuthenticationResponseBytes + 1},
{"unauthorized", http.StatusUnauthorized, 0},
{"other status", http.StatusServiceUnavailable, 0},
} {
t.Run(tc.name, func(t *testing.T) {
body := &observedAuthenticationBody{reader: strings.NewReader(strings.Repeat("x", 2*maxAuthenticationResponseBytes))}
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
})}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
}
})
}
}
func TestOpenRouterKeyEndpoint(t *testing.T) {
got, err := OpenRouterKeyEndpoint("https://example.invalid/api/v1/")
if err != nil || got != "https://example.invalid/api/v1/key" {
t.Fatalf("endpoint = %q, error = %v", got, err)
}
}
func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
<-req.Context().Done()
return nil, req.Context().Err()
})}})
ctx, cancel := context.WithCancel(context.Background())
cancel()
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{})
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
t.Fatalf("evidence = %+v, error = %v", got, err)
}
}

View File

@@ -12,13 +12,7 @@ import (
var prepareGenerationBenchmarkSink *PreparedGeneration var prepareGenerationBenchmarkSink *PreparedGeneration
func BenchmarkPrepareGeneration(b *testing.B) { func BenchmarkPrepareGeneration(b *testing.B) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
BaseURL: "https://api.example.test/v1",
Model: "benchmark-model",
})
if err != nil {
b.Fatalf("construct benchmark client: %v", err)
}
prompt := domain.RenderedPrompt{ prompt := domain.RenderedPrompt{
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}}, Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
@@ -31,14 +25,14 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "no_extras", name: "no_extras",
request: domain.GenerateRequest{ request: domain.GenerateRequest{
Prompt: prompt, Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"}, Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
}, },
}, },
{ {
name: "small_extras", name: "small_extras",
request: domain.GenerateRequest{ request: domain.GenerateRequest{
Prompt: prompt, Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{ Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
"priority": "low", "priority": "low",
"attempt": 3, "attempt": 3,
"enabled": true, "enabled": true,
@@ -49,7 +43,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "1MiB_string_extra", name: "1MiB_string_extra",
request: domain.GenerateRequest{ request: domain.GenerateRequest{
Prompt: prompt, Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{ Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
"payload": strings.Repeat("x", 1<<20), "payload": strings.Repeat("x", 1<<20),
}}, }},
}, },
@@ -58,7 +52,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "nested_json_extras", name: "nested_json_extras",
request: domain.GenerateRequest{ request: domain.GenerateRequest{
Prompt: prompt, Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{ Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model", ExtraParams: map[string]any{
"options": map[string]any{ "options": map[string]any{
"enabled": true, "enabled": true,
"limits": []any{2, 4, 8}, "limits": []any{2, 4, 8},
@@ -71,7 +65,7 @@ func BenchmarkPrepareGeneration(b *testing.B) {
name: "structured_output", name: "structured_output",
request: domain.GenerateRequest{ request: domain.GenerateRequest{
Prompt: prompt, Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"}, Target: domain.ExecutionTarget{Endpoint: "https://api.example.test/v1", Model: "benchmark-model"},
StructuredOutput: &domain.StructuredOutputSpec{ StructuredOutput: &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema, Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{ JSONSchema: &domain.StructuredOutputJSONSpec{

View File

@@ -73,9 +73,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
response := responseFor(request, test.status, "") response := responseFor(request, test.status, "")
response.Body = io.NopCloser(strings.NewReader(test.body)) response.Body = io.NopCloser(strings.NewReader(test.body))
@@ -83,9 +81,6 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
return response, nil return response, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
request := ordinaryGenerateRequest() request := ordinaryGenerateRequest()
if test.structured { if test.structured {
request.StructuredOutput = &domain.StructuredOutputSpec{ request.StructuredOutput = &domain.StructuredOutputSpec{
@@ -136,9 +131,7 @@ func TestPreparedGenerationEvidenceSeparatesResponseAndGenerationFacts(t *testin
func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) { func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}` body := `{"choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":0}}`
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
return &http.Response{ return &http.Response{
StatusCode: http.StatusOK, StatusCode: http.StatusOK,
@@ -149,9 +142,6 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
prepared, err := client.PrepareGeneration(ordinaryGenerateRequest()) prepared, err := client.PrepareGeneration(ordinaryGenerateRequest())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -174,16 +164,11 @@ func TestPreparedGenerationEvidenceRetainsDecodedResponseOnCancellation(t *testi
func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) { func TestPreparedGenerationEvidenceRecordsStructuredRequestAtDispatch(t *testing.T) {
transportErr := errors.New("provider unavailable") transportErr := errors.New("provider unavailable")
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportErr return nil, transportErr
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
request := ordinaryGenerateRequest() request := ordinaryGenerateRequest()
request.StructuredOutput = &domain.StructuredOutputSpec{ request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema, Type: domain.StructuredOutputJSONSchema,

View File

@@ -26,8 +26,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY" const envName = "PROMPTKIT_PREPARED_GENERATION_TEST_KEY"
t.Setenv(envName, tc.initialKey) t.Setenv(envName, tc.initialKey)
var bodies, auth, urls []string var bodies, auth, urls []string
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://example.invalid/v1", Model: "default-model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
body, err := io.ReadAll(req.Body) body, err := io.ReadAll(req.Body)
if err != nil { if err != nil {
@@ -39,12 +38,9 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"ok"}}]}`))}, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
req := domain.GenerateRequest{ req := domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "original"}}},
Target: domain.ExecutionTarget{APIKeyEnv: envName}, Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "default-model", APIKeyEnv: envName},
} }
prepared, err := client.PrepareGeneration(req) prepared, err := client.PrepareGeneration(req)
if err != nil { if err != nil {
@@ -68,8 +64,7 @@ func TestPreparedGenerationFreezesPayloadAndAuthentication(t *testing.T) {
func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) { func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *testing.T) {
var contexts []context.Context var contexts []context.Context
var closed int var closed int
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://example.invalid/v1", Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
if err := req.Context().Err(); err != nil { if err := req.Context().Err(); err != nil {
return nil, err return nil, err
@@ -81,12 +76,9 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
}}, nil }}, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
prepared, err := client.PrepareGeneration(domain.GenerateRequest{ prepared, err := client.PrepareGeneration(domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{TimeoutSeconds: 1}, Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model", TimeoutSeconds: 1},
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -115,17 +107,14 @@ func TestPreparedGenerationAttemptsHaveIndependentContextsAndCloseBodies(t *test
} }
func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) { func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://example.invalid/v1", Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":0},"choices":[]}`))}, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
prepared, err := client.PrepareGeneration(domain.GenerateRequest{ prepared, err := client.PrepareGeneration(domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -137,17 +126,14 @@ func TestPreparedGenerationRetainsUsageFromMalformedChoice(t *testing.T) {
} }
func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) { func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://example.invalid/v1", Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"usage":{"prompt_tokens":9},"choices":[]}{}`))}, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
prepared, err := client.PrepareGeneration(domain.GenerateRequest{ prepared, err := client.PrepareGeneration(domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -160,17 +146,14 @@ func TestPreparedGenerationDoesNotUseInvalidlyFramedUsage(t *testing.T) {
func TestPreparedGenerationRetainsReadCancellation(t *testing.T) { func TestPreparedGenerationRetainsReadCancellation(t *testing.T) {
closed := false closed := false
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://example.invalid/v1", Model: "model",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil return &http.Response{StatusCode: http.StatusOK, Body: &readErrorBody{err: context.Canceled, closed: &closed}}, nil
})}, })},
}) })
if err != nil {
t.Fatal(err)
}
prepared, err := client.PrepareGeneration(domain.GenerateRequest{ prepared, err := client.PrepareGeneration(domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Endpoint: "https://example.invalid/v1", Model: "model"},
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -127,9 +127,7 @@ func TestOpenAICompatibleClientNonSuccessBodyOwnership(t *testing.T) {
func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient { func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int64, body io.ReadCloser) *OpenAICompatibleClient {
t.Helper() t.Helper()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://provider.example/v1",
Model: "m",
HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{ return &http.Response{
StatusCode: statusCode, StatusCode: statusCode,
@@ -138,8 +136,5 @@ func newNonSuccessResponseClient(t *testing.T, statusCode int, contentLength int
}, nil }, nil
})}, })},
}) })
if err != nil {
t.Fatalf("construct client: %v", err)
}
return client return client
} }

View File

@@ -0,0 +1,48 @@
package llm
import (
"errors"
"io"
"strings"
"testing"
)
type responseBytesAndError struct {
data []byte
err error
}
func (r *responseBytesAndError) Read(p []byte) (int, error) {
if len(r.data) == 0 {
return 0, r.err
}
n := copy(p, r.data)
r.data = r.data[n:]
if len(r.data) == 0 {
return n, r.err
}
return n, nil
}
func TestSuccessfulResponseReadFailuresRetainCause(t *testing.T) {
cause := errors.New("private reader diagnostic")
for _, test := range []struct {
name, prefix string
together bool
}{
{"partial", `{"choices":[`, false},
{"complete then failure", successfulProviderResponse, false},
{"bytes and error together", successfulProviderResponse, true},
} {
t.Run(test.name, func(t *testing.T) {
var reader io.Reader = io.MultiReader(strings.NewReader(test.prefix), failingReader{err: cause})
if test.together {
reader = &responseBytesAndError{data: []byte(test.prefix), err: cause}
}
_, err := decodeOpenAIChatResponse(reader)
if !errors.Is(err, ErrTransport) || !errors.Is(err, cause) || errors.Is(err, ErrMalformedResponse) || strings.Contains(err.Error(), cause.Error()) {
t.Fatalf("read failure = %v", err)
}
})
}
}

View File

@@ -13,4 +13,5 @@ var ErrProfileNotFound = errors.New("execution profile not found")
// Repository selects execution profiles from a captured index. // Repository selects execution profiles from a captured index.
type Repository interface { type Repository interface {
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
GetProfileIdentity(ctx context.Context, id string) (string, error)
} }

View File

@@ -24,10 +24,6 @@ type goRenderer struct {
templates TemplateLookup templates TemplateLookup
} }
func NewGoRenderer() Renderer {
return &goRenderer{}
}
// NewCompiledRenderer renders definitions using templates captured with the // NewCompiledRenderer renders definitions using templates captured with the
// configuration snapshot. Per-call clones bind request-specific input helpers. // configuration snapshot. Per-call clones bind request-specific input helpers.
func NewCompiledRenderer(templates TemplateLookup) Renderer { func NewCompiledRenderer(templates TemplateLookup) Renderer {
@@ -56,13 +52,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
funcs := template.FuncMap{ funcs := template.FuncMap{
"input": resolver.resolve, "input": resolver.resolve,
} }
var compiled CompiledTemplates if r.templates == nil {
if r.templates != nil { return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
var found bool }
compiled, found = r.templates.GetCompiledTemplates(definition.ID) compiled, found := r.templates.GetCompiledTemplates(definition.ID)
if !found || len(compiled.Messages) != len(definition.Templates) { if !found || len(compiled.Messages) != len(definition.Templates) {
return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure) return nil, fmt.Errorf("%w: captured templates are unavailable", ErrRenderFailure)
}
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
@@ -70,15 +65,11 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
} }
var sessionID string var sessionID string
var err error var err error
if r.templates != nil { if definition.SessionID != "" {
if definition.SessionID != "" { if compiled.Session == nil {
if compiled.Session == nil { return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
return nil, fmt.Errorf("%w: captured session template is unavailable", ErrRenderFailure)
}
sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
} }
} else { sessionID, err = renderCompiledSessionID(ctx, compiled.Session, funcs, vars)
sessionID, err = renderSessionID(ctx, definition.SessionID, funcs, vars)
} }
if err != nil { if err != nil {
return nil, err return nil, err
@@ -102,18 +93,12 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, err return nil, err
} }
var tmpl *template.Template if compiled.Messages[i] == nil {
var parseErr error return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure)
if r.templates != nil { }
if compiled.Messages[i] == nil { tmpl, parseErr := compiled.Messages[i].Clone()
return nil, fmt.Errorf("%w: captured message template is unavailable", ErrRenderFailure) if parseErr == nil {
} tmpl.Funcs(funcs)
tmpl, parseErr = compiled.Messages[i].Clone()
if parseErr == nil {
tmpl.Funcs(funcs)
}
} else {
tmpl, parseErr = parseTemplate(fmt.Sprintf("msg_%d", i), tmplMsg.Content, funcs)
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, err return nil, err
@@ -201,20 +186,6 @@ func (r *artifactTextResolver) resolve(name string) (string, error) {
return text, nil return text, nil
} }
func renderSessionID(ctx context.Context, raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
if err := ctx.Err(); err != nil {
return "", err
}
tmpl, parseErr := parseTemplate("session_id", raw, funcs)
if err := ctx.Err(); err != nil {
return "", err
}
if parseErr != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, parseErr)
}
return executeSessionTemplate(ctx, tmpl, vars)
}
func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) { func renderCompiledSessionID(ctx context.Context, compiled *template.Template, funcs template.FuncMap, vars map[string]string) (string, error) {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return "", err return "", err

View File

@@ -14,7 +14,6 @@ import (
) )
func TestGoRenderer_Render(t *testing.T) { func TestGoRenderer_Render(t *testing.T) {
renderer := NewGoRenderer()
ctx := context.Background() ctx := context.Background()
inputs := map[string]*domain.Artifact{ inputs := map[string]*domain.Artifact{
@@ -33,7 +32,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -49,11 +48,11 @@ func TestGoRenderer_Render(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{ Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "From file: {{input \"transcript\"}}", ContentFile: "/tmp/user.tmpl"}, {Role: "user", Content: "From file: {{input \"transcript\"}}"},
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -71,7 +70,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -99,7 +98,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -129,7 +128,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -151,7 +150,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -169,7 +168,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, map[string]string{ res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
"tone": "concise", "tone": "concise",
"session_id": "agent-session-123", "session_id": "agent-session-123",
}) })
@@ -190,7 +189,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -208,7 +207,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
_, err := renderer.Render(ctx, def, inputs, vars) _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) { if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err) t.Fatalf("expected ErrRenderFailure, got %v", err)
} }
@@ -223,7 +222,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
_, err := renderer.Render(ctx, def, inputs, map[string]string{ _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
"tone": "concise", "tone": "concise",
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1), "session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
}) })
@@ -241,7 +240,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
_, err := renderer.Render(ctx, def, inputs, map[string]string{ _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
"tone": "concise", "tone": "concise",
"session_id": "session" + string([]byte{0xff}), "session_id": "session" + string([]byte{0xff}),
}) })
@@ -258,7 +257,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -278,7 +277,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
res, err := renderer.Render(ctx, def, inputs, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -298,7 +297,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
_, err := renderer.Render(ctx, def, inputs, vars) _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) { if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err) t.Fatalf("expected ErrRenderFailure, got %v", err)
} }
@@ -315,7 +314,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
_, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars) _, err := capturedTestRenderer(t, def).Render(ctx, def, map[string]*domain.Artifact{}, vars)
if !errors.Is(err, ErrMissingRequiredInput) { if !errors.Is(err, ErrMissingRequiredInput) {
t.Fatalf("expected ErrMissingRequiredInput, got %v", err) t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
} }
@@ -329,9 +328,9 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
_, err := renderer.Render(ctx, def, inputs, vars) _, err := CompileTemplate("message", def.Templates[0].Content)
if !errors.Is(err, ErrInvalidTemplate) { if err == nil {
t.Fatalf("expected ErrInvalidTemplate, got %v", err) t.Fatal("expected template compilation to fail")
} }
}) })
@@ -343,7 +342,7 @@ func TestGoRenderer_Render(t *testing.T) {
}, },
} }
_, err := renderer.Render(ctx, def, inputs, vars) _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) { if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err) t.Fatalf("expected ErrRenderFailure, got %v", err)
} }
@@ -359,7 +358,7 @@ func TestGoRenderer_Render(t *testing.T) {
{Role: "", Content: "Hello"}, {Role: "", Content: "Hello"},
}, },
} }
_, err := renderer.Render(ctx, def, inputs, vars) _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrInvalidMessageRole) { if !errors.Is(err, ErrInvalidMessageRole) {
t.Fatalf("expected ErrInvalidMessageRole, got %v", err) t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
} }
@@ -371,7 +370,7 @@ func TestGoRenderer_Render(t *testing.T) {
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"}, {Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
}, },
} }
res, err := renderer.Render(ctx, def, nil, vars) res, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@@ -385,7 +384,7 @@ func TestGoRenderer_Render(t *testing.T) {
def := &domain.PromptDefinition{ def := &domain.PromptDefinition{
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}}, Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
} }
_, err := renderer.Render(ctx, def, nil, vars) _, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
if !errors.Is(err, ErrInvalidMessageRole) { if !errors.Is(err, ErrInvalidMessageRole) {
t.Fatalf("expected ErrInvalidMessageRole, got %v", err) t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
} }
@@ -396,9 +395,9 @@ func TestGoRenderer_Render(t *testing.T) {
} }
func TestGoRendererCancellation(t *testing.T) { func TestGoRendererCancellation(t *testing.T) {
t.Run("before session parsing", func(t *testing.T) { t.Run("before rendering", func(t *testing.T) {
definition := &domain.PromptDefinition{ definition := &domain.PromptDefinition{
SessionID: "{{ malformed", SessionID: "{{.session}}",
Templates: []domain.PromptMessageTemplate{ Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "not rendered"}, {Role: "user", Content: "not rendered"},
}, },
@@ -406,18 +405,11 @@ func TestGoRendererCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
result, err := NewGoRenderer().Render(ctx, definition, nil, nil) renderer := capturedTestRenderer(t, definition)
result, err := renderer.Render(ctx, definition, nil, nil)
if result != nil || !errors.Is(err, context.Canceled) { if result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err) t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
} }
if errors.Is(err, ErrInvalidTemplate) {
t.Fatalf("pre-canceled render parsed the malformed session: %v", err)
}
result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil)
if result != nil || !errors.Is(err, ErrInvalidTemplate) {
t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err)
}
}) })
t.Run("during artifact text conversion", func(t *testing.T) { t.Run("during artifact text conversion", func(t *testing.T) {
@@ -447,14 +439,15 @@ func TestGoRendererCancellation(t *testing.T) {
}, },
} }
counter := &checkCountingContext{Context: context.Background()} counter := &checkCountingContext{Context: context.Background()}
if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil { renderer := capturedTestRenderer(t, definition)
if _, err := renderer.Render(counter, definition, nil, nil); err != nil {
t.Fatalf("count render checkpoints: %v", err) t.Fatalf("count render checkpoints: %v", err)
} }
// The final three checks occur after template execution, after the // The final three checks occur after template execution, after the
// message is assembled, and immediately before publication. // message is assembled, and immediately before publication.
ctx := newCancelOnCheckContext(counter.checks - 2) ctx := newCancelOnCheckContext(counter.checks - 2)
result, err := NewGoRenderer().Render(ctx, definition, nil, nil) result, err := renderer.Render(ctx, definition, nil, nil)
if result != nil || !errors.Is(err, context.Canceled) { if result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err) t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
} }
@@ -472,7 +465,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
}, },
} }
first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil) renderer := capturedTestRenderer(t, definition)
first, err := renderer.Render(context.Background(), definition, inputs, nil)
if err != nil { if err != nil {
t.Fatalf("first render: %v", err) t.Fatalf("first render: %v", err)
} }
@@ -488,7 +482,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) { if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
t.Fatal("completed render aliases the artifact body") t.Fatal("completed render aliases the artifact body")
} }
second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil) second, err := renderer.Render(context.Background(), definition, inputs, nil)
if err != nil { if err != nil {
t.Fatalf("second render: %v", err) t.Fatalf("second render: %v", err)
} }
@@ -498,7 +492,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
} }
nilInputs := map[string]*domain.Artifact{"document": nil} nilInputs := map[string]*domain.Artifact{"document": nil}
result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil) result, err := renderer.Render(context.Background(), definition, nilInputs, nil)
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) { if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err) t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
} }
@@ -559,11 +553,56 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
} }
} }
func TestCompiledRendererRequiresCapturedTemplates(t *testing.T) {
message, err := CompileTemplate("message", "hello")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
lookup TemplateLookup
sessionID string
}{
{name: "missing lookup"},
{name: "missing prompt", lookup: compiledRendererTestLookup{id: "other"}},
{name: "missing message", lookup: compiledRendererTestLookup{id: "prompt"}},
{name: "nil message", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{nil}}}},
{name: "missing session", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{message}}}, sessionID: "session"},
} {
t.Run(tc.name, func(t *testing.T) {
definition := &domain.PromptDefinition{ID: "prompt", SessionID: tc.sessionID, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hello"}}}
if _, err := NewCompiledRenderer(tc.lookup).Render(context.Background(), definition, nil, nil); !errors.Is(err, ErrRenderFailure) {
t.Fatalf("render error = %v, want render failure", err)
}
})
}
}
type compiledRendererTestLookup struct { type compiledRendererTestLookup struct {
id string id string
templates CompiledTemplates templates CompiledTemplates
} }
func capturedTestRenderer(t testing.TB, definition *domain.PromptDefinition) Renderer {
t.Helper()
var compiled CompiledTemplates
var err error
if definition.SessionID != "" {
compiled.Session, err = CompileTemplate("session", definition.SessionID)
if err != nil {
t.Fatal(err)
}
}
for _, message := range definition.Templates {
parsed, err := CompileTemplate("message", message.Content)
if err != nil {
t.Fatal(err)
}
compiled.Messages = append(compiled.Messages, parsed)
}
return NewCompiledRenderer(compiledRendererTestLookup{id: definition.ID, templates: compiled})
}
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) { func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
if id != lookup.id { if id != lookup.id {
return CompiledTemplates{}, false return CompiledTemplates{}, false
@@ -571,8 +610,7 @@ func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (Compil
return lookup.templates, true return lookup.templates, true
} }
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call // BenchmarkGoRendererArtifactReferences measures rendering captured templates.
// template-parsing fallback rather than the captured compiled path.
func BenchmarkGoRendererArtifactReferences(b *testing.B) { func BenchmarkGoRendererArtifactReferences(b *testing.B) {
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content ")) body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
inputs := map[string]*domain.Artifact{ inputs := map[string]*domain.Artifact{
@@ -604,7 +642,7 @@ func BenchmarkGoRendererArtifactReferences(b *testing.B) {
for _, tc := range tests { for _, tc := range tests {
b.Run(tc.name, func(b *testing.B) { b.Run(tc.name, func(b *testing.B) {
renderer := NewGoRenderer() renderer := capturedTestRenderer(b, tc.definition)
b.ReportAllocs() b.ReportAllocs()
b.SetBytes(int64(len(body))) b.SetBytes(int64(len(body)))
for range b.N { for range b.N {

View File

@@ -319,24 +319,27 @@ func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluat
result := conditionEvaluation{ result := conditionEvaluation{
unsupported: left.unsupported || right.unsupported, unsupported: left.unsupported || right.unsupported,
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown, inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)), missingVariables: mergeOwnedNames(left.missingVariables, right.missingVariables),
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)), contentInputs: mergeOwnedNames(left.contentInputs, right.contentInputs),
}
for name := range left.missingVariables {
result.missingVariables[name] = struct{}{}
}
for name := range right.missingVariables {
result.missingVariables[name] = struct{}{}
}
for name := range left.contentInputs {
result.contentInputs[name] = struct{}{}
}
for name := range right.contentInputs {
result.contentInputs[name] = struct{}{}
} }
return result return result
} }
// Condition results own their sets; callers retain operand values and flags,
// not independent snapshots of these maps after a merge.
func mergeOwnedNames(left, right map[string]struct{}) map[string]struct{} {
if len(left) < len(right) {
left, right = right, left
}
if left == nil {
return right
}
for name := range right {
left[name] = struct{}{}
}
return left
}
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability { func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
if parent == requirementInactive { if parent == requirementInactive {
return requirementInactive return requirementInactive
@@ -448,6 +451,34 @@ func (walker *requirementWalker) walkTemplate(compiled *template.Template) error
return err return err
} }
func (walker *requirementWalker) visitBranch(branch *parse.BranchNode, reach requirementReachability, dot dotIdentity, changesDot bool) error {
condition := conditionEvaluation{}
if reach == requirementInactive {
if err := walker.visit(branch.Pipe, requirementInactive, dot); err != nil {
return err
}
} else {
condition = walker.evaluateCondition(branch.Pipe, reach, dot)
}
before := walker.dependencies
bodyDot := dot
if changesDot {
bodyDot = walker.withDotIdentity(branch.Pipe, dot, condition)
}
thenReach := branchReach(reach, condition.known, condition.truth(), true)
elseReach := branchReach(reach, condition.known, condition.truth(), false)
if err := walker.visit(branch.List, thenReach, bodyDot); err != nil {
return err
}
if err := walker.visit(branch.ElseList, elseReach, dot); err != nil {
return err
}
if reach != requirementInactive && !condition.known && walker.dependencies != before {
walker.recordConditionLimitation(condition)
}
return nil
}
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error { func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
if err := walker.ctx.Err(); err != nil { if err := walker.ctx.Err(); err != nil {
return err return err
@@ -468,55 +499,9 @@ func (walker *requirementWalker) visit(node parse.Node, reach requirementReachab
case *parse.ActionNode: case *parse.ActionNode:
return walker.visit(node.Pipe, reach, dot) return walker.visit(node.Pipe, reach, dot)
case *parse.IfNode: case *parse.IfNode:
if reach == requirementInactive { return walker.visitBranch(&node.BranchNode, reach, dot, false)
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
return err
}
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
return err
}
return walker.visit(node.ElseList, requirementInactive, dot)
}
condition := walker.evaluateCondition(node.Pipe, reach, dot)
before := walker.dependencies
thenReach := branchReach(reach, condition.known, condition.truth(), true)
elseReach := branchReach(reach, condition.known, condition.truth(), false)
if err := walker.visit(node.List, thenReach, dot); err != nil {
return err
}
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
return err
}
if !condition.known && walker.dependencies != before {
walker.recordConditionLimitation(condition)
}
return nil
case *parse.WithNode: case *parse.WithNode:
if reach == requirementInactive { return walker.visitBranch(&node.BranchNode, reach, dot, true)
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
return err
}
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
return err
}
return walker.visit(node.ElseList, requirementInactive, dot)
}
condition := walker.evaluateCondition(node.Pipe, reach, dot)
before := walker.dependencies
thenReach := branchReach(reach, condition.known, condition.truth(), true)
elseReach := branchReach(reach, condition.known, condition.truth(), false)
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
return err
}
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
return err
}
if !condition.known && walker.dependencies != before {
walker.recordConditionLimitation(condition)
}
return nil
case *parse.RangeNode: case *parse.RangeNode:
if err := walker.visit(node.Pipe, reach, dot); err != nil { if err := walker.visit(node.Pipe, reach, dot); err != nil {
return err return err

View File

@@ -314,7 +314,7 @@ func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
"active": {Name: "active", Body: []byte("active")}, "active": {Name: "active", Body: []byte("active")},
"inactive": {Name: "inactive", Body: []byte("inactive")}, "inactive": {Name: "inactive", Body: []byte("inactive")},
} }
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars) rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -406,6 +406,35 @@ func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *tes
} }
} }
func TestConditionEvaluationMergesOwnedEvidence(t *testing.T) {
left := conditionEvaluation{missingVariables: map[string]struct{}{"a": {}, "b": {}}, contentInputs: map[string]struct{}{"document": {}}, unsupported: true}
right := conditionEvaluation{missingVariables: map[string]struct{}{"b": {}, "c": {}}, inputNameUnknown: true}
merged := mergeConditionEvaluations(left, right)
if !merged.unsupported || !merged.inputNameUnknown || len(merged.missingVariables) != 3 || len(merged.contentInputs) != 1 {
t.Fatalf("merged evidence = %+v", merged)
}
transferred := mergeConditionEvaluations(conditionEvaluation{}, conditionEvaluation{missingVariables: map[string]struct{}{"only": {}}})
if len(transferred.missingVariables) != 1 || transferred.contentInputs != nil {
t.Fatalf("transferred evidence = %+v", transferred)
}
}
func TestAnalyzeRequirementsRetainsLeftHeavyMissingConditions(t *testing.T) {
compiled := compileRequirementTemplate(t, "nested", `{{if eq (eq (eq .a .b) .c) .d}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{compiled}}}, nil, false)
if err != nil {
t.Fatal(err)
}
if result.Complete || len(result.Variables) != 4 || len(result.Inputs) != 2 {
t.Fatalf("requirements = %+v", result)
}
for i, name := range []string{"a", "b", "c", "d"} {
if result.Variables[i].Name != name {
t.Fatalf("variables = %+v", result.Variables)
}
}
}
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) { func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
description := "declared for a reason" description := "declared for a reason"
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{ definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
@@ -497,7 +526,7 @@ func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables) t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
} }
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}} artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars) rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -5,6 +5,7 @@ import (
"errors" "errors"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/prompt"
) )
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition. // ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
@@ -12,5 +13,7 @@ var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
// Repository selects prompt definitions from a captured index. // Repository selects prompt definitions from a captured index.
type Repository interface { type Repository interface {
GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
} }

View File

@@ -0,0 +1,38 @@
package usecase
import (
"errors"
"fmt"
"go.maximumdirect.net/promptkit/internal/profile"
"go.maximumdirect.net/promptkit/internal/promptdef"
)
// OperationError identifies the operation that owns a failure while retaining
// the original cause for inspection.
type OperationError struct {
Category error
Cause error
}
func (e *OperationError) Error() string { return fmt.Sprintf("%v: %v", e.Category, e.Cause) }
func (e *OperationError) Unwrap() error { return e.Cause }
func (e *OperationError) Is(target error) bool { return target == e.Category }
func promptLookupError(cause error) error {
category := ErrPromptLoad
if errors.Is(cause, promptdef.ErrPromptDefinitionNotFound) {
category = promptdef.ErrPromptDefinitionNotFound
cause = fmt.Errorf("%w: %w", ErrPromptLoad, cause)
}
return &OperationError{Category: category, Cause: cause}
}
func profileLookupError(cause error) error {
category := ErrProfileLoad
if errors.Is(cause, profile.ErrProfileNotFound) {
category = profile.ErrProfileNotFound
cause = fmt.Errorf("%w: %w", ErrProfileLoad, cause)
}
return &OperationError{Category: category, Cause: cause}
}

View File

@@ -0,0 +1,34 @@
package usecase
import (
"errors"
"fmt"
"testing"
"go.maximumdirect.net/promptkit/internal/profile"
"go.maximumdirect.net/promptkit/internal/promptdef"
)
func TestOperationErrorRetainsOwnerAndCause(t *testing.T) {
cause := fmt.Errorf("nested: %w", promptdef.ErrPromptDefinitionNotFound)
outer := &OperationError{Category: ErrLLMGenerate, Cause: cause}
if !errors.Is(outer, ErrLLMGenerate) || !errors.Is(outer, promptdef.ErrPromptDefinitionNotFound) || errors.Unwrap(outer) != cause {
t.Fatalf("operation error = %v", outer)
}
for _, test := range []struct {
err error
category error
}{
{promptLookupError(cause), promptdef.ErrPromptDefinitionNotFound},
{profileLookupError(profile.ErrProfileNotFound), profile.ErrProfileNotFound},
{promptLookupError(contextFailure), ErrPromptLoad},
{profileLookupError(contextFailure), ErrProfileLoad},
} {
var operation *OperationError
if !errors.As(test.err, &operation) || operation.Category != test.category {
t.Errorf("operation = %+v", operation)
}
}
}
var contextFailure = errors.New("lookup failed")

View File

@@ -172,14 +172,13 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) { func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
t.Helper() t.Helper()
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 || if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
collaborators.validator.prepareCalls != 0 || collaborators.validator.directValidateCalls != 0 || collaborators.validator.prepareCalls != 0 ||
len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 { len(collaborators.admitter.backendIDs) != 0 || collaborators.llm.calls != 0 {
t.Fatalf( t.Fatalf(
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d validation=%d admissions=%d generation=%d", "invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d admissions=%d generation=%d",
collaborators.artifacts.calls, collaborators.artifacts.calls,
collaborators.renderer.calls, collaborators.renderer.calls,
collaborators.validator.prepareCalls, collaborators.validator.prepareCalls,
collaborators.validator.directValidateCalls,
len(collaborators.admitter.backendIDs), len(collaborators.admitter.backendIDs),
collaborators.llm.calls, collaborators.llm.calls,
) )

View File

@@ -7,7 +7,6 @@ import (
"time" "time"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/observation" "go.maximumdirect.net/promptkit/internal/observation"
"go.maximumdirect.net/promptkit/internal/validate" "go.maximumdirect.net/promptkit/internal/validate"
) )
@@ -22,7 +21,7 @@ const (
// PreparedExecution owns one frozen, single-use runner execution. // PreparedExecution owns one frozen, single-use runner execution.
type PreparedExecution struct { type PreparedExecution struct {
owner *Runner owner *runnerIdentity
mu sync.Mutex mu sync.Mutex
state preparedExecutionState state preparedExecutionState
details *domain.PreparedRun details *domain.PreparedRun
@@ -50,24 +49,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
return nil, err return nil, err
} }
executionSnapshot := operation.run
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
if err != nil {
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
}
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
details, err := clonePreparedRun(executionSnapshot)
if err != nil {
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
}
return &PreparedExecution{ return &PreparedExecution{
owner: r, owner: r.identity,
state: preparedExecutionReady, state: preparedExecutionReady,
details: details, details: operation.run,
payload: &preparedExecutionPayload{ payload: &preparedExecutionPayload{
prepared: executionSnapshot, prepared: operation.run,
validation: operation.validation, validation: operation.validation,
directKey: state.effectiveModel.APIKey, directKey: state.effectiveModel.APIKey,
transientRetries: state.transientRetries, transientRetries: state.transientRetries,
@@ -76,7 +63,8 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
}, nil }, nil
} }
// Details returns a fresh credential-redacted copy of the prepared run. // Details returns immutable, credential-redacted metadata borrowed from the
// handle. Callers must copy it before exposing it to mutable external code.
func (p *PreparedExecution) Details() *domain.PreparedRun { func (p *PreparedExecution) Details() *domain.PreparedRun {
if p == nil { if p == nil {
return nil return nil
@@ -85,11 +73,7 @@ func (p *PreparedExecution) Details() *domain.PreparedRun {
detailsSnapshot := p.details detailsSnapshot := p.details
p.mu.Unlock() p.mu.Unlock()
details, err := clonePreparedRun(detailsSnapshot) return detailsSnapshot
if err != nil {
return nil
}
return details
} }
// Discard invalidates an unclaimed execution and drops its private payload. // Discard invalidates an unclaimed execution and drops its private payload.
@@ -170,7 +154,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
} }
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) { func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
if p == nil || owner == nil || p.owner != owner { if p == nil || owner == nil || owner.identity == nil || p.owner != owner.identity {
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest) return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
} }
@@ -189,9 +173,6 @@ func (p *preparedExecutionPayload) clear() {
if p == nil { if p == nil {
return return
} }
if p.prepared != nil {
p.prepared.EffectiveModelParams.APIKey = ""
}
p.prepared = nil p.prepared = nil
p.validation = nil p.validation = nil
p.directKey = "" p.directKey = ""
@@ -222,43 +203,3 @@ func (p noOpPreparedValidation) Validate(
func (noOpPreparedValidation) SchemaDocument() any { func (noOpPreparedValidation) SchemaDocument() any {
return nil return nil
} }
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
if source == nil {
return nil, nil
}
copied := *source
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
if err != nil {
return nil, err
}
copied.EffectiveModelParams.ExtraParams = extraParams
if source.InputHashes != nil {
copied.InputHashes = make(map[string]string, len(source.InputHashes))
for name, hash := range source.InputHashes {
copied.InputHashes[name] = hash
}
}
copied.Messages = domain.CloneRenderedMessages(source.Messages)
if source.StructuredOutput != nil {
structuredOutput := *source.StructuredOutput
copied.StructuredOutput = &structuredOutput
if source.StructuredOutput.JSONSchema != nil {
jsonSchema := *source.StructuredOutput.JSONSchema
copied.StructuredOutput.JSONSchema = &jsonSchema
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
if copyErr != nil {
return nil, copyErr
}
copied.StructuredOutput.JSONSchema.Schema = schema
}
}
return &copied, nil
}
func cloneJSONValue(source any) (any, error) {
return jsonvalue.Copy(source)
}

View File

@@ -8,6 +8,7 @@ import (
"testing" "testing"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
"go.maximumdirect.net/promptkit/internal/validate" "go.maximumdirect.net/promptkit/internal/validate"
) )
@@ -47,25 +48,15 @@ func (p *recordingPreparedValidation) SchemaDocument() any {
} }
type recordingValidationPreparer struct { type recordingValidationPreparer struct {
plan *recordingPreparedValidation plan *recordingPreparedValidation
prepareErr error prepareErr error
prepareCalls int prepareCalls int
directValidateCalls int
} }
type validationOnly struct{} type nilPlanPreparer struct{}
func (validationOnly) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) { func (nilPlanPreparer) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
return domain.ValidationResult{}, nil return nil, nil
}
func (v *recordingValidationPreparer) Validate(
context.Context,
*domain.Artifact,
domain.OutputContract,
) (domain.ValidationResult, error) {
v.directValidateCalls++
return domain.ValidationResult{}, errors.New("live validation must not be used")
} }
func (v *recordingValidationPreparer) PrepareValidation( func (v *recordingValidationPreparer) PrepareValidation(
@@ -129,12 +120,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
} }
defer prepared.Discard() defer prepared.Discard()
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 { if validator.prepareCalls != 1 {
t.Fatalf( t.Fatalf("validation preparations = %d, want 1", validator.prepareCalls)
"validation calls=(prepare=%d direct=%d), want (1, 0)",
validator.prepareCalls,
validator.directValidateCalls,
)
} }
if reader.calls != 1 || renderer.calls != 1 { if reader.calls != 1 || renderer.calls != 1 {
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls) t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
@@ -156,90 +143,37 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput) t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
} }
first.Messages[0].Content = "caller mutation"
first.InputHashes["input"] = "caller mutation"
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
renderer.rendered.Messages[0].Content = "source mutation" renderer.rendered.Messages[0].Content = "source mutation"
second := prepared.Details() second := prepared.Details()
if second.Messages[0].Content != "original message" || if second != first || second.Messages[0].Content != "original message" ||
second.InputHashes["input"] == "caller mutation" ||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" || second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" { second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
t.Fatalf("details did not preserve an independent snapshot: %#v", second) t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
} }
} }
func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) { func TestRunnerPreparationDetachesDeepPreparedSchema(t *testing.T) {
operations := []struct { var nested any = "original"
name string for range 150 {
run func(*Runner, domain.RunRequest) error nested = map[string]any{"child": nested}
}{
{
name: "Prepare",
run: func(runner *Runner, request domain.RunRequest) error {
_, err := runner.Prepare(context.Background(), request)
return err
},
},
{
name: "Run",
run: func(runner *Runner, request domain.RunRequest) error {
_, err := runner.Run(context.Background(), request)
return err
},
},
{
name: "PrepareExecution",
run: func(runner *Runner, request domain.RunRequest) error {
_, err := runner.PrepareExecution(context.Background(), request)
return err
},
},
} }
schema := map[string]any{"annotation": nested}
for _, operation := range operations { def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
t.Run(operation.name, func(t *testing.T) { def.Validation.SchemaPath = "schema.json"
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0) validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{schemaDocument: schema}}
def.Validation.SchemaPath = "schema.json" runner := NewRunner(&fakePromptRepo{def: def},
llmClient := &fakeLLM{forbid: true} &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
validator := &recordingValidationPreparer{ defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, validator, nil)
plan: &recordingPreparedValidation{schemaDocument: excessivelyDeepPreparedJSONValue()}, prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
} if err != nil {
runner := NewRunner( t.Fatalf("prepare execution: %v", err)
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
nil,
)
err := operation.run(runner, domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrValidation) {
t.Fatalf("expected ErrValidation, got %v", err)
}
if llmClient.calls != 0 {
t.Fatalf("invalid prepared schema reached generation: %d calls", llmClient.calls)
}
})
} }
} schema["annotation"].(map[string]any)["child"] = "source mutation"
first := prepared.Details().StructuredOutput.JSONSchema.Schema.(map[string]any)
func excessivelyDeepPreparedJSONValue() any { if _, ok := first["annotation"].(map[string]any)["child"].(map[string]any); !ok {
const clearlyUnsafeContainerDepth = 1_000 t.Fatal("prepared schema aliases the validator document")
var value any = true
for level := 0; level < clearlyUnsafeContainerDepth; level++ {
value = map[string]any{"child": value}
} }
return value
} }
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) { func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
@@ -421,8 +355,8 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) { if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts) t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
} }
if validator.directValidateCalls != 0 || repairer.calls != 1 { if repairer.calls != 1 {
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls) t.Fatalf("repair calls = %d, want 1", repairer.calls)
} }
if len(client.requests) != 1 || len(repairer.reqs) != 1 || if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) { !reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
@@ -498,7 +432,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
}}, }},
}, },
repairer: &fakeRepairer{err: repairInvalidRequest}, repairer: &fakeRepairer{err: repairInvalidRequest},
wantError: ErrInvalidRequest, wantError: ErrLLMGenerate,
wantSource: repairInvalidRequest, wantSource: repairInvalidRequest,
}, },
{ {
@@ -600,6 +534,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
} }
owner := newRunner() owner := newRunner()
if other := newRunner(); owner.identity == nil || owner.identity == other.identity {
t.Fatal("constructed runners must have independent identity tokens")
}
prepared, err := owner.PrepareExecution(context.Background(), request) prepared, err := owner.PrepareExecution(context.Background(), request)
if err != nil { if err != nil {
t.Fatalf("prepare execution: %v", err) t.Fatalf("prepare execution: %v", err)
@@ -616,6 +553,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
if prepared.Details() == nil { if prepared.Details() == nil {
t.Fatal("details unavailable after execution") t.Fatal("details unavailable after execution")
} }
if prepared.owner != owner.identity || prepared.payload != nil {
t.Fatal("claimed handle retained payload or lost owner identity")
}
discarded, err := owner.PrepareExecution(context.Background(), request) discarded, err := owner.PrepareExecution(context.Background(), request)
if err != nil { if err != nil {
@@ -623,6 +563,9 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
} }
discarded.Discard() discarded.Discard()
discarded.Discard() discarded.Discard()
if discarded.owner != owner.identity || discarded.payload != nil {
t.Fatal("discarded handle retained payload or lost owner identity")
}
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) { if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err) t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
} }
@@ -687,7 +630,7 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
} }
wantLabel := "profile source" wantLabel := "profile source"
if test.requestOverride { if test.requestOverride {
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras} request.Execution = &domain.ExecutionTargetOverride{ExtraParams: jsonvalue.CloneTrusted(requestExtras).(map[string]any)}
wantLabel = "request source" wantLabel = "request source"
} }
@@ -711,11 +654,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" { details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
t.Fatalf("prepared details retained a source alias: %#v", details) t.Fatalf("prepared details retained a source alias: %#v", details)
} }
details.Messages[0].Content = "mutated details"
if again := prepared.Details(); again == nil || len(again.Messages) != 1 || again.Messages[0].Content != "captured rendering" {
t.Fatalf("details mutation escaped into the retained snapshot: %+v", again)
}
result, err := runner.RunPrepared(context.Background(), prepared) result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" { if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err) t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
@@ -737,22 +675,6 @@ func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T)
} }
} }
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
profile := defaultExecutionProfile()
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec",
})
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
}
}
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) { func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)}, &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
@@ -782,7 +704,7 @@ func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
} }
} }
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) { func TestRunnerPrepareExecutionRejectsNilValidationPlan(t *testing.T) {
reader := defaultArtifactReader() reader := defaultArtifactReader()
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
@@ -791,7 +713,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
reader, reader,
defaultRenderer(), defaultRenderer(),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
validationOnly{}, nilPlanPreparer{},
nil, nil,
) )

View File

@@ -7,7 +7,6 @@ import (
"strings" "strings"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
) )
type resolvedProfileSelection struct { type resolvedProfileSelection struct {
@@ -15,10 +14,6 @@ type resolvedProfileSelection struct {
profile *domain.ExecutionProfile profile *domain.ExecutionProfile
} }
type profileIdentityRepository interface {
GetProfileIdentity(context.Context, string) (string, error)
}
func (r *Runner) profileLookupID(profileID string) (string, error) { func (r *Runner) profileLookupID(profileID string) (string, error) {
if strings.TrimSpace(profileID) == "" { if strings.TrimSpace(profileID) == "" {
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest) return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
@@ -39,7 +34,7 @@ func (r *Runner) resolveProfileSelection(
} }
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID) selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) return nil, profileLookupError(err)
} }
if selectedProfile == nil { if selectedProfile == nil {
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad) return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
@@ -61,18 +56,11 @@ func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (
if err != nil { if err != nil {
return "", err return "", err
} }
if indexed, ok := r.profiles.(profileIdentityRepository); ok { identity, err := r.profiles.GetProfileIdentity(ctx, lookupID)
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
if err != nil {
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
return identity, nil
}
selection, err := r.resolveProfileSelection(ctx, lookupID)
if err != nil { if err != nil {
return "", err return "", profileLookupError(err)
} }
return selection.id, nil return identity, nil
} }
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) { func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
@@ -108,7 +96,7 @@ func (r *Runner) InspectProfile(
} }
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err()) return nil, &OperationError{Category: ErrProfileLoad, Cause: ctx.Err()}
default: default:
} }
@@ -118,12 +106,6 @@ func (r *Runner) InspectProfile(
} }
target, _ := resolveExecutionTarget(selection.profile, nil) target, _ := resolveExecutionTarget(selection.profile, nil)
target.APIKey = "" target.APIKey = ""
params, err := jsonvalue.CopyMap(target.ExtraParams)
if err != nil {
return nil, fmt.Errorf("%w: profile extra parameters: %w", ErrProfileLoad, err)
}
target.ExtraParams = params
return &domain.ProfileInspection{ return &domain.ProfileInspection{
ProfileID: selection.id, ProfileID: selection.id,
EffectiveModelParams: target, EffectiveModelParams: target,

View File

@@ -7,6 +7,7 @@ import (
"testing" "testing"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/profile" "go.maximumdirect.net/promptkit/internal/profile"
) )
@@ -23,7 +24,21 @@ func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (
if r.err != nil { if r.err != nil {
return nil, r.err return nil, r.err
} }
return r.profile, nil if r.profile == nil {
return nil, nil
}
profile := *r.profile
if profile.ResolvedTarget != nil {
target := *profile.ResolvedTarget
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
profile.ResolvedTarget = &target
}
return &profile, nil
}
func (r *inspectionProfileRepository) GetProfileIdentity(ctx context.Context, id string) (string, error) {
_, err := r.GetProfile(ctx, id)
return id, err
} }
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) { func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {

View File

@@ -13,10 +13,6 @@ type resolvedPromptDefinition struct {
hash string hash string
} }
type promptInspectionRepository interface {
GetPromptInspection(context.Context, string) (*domain.PromptInspection, error)
}
func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) { func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (*resolvedPromptDefinition, error) {
if strings.TrimSpace(promptID) == "" { if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
@@ -25,29 +21,14 @@ func (r *Runner) resolvePromptDefinition(ctx context.Context, promptID string) (
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad) return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
} }
var definition *domain.PromptDefinition definition, hash, err := r.promptDefs.GetPromptSnapshot(ctx, promptID)
var hash string
var err error
if indexed, ok := r.promptDefs.(interface {
GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error)
}); ok {
definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID)
} else {
definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID)
}
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) return nil, promptLookupError(err)
} }
if definition == nil { if definition == nil {
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad) return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
} }
if hash == "" {
hash, err = hashPromptDefinition(definition)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
}
}
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
} }
@@ -58,29 +39,14 @@ func (r *Runner) resolvePromptInspection(ctx context.Context, promptID string) (
if r == nil || r.promptDefs == nil { if r == nil || r.promptDefs == nil {
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad) return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
} }
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok { inspection, err := r.promptDefs.GetPromptInspection(ctx, promptID)
inspection, err := indexed.GetPromptInspection(ctx, promptID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
if inspection == nil {
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
}
return inspection, nil
}
selection, err := r.resolvePromptDefinition(ctx, promptID)
if err != nil { if err != nil {
return nil, err return nil, promptLookupError(err)
} }
definition := selection.definition if inspection == nil {
return &domain.PromptInspection{ return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
PromptID: definition.ID, }
PromptHash: selection.hash, return inspection, nil
DefaultProfileID: definition.DefaultProfile,
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
OutputContract: definition.Validation,
}, nil
} }
// InspectPrompt resolves one explicit prompt without execution work. // InspectPrompt resolves one explicit prompt without execution work.
@@ -90,7 +56,7 @@ func (r *Runner) InspectPrompt(ctx context.Context, promptID string) (*domain.Pr
} }
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err()) return nil, &OperationError{Category: ErrPromptLoad, Cause: ctx.Err()}
default: default:
} }

View File

@@ -7,6 +7,7 @@ import (
"testing" "testing"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/prompt"
"go.maximumdirect.net/promptkit/internal/promptdef" "go.maximumdirect.net/promptkit/internal/promptdef"
) )
@@ -23,20 +24,33 @@ type promptMetadataRepository struct {
lookupID string lookupID string
} }
func (r *inspectionPromptRepository) GetPromptDefinition( func (r *inspectionPromptRepository) GetPromptSnapshot(
_ context.Context, _ context.Context,
id string, id string,
) (*domain.PromptDefinition, error) { ) (*domain.PromptDefinition, string, error) {
r.calls++ r.calls++
r.id = id r.id = id
if r.err != nil { if r.err != nil {
return nil, r.err return nil, "", r.err
} }
return r.definition, nil return r.definition, "captured-hash", nil
} }
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string) (*domain.PromptDefinition, error) { func (r *inspectionPromptRepository) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
return nil, errors.New("full prompt definition should not be requested") definition, hash, err := r.GetPromptSnapshot(ctx, id)
return fixturePromptInspection(definition, hash), err
}
func (*inspectionPromptRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
return prompt.RequirementSnapshot{}, false
}
func (r *promptMetadataRepository) GetPromptSnapshot(context.Context, string) (*domain.PromptDefinition, string, error) {
return nil, "", errors.New("full prompt definition should not be requested")
}
func (*promptMetadataRepository) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
return prompt.RequirementSnapshot{}, false
} }
func (r *promptMetadataRepository) GetPromptInspection( func (r *promptMetadataRepository) GetPromptInspection(
@@ -71,15 +85,11 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
if err != nil { if err != nil {
t.Fatalf("inspect prompt: %v", err) t.Fatalf("inspect prompt: %v", err)
} }
wantHash, err := hashPromptDefinition(definition)
if err != nil {
t.Fatalf("hash prompt definition: %v", err)
}
if repository.calls != 1 || repository.id != " prompt-id " { if repository.calls != 1 || repository.id != " prompt-id " {
t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id) t.Fatalf("prompt lookup=(calls=%d id=%q), want one unchanged ID lookup", repository.calls, repository.id)
} }
if inspection.PromptID != definition.ID || if inspection.PromptID != definition.ID ||
inspection.PromptHash != wantHash || inspection.PromptHash != "captured-hash" ||
inspection.DefaultProfileID != definition.DefaultProfile || inspection.DefaultProfileID != definition.DefaultProfile ||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) || !reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
inspection.OutputContract != definition.Validation { inspection.OutputContract != definition.Validation {
@@ -96,7 +106,7 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
} }
} }
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) { func TestRunnerInspectPromptUsesOwnedCapturedMetadata(t *testing.T) {
repository := &promptMetadataRepository{inspection: domain.PromptInspection{ repository := &promptMetadataRepository{inspection: domain.PromptInspection{
PromptID: "captured.prompt", PromptHash: "cached-prompt-hash", PromptID: "captured.prompt", PromptHash: "cached-prompt-hash",
DefaultProfileID: "partial-profile", DefaultProfileID: "partial-profile",
@@ -119,7 +129,7 @@ func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing
} }
} }
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) { func TestRunnerInspectPromptUsesCapturedHashAndMetadata(t *testing.T) {
repository := &requirementInspectionPromptRepository{ repository := &requirementInspectionPromptRepository{
definition: &domain.PromptDefinition{ definition: &domain.PromptDefinition{
ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}}, ID: "captured.prompt", Inputs: []domain.PromptInput{{Name: "document"}},
@@ -128,7 +138,7 @@ func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *te
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ") inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ")
if err != nil || inspection.PromptID != "captured.prompt" || if err != nil || inspection.PromptID != "captured.prompt" ||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" { inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err) t.Fatalf("captured inspection=(%+v, %v)", inspection, err)
} }
} }

View File

@@ -0,0 +1,48 @@
package usecase
import (
"encoding/json"
"strings"
"unicode/utf8"
)
// boundedDiagnosticJSON returns a JSON string prefix and whether it represents
// the complete normalized input. It retains at most the useful prefix.
func boundedDiagnosticJSON(value string, maxBytes int) ([]byte, bool) {
if maxBytes < 2 {
return nil, false
}
limit := maxBytes - 2
var prefix strings.Builder
used := 0
more := false
for i := 0; i < len(value); {
r, size := utf8.DecodeRuneInString(value[i:])
piece := value[i : i+size]
if r == utf8.RuneError && size == 1 {
piece = "\uFFFD"
i++
for i < len(value) {
r, size = utf8.DecodeRuneInString(value[i:])
if r != utf8.RuneError || size != 1 {
break
}
i++
}
} else {
i += size
}
if used+len(piece) > limit {
more = true
break
}
prefix.WriteString(piece)
used += len(piece)
}
normalized := prefix.String()
encoded, _ := json.Marshal(normalized)
if len(encoded) > maxBytes {
return truncateDiagnosticJSONValue(normalized, maxBytes), false
}
return encoded, !more
}

View File

@@ -0,0 +1,33 @@
package usecase
import (
"encoding/json"
"strings"
"testing"
"unicode/utf8"
)
func TestBoundedDiagnosticJSON(t *testing.T) {
for _, input := range []string{"", "plain", "<&>\n\"", "雪☃", "a\xff\xfe\xffb", "\xff\xfe" + strings.Repeat("z", 100)} {
for budget := 0; budget <= 30; budget++ {
encoded, complete := boundedDiagnosticJSON(input, budget)
if budget < 2 {
if encoded != nil || complete {
t.Fatalf("budget %d: %q %v", budget, encoded, complete)
}
continue
}
if len(encoded) > budget || !utf8.Valid(encoded) {
t.Fatalf("budget %d: %q", budget, encoded)
}
var decoded string
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
normalized := strings.ToValidUTF8(input, "\uFFFD")
if !strings.HasPrefix(normalized, decoded) || complete != (decoded == normalized) {
t.Fatalf("input %q budget %d: %q complete=%v normalized=%q", input, budget, decoded, complete, normalized)
}
}
}
}

View File

@@ -117,27 +117,38 @@ func repairGuidance(mode domain.ValidationMode) (string, error) {
} }
func formatRepairDiagnostics(errors []string) string { func formatRepairDiagnostics(errors []string) string {
diagnostics := make([]string, len(errors)) complete := []byte{'['}
for index, diagnostic := range errors { fits := true
diagnostics[index] = strings.ToValidUTF8(diagnostic, "\uFFFD") for _, diagnostic := range errors {
separator := 0
if len(complete) > 1 {
separator = 1
}
entry, whole := boundedDiagnosticJSON(diagnostic, maxRepairDiagnosticBytes-len(complete)-separator-1)
if !whole {
fits = false
break
}
if separator != 0 {
complete = append(complete, ',')
}
complete = append(complete, entry...)
} }
if fits {
complete, _ := json.Marshal(diagnostics) return string(append(complete, ']'))
if len(complete) <= maxRepairDiagnosticBytes {
return string(complete)
} }
omission, _ := json.Marshal(omittedRepairDiagnostics) omission, _ := json.Marshal(omittedRepairDiagnostics)
encoded := make([]byte, 0, maxRepairDiagnosticBytes) encoded := make([]byte, 0, maxRepairDiagnosticBytes)
encoded = append(encoded, '[') encoded = append(encoded, '[')
for _, diagnostic := range diagnostics { for _, diagnostic := range errors {
entry, _ := json.Marshal(diagnostic)
separator := 0 separator := 0
if len(encoded) > 1 { if len(encoded) > 1 {
separator = 1 separator = 1
} }
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1 available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
if len(entry) <= available { entry, whole := boundedDiagnosticJSON(diagnostic, available)
if whole {
if separator != 0 { if separator != 0 {
encoded = append(encoded, ',') encoded = append(encoded, ',')
} }
@@ -151,7 +162,7 @@ func formatRepairDiagnostics(errors []string) string {
if separator != 0 { if separator != 0 {
encoded = append(encoded, ',') encoded = append(encoded, ',')
} }
encoded = append(encoded, truncateDiagnosticJSONValue(diagnostic, available)...) encoded = append(encoded, entry...)
break break
} }
if len(encoded) > 1 { if len(encoded) > 1 {

View File

@@ -298,6 +298,23 @@ func TestFormatRepairDiagnosticsBoundsAndPreservesData(t *testing.T) {
} }
} }
func TestFormatRepairDiagnosticsEmptyAndEscapedOversize(t *testing.T) {
for _, values := range [][]string{nil, {}} {
if got := formatRepairDiagnostics(values); got != "[]" {
t.Fatalf("empty diagnostics = %q", got)
}
}
value := strings.Repeat("<&>\x00\xff\xfe", maxRepairDiagnosticBytes)
encoded := formatRepairDiagnostics([]string{value})
var got []string
if err := json.Unmarshal([]byte(encoded), &got); err != nil {
t.Fatal(err)
}
if len(encoded) > maxRepairDiagnosticBytes || len(got) != 2 || got[1] != omittedRepairDiagnostics || !strings.HasPrefix(strings.ToValidUTF8(value, "\uFFFD"), got[0]) {
t.Fatalf("oversize diagnostics length=%d, values=%d", len(encoded), len(got))
}
}
func manyRepairDiagnostics() []string { func manyRepairDiagnostics() []string {
diagnostics := make([]string, 1_000) diagnostics := make([]string, 1_000)
for index := range diagnostics { for index := range diagnostics {

View File

@@ -18,10 +18,6 @@ type RequirementInspectionRequest struct {
SessionID string SessionID string
} }
type requirementSnapshotRepository interface {
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
}
// InspectRequirements discovers prompt inputs and caller variables without // InspectRequirements discovers prompt inputs and caller variables without
// loading artifacts or performing execution preparation. // loading artifacts or performing execution preparation.
func (r *Runner) InspectRequirements( func (r *Runner) InspectRequirements(
@@ -36,7 +32,7 @@ func (r *Runner) InspectRequirements(
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
} }
selection, err := r.resolvePromptInspection(ctx, req.PromptID) selection, err := r.resolvePromptInspection(ctx, req.PromptID)
@@ -44,7 +40,7 @@ func (r *Runner) InspectRequirements(
return nil, err return nil, err
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) return nil, &OperationError{Category: ErrProfileLoad, Cause: err}
} }
selectedProfileID := req.ProfileID selectedProfileID := req.ProfileID
if strings.TrimSpace(selectedProfileID) == "" { if strings.TrimSpace(selectedProfileID) == "" {
@@ -58,21 +54,17 @@ func (r *Runner) InspectRequirements(
return nil, err return nil, err
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
} }
repository, ok := r.promptDefs.(requirementSnapshotRepository) snapshot, ok := r.promptDefs.GetRequirementSnapshot(selection.PromptID)
if !ok {
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
}
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID)
if !ok { if !ok {
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad) return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
} }
definition := &domain.PromptDefinition{ID: selection.PromptID} definition := &domain.PromptDefinition{ID: selection.PromptID}
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "") inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err) return nil, &OperationError{Category: ErrPromptLoad, Cause: err}
} }
inspection.ProfileID = profileIdentity inspection.ProfileID = profileIdentity
return &inspection, nil return &inspection, nil

View File

@@ -45,13 +45,6 @@ func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
return repository.identity, repository.identityErr return repository.identity, repository.identityErr
} }
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
context.Context,
string,
) (*domain.PromptDefinition, error) {
return repository.definition, repository.lookupErr
}
func (repository *requirementPromptMetadataRepository) GetPromptInspection( func (repository *requirementPromptMetadataRepository) GetPromptInspection(
_ context.Context, _ context.Context,
id string, id string,
@@ -71,6 +64,11 @@ func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
return repository.definition, "captured-hash", repository.lookupErr return repository.definition, "captured-hash", repository.lookupErr
} }
func (repository *requirementInspectionPromptRepository) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
definition, hash, err := repository.GetPromptSnapshot(ctx, id)
return fixturePromptInspection(definition, hash), err
}
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot( func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
id string, id string,
) (prompt.RequirementSnapshot, bool) { ) (prompt.RequirementSnapshot, bool) {
@@ -163,7 +161,7 @@ func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(
} }
} }
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) { func TestRunnerInspectRequirementsUsesProfileIdentity(t *testing.T) {
message, err := prompt.CompileTemplate("message", `{{.name}}`) message, err := prompt.CompileTemplate("message", `{{.name}}`)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -177,7 +175,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
} }
requestedID := " literal-profile " requestedID := " literal-profile "
t.Run("optional capability", func(t *testing.T) { t.Run("identity lookup", func(t *testing.T) {
profiles := &inspectionProfileIdentityRepository{ profiles := &inspectionProfileIdentityRepository{
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{}, ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
@@ -196,21 +194,7 @@ func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *tes
} }
}) })
t.Run("full-profile fallback", func(t *testing.T) { t.Run("identity failure", func(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
}}
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
},
)
if err != nil || inspection.ProfileID != requestedID || profiles.calls != 1 || profiles.id != requestedID {
t.Fatalf("fallback inspection=(%+v, %v), full lookup=(%d,%q)", inspection, err, profiles.calls, profiles.id)
}
})
t.Run("capability failure", func(t *testing.T) {
lookupErr := errors.New("profile identity lookup failed") lookupErr := errors.New("profile identity lookup failed")
profiles := &inspectionProfileIdentityRepository{ profiles := &inspectionProfileIdentityRepository{
inspectionProfileRepository: &inspectionProfileRepository{}, inspectionProfileRepository: &inspectionProfileRepository{},
@@ -329,14 +313,6 @@ func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *test
} }
}) })
t.Run("repository lacks snapshot capability", func(t *testing.T) {
_, err := (&Runner{promptDefs: &inspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}},
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("missing capability error=%v", err)
}
})
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) { if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
t.Fatalf("nil runner error=%v", err) t.Fatalf("nil runner error=%v", err)
} }
@@ -348,11 +324,6 @@ func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *test
); !errors.Is(err, ErrProfileLoad) { ); !errors.Is(err, ErrProfileLoad) {
t.Fatalf("missing profile error=%v", err) t.Fatalf("missing profile error=%v", err)
} }
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}},
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrProfileLoad) {
t.Fatalf("unresolved profile error=%v", err)
}
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements( if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
context.Background(), RequirementInspectionRequest{PromptID: "missing"}, context.Background(), RequirementInspectionRequest{PromptID: "missing"},
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) { ); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {

View File

@@ -58,6 +58,19 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
} }
} }
func TestResolvedTargetPreservesProviderMapSelection(t *testing.T) {
profile := &domain.ExecutionProfile{ResolvedTarget: &domain.ExecutionTarget{ExtraParams: map[string]any{"configured": true}}}
selected, _ := resolveExecutionTarget(profile, &domain.ExecutionTargetOverride{ExtraParams: map[string]any{}})
if selected.ExtraParams["configured"] != true {
t.Fatalf("empty override did not inherit provider settings: %#v", selected.ExtraParams)
}
profile.ResolvedTarget.ExtraParams = map[string]any{}
selected, _ = resolveExecutionTarget(profile, nil)
if selected.ExtraParams != nil {
t.Fatalf("empty provider settings changed projection: %#v", selected.ExtraParams)
}
}
func TestResolvedTargetCredentialAvailability(t *testing.T) { func TestResolvedTargetCredentialAvailability(t *testing.T) {
profile := &domain.ExecutionProfile{ID: "direct", ResolvedTarget: &domain.ExecutionTarget{ profile := &domain.ExecutionProfile{ID: "direct", ResolvedTarget: &domain.ExecutionTarget{
Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true, Endpoint: "https://provider.example/v1", Model: "model", APIKeyRequired: true,

View File

@@ -18,6 +18,7 @@ import (
"go.maximumdirect.net/promptkit/internal/capacity" "go.maximumdirect.net/promptkit/internal/capacity"
"go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/generation"
"go.maximumdirect.net/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
"go.maximumdirect.net/promptkit/internal/observation" "go.maximumdirect.net/promptkit/internal/observation"
@@ -42,17 +43,22 @@ var (
// Runner executes the Promptkit core use case. // Runner executes the Promptkit core use case.
type Runner struct { type Runner struct {
identity *runnerIdentity
promptDefs promptdef.Repository promptDefs promptdef.Repository
profiles profile.Repository profiles profile.Repository
artifacts artifact.Reader artifacts artifact.Reader
renderer prompt.Renderer renderer prompt.Renderer
llm llm.Client llm llm.Client
validator validate.Validator validator validate.ValidationPreparer
repairer OutputRepairer repairer OutputRepairer
admitter RunAdmitter admitter RunAdmitter
defaultTransientRetries uint64 defaultTransientRetries uint64
} }
// A non-zero-sized token gives each constructed runner a distinct identity
// without retaining its collaborators in prepared handles.
type runnerIdentity struct{ marker byte }
// RunAdmitter reserves capacity for one resolved backend run. // RunAdmitter reserves capacity for one resolved backend run.
type RunAdmitter interface { type RunAdmitter interface {
Admit(context.Context, string) (func(), error) Admit(context.Context, string) (func(), error)
@@ -81,7 +87,7 @@ func NewRunner(
artifacts artifact.Reader, artifacts artifact.Reader,
renderer prompt.Renderer, renderer prompt.Renderer,
llmClient llm.Client, llmClient llm.Client,
validator validate.Validator, validator validate.ValidationPreparer,
admitter RunAdmitter, admitter RunAdmitter,
) *Runner { ) *Runner {
return NewRunnerWithRepairer( return NewRunnerWithRepairer(
@@ -102,7 +108,7 @@ func NewRunnerWithRepairer(
artifacts artifact.Reader, artifacts artifact.Reader,
renderer prompt.Renderer, renderer prompt.Renderer,
llmClient llm.Client, llmClient llm.Client,
validator validate.Validator, validator validate.ValidationPreparer,
repairer OutputRepairer, repairer OutputRepairer,
admitter RunAdmitter, admitter RunAdmitter,
defaultTransientRetries ...uint64, defaultTransientRetries ...uint64,
@@ -112,6 +118,7 @@ func NewRunnerWithRepairer(
retries = defaultTransientRetries[0] retries = defaultTransientRetries[0]
} }
return &Runner{ return &Runner{
identity: &runnerIdentity{},
promptDefs: promptDefs, promptDefs: promptDefs,
profiles: profiles, profiles: profiles,
artifacts: artifacts, artifacts: artifacts,
@@ -204,7 +211,7 @@ func (r *Runner) executePreparedRun(
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format) outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
validationResult, err := validateArtifact(ctx, &outputArtifact, 0) validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err) return nil, &OperationError{Category: ErrValidation, Cause: err}
} }
if r.shouldAttemptRepair(prepared.OutputContract, validationResult) { if r.shouldAttemptRepair(prepared.OutputContract, validationResult) {
@@ -238,7 +245,7 @@ func (r *Runner) executePreparedRun(
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed) validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err) return nil, &OperationError{Category: ErrValidation, Cause: err}
} }
} }
} }
@@ -269,10 +276,10 @@ func (r *Runner) executePreparedRun(
} }
func wrapGenerationError(err error) error { func wrapGenerationError(err error) error {
if errors.Is(err, llm.ErrInvalidRequest) { if _, ok := err.(*generation.PreparationError); ok {
return fmt.Errorf("%w: %w", ErrInvalidRequest, err) return &OperationError{Category: ErrInvalidRequest, Cause: err}
} }
return fmt.Errorf("%w: %w", ErrLLMGenerate, err) return &OperationError{Category: ErrLLMGenerate, Cause: err}
} }
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage { func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
@@ -402,13 +409,9 @@ func (r *Runner) prepareValidation(
if r.validator == nil { if r.validator == nil {
return noOpPreparedValidation{contract: contract}, nil return noOpPreparedValidation{contract: contract}, nil
} }
preparer, ok := r.validator.(validate.ValidationPreparer) plan, err := r.validator.PrepareValidation(ctx, contract)
if !ok {
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
}
plan, err := preparer.PrepareValidation(ctx, contract)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err) return nil, &OperationError{Category: ErrValidation, Cause: err}
} }
if plan == nil { if plan == nil {
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation) return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
@@ -432,11 +435,7 @@ func (r *Runner) structuredOutputFromValidationPlan(
} }
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation) return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
} }
schemaDocument, err := jsonvalue.Copy(schemaDocument) return structuredOutputSpec(def, jsonvalue.CloneTrusted(schemaDocument)), nil
if err != nil {
return nil, fmt.Errorf("%w: invalid prepared json_schema schema document: %v", ErrValidation, err)
}
return structuredOutputSpec(def, schemaDocument), nil
} }
func (r *Runner) completePreparationWithStructuredOutput( func (r *Runner) completePreparationWithStructuredOutput(
@@ -450,7 +449,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
for name, ref := range req.Inputs { for name, ref := range req.Inputs {
art, readErr := r.artifacts.Read(ctx, ref) art, readErr := r.artifacts.Read(ctx, ref)
if readErr != nil { if readErr != nil {
return nil, fmt.Errorf("%w: input %q: %w", ErrArtifactLoad, name, readErr) return nil, &OperationError{Category: ErrArtifactLoad, Cause: fmt.Errorf("input %q: %w", name, readErr)}
} }
if art.Name == "" { if art.Name == "" {
art.Name = name art.Name = name
@@ -467,7 +466,7 @@ func (r *Runner) completePreparationWithStructuredOutput(
} }
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars) renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err) return nil, &OperationError{Category: ErrPromptRender, Cause: err}
} }
effectivePrompt := *renderedPrompt effectivePrompt := *renderedPrompt
if state.directSessionID != "" { if state.directSessionID != "" {
@@ -593,14 +592,16 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, presence domain.E
out.APIKeyEnv = override.APIKeyEnv out.APIKeyEnv = override.APIKeyEnv
} }
if len(override.ExtraParams) > 0 { if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams) out.ExtraParams = override.ExtraParams
} }
return out, presence return out, presence
} }
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) { func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
out := *profileValue.ResolvedTarget out := *profileValue.ResolvedTarget
out.ExtraParams = copyExtraParams(out.ExtraParams) if len(out.ExtraParams) == 0 {
out.ExtraParams = nil
}
presence := profileValue.TargetPresence presence := profileValue.TargetPresence
if override != nil { if override != nil {
return mergeExecutionTargetOverride(out, presence, *override) return mergeExecutionTargetOverride(out, presence, *override)
@@ -625,22 +626,8 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error
return nil return nil
} }
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil
}
cp := make(map[string]any, len(src))
for k, v := range src {
cp[k] = v
}
return cp
}
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) { func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) {
contract := def.Validation contract := def.Validation
if contract.Format == "" {
contract.Format = def.OutputFormat
}
if override != nil { if override != nil {
contract = *override contract = *override
if contract.Format == "" { if contract.Format == "" {
@@ -707,7 +694,3 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti
Hash: hex.EncodeToString(hash[:]), Hash: hex.EncodeToString(hash[:]),
} }
} }
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
return promptdef.HashDefinition(def)
}

View File

@@ -13,13 +13,17 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"text/template"
"time" "time"
"go.maximumdirect.net/promptkit/internal/capacity" "go.maximumdirect.net/promptkit/internal/capacity"
"go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/generation"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
"go.maximumdirect.net/promptkit/internal/prompt" "go.maximumdirect.net/promptkit/internal/prompt"
"go.maximumdirect.net/promptkit/internal/promptdef"
"go.maximumdirect.net/promptkit/internal/validate" "go.maximumdirect.net/promptkit/internal/validate"
) )
@@ -39,11 +43,61 @@ type staticPromptRepo struct {
def *domain.PromptDefinition def *domain.PromptDefinition
} }
func (r staticPromptRepo) GetPromptDefinition( type capturedTemplateFixture struct {
id string
templates prompt.CompiledTemplates
}
func (fixture capturedTemplateFixture) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
return fixture.templates, id == fixture.id
}
func testCompiledRenderer(t *testing.T, definition *domain.PromptDefinition) prompt.Renderer {
t.Helper()
var templates prompt.CompiledTemplates
if definition.SessionID != "" {
var err error
templates.Session, err = prompt.CompileTemplate("session", definition.SessionID)
if err != nil {
t.Fatal(err)
}
}
templates.Messages = make([]*template.Template, 0, len(definition.Templates))
for _, message := range definition.Templates {
compiled, err := prompt.CompileTemplate("message", message.Content)
if err != nil {
t.Fatal(err)
}
templates.Messages = append(templates.Messages, compiled)
}
return prompt.NewCompiledRenderer(capturedTemplateFixture{id: definition.ID, templates: templates})
}
func (r staticPromptRepo) GetPromptSnapshot(
context.Context, context.Context,
string, string,
) (*domain.PromptDefinition, error) { ) (*domain.PromptDefinition, string, error) {
return r.def, nil hash, err := promptdef.HashDefinition(r.def)
return r.def, hash, err
}
func (r staticPromptRepo) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
definition, hash, err := r.GetPromptSnapshot(ctx, id)
return fixturePromptInspection(definition, hash), err
}
func (staticPromptRepo) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
return prompt.RequirementSnapshot{}, false
}
func fixturePromptInspection(definition *domain.PromptDefinition, hash string) *domain.PromptInspection {
if definition == nil {
return nil
}
return &domain.PromptInspection{
PromptID: definition.ID, PromptHash: hash, DefaultProfileID: definition.DefaultProfile,
Inputs: append([]domain.PromptInput(nil), definition.Inputs...), OutputContract: definition.Validation,
}
} }
type staticExecutionProfileRepo struct { type staticExecutionProfileRepo struct {
@@ -58,6 +112,10 @@ func (r staticExecutionProfileRepo) GetProfile(
return &value, nil return &value, nil
} }
func (staticExecutionProfileRepo) GetProfileIdentity(_ context.Context, id string) (string, error) {
return id, nil
}
type staticArtifactReader struct{} type staticArtifactReader struct{}
func (staticArtifactReader) Read( func (staticArtifactReader) Read(
@@ -87,17 +145,40 @@ func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*
} }
if p, ok := f.profiles[id]; ok { if p, ok := f.profiles[id]; ok {
cp := *p cp := *p
if cp.ResolvedTarget != nil {
target := *cp.ResolvedTarget
target.ExtraParams = jsonvalue.CloneTrusted(target.ExtraParams).(map[string]any)
cp.ResolvedTarget = &target
}
return &cp, nil return &cp, nil
} }
return nil, errors.New("profile not found") return nil, errors.New("profile not found")
} }
func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string) (*domain.PromptDefinition, error) { func (f *fakeExecutionProfileRepo) GetProfileIdentity(ctx context.Context, id string) (string, error) {
_, err := f.GetProfile(ctx, id)
return id, err
}
func (f *fakePromptRepo) GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error) {
f.lastID = id f.lastID = id
if f.err != nil { if f.err != nil {
return nil, f.err return nil, "", f.err
} }
return f.def, nil if f.def == nil {
return nil, "", nil
}
hash, err := promptdef.HashDefinition(f.def)
return f.def, hash, err
}
func (f *fakePromptRepo) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
definition, hash, err := f.GetPromptSnapshot(ctx, id)
return fixturePromptInspection(definition, hash), err
}
func (*fakePromptRepo) GetRequirementSnapshot(string) (prompt.RequirementSnapshot, bool) {
return prompt.RequirementSnapshot{}, false
} }
type fakeArtifactReader struct { type fakeArtifactReader struct {
@@ -165,14 +246,6 @@ type fakeValidator struct {
validateCalls int validateCalls int
} }
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
f.validateCalls++
if f.err != nil {
return domain.ValidationResult{}, f.err
}
return f.result, nil
}
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) { func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
var schemaDocument any var schemaDocument any
if contract.ValidationMode == domain.ValidationJSONSchema { if contract.ValidationMode == domain.ValidationJSONSchema {
@@ -186,17 +259,20 @@ func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.Out
schemaDocument = map[string]any{"type": "object"} schemaDocument = map[string]any{"type": "object"}
} }
} }
return &fakePreparedValidator{validator: f, contract: contract, schemaDocument: schemaDocument}, nil return &fakePreparedValidator{validator: f, schemaDocument: schemaDocument}, nil
} }
type fakePreparedValidator struct { type fakePreparedValidator struct {
validator *fakeValidator validator *fakeValidator
contract domain.OutputContract
schemaDocument any schemaDocument any
} }
func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) { func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
return p.validator.Validate(ctx, artifact, p.contract) p.validator.validateCalls++
if p.validator.err != nil {
return domain.ValidationResult{}, p.validator.err
}
return p.validator.result, nil
} }
func (p *fakePreparedValidator) SchemaDocument() any { func (p *fakePreparedValidator) SchemaDocument() any {
@@ -405,7 +481,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(), defaultArtifactReader(),
prompt.NewGoRenderer(), testCompiledRenderer(t, def),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil) nil, nil)
@@ -452,7 +528,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(), defaultArtifactReader(),
prompt.NewGoRenderer(), testCompiledRenderer(t, def),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil) nil, nil)
@@ -482,7 +558,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(), defaultArtifactReader(),
prompt.NewGoRenderer(), testCompiledRenderer(t, def),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil) nil, nil)
@@ -842,7 +918,7 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(), defaultArtifactReader(),
prompt.NewGoRenderer(), testCompiledRenderer(t, def),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil) nil, nil)
@@ -869,7 +945,7 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(), defaultArtifactReader(),
prompt.NewGoRenderer(), testCompiledRenderer(t, def),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil) nil, nil)
@@ -2074,7 +2150,7 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
} }
} }
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) { func TestRunnerRunInjectedInvalidRequestRemainsGenerationFailure(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
@@ -2089,11 +2165,8 @@ func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
ProfileID: "exec", ProfileID: "exec",
Inputs: singleInputRef(), Inputs: singleInputRef(),
}) })
if !errors.Is(err, ErrInvalidRequest) { if !errors.Is(err, ErrLLMGenerate) || !errors.Is(err, llm.ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err) t.Fatalf("expected generation error retaining cause, got %v", err)
}
if errors.Is(err, ErrLLMGenerate) {
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
} }
} }
@@ -2401,7 +2474,7 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
baseClient := &controlledRepairLLM{ baseClient := &controlledRepairLLM{
events: make(chan controlledGeneration, 4), events: make(chan controlledGeneration, 4),
} }
scheduledClient := capacity.NewClient(manager, baseClient) scheduledClient := generation.NewInjected(manager, baseClient)
admitter := &recordingRunAdmitter{next: manager} admitter := &recordingRunAdmitter{next: manager}
runner := NewRunnerWithRepairer( runner := NewRunnerWithRepairer(
staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
@@ -2507,7 +2580,6 @@ func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts
DefaultProfile: "exec", DefaultProfile: "exec",
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}},
OutputFormat: format,
Validation: domain.OutputContract{ Validation: domain.OutputContract{
ValidationMode: mode, ValidationMode: mode,
RepairAttempts: attempts, RepairAttempts: attempts,
@@ -2589,10 +2661,6 @@ func float64Ptr(v float64) *float64 {
return &v return &v
} }
func stringPtr(v string) *string {
return &v
}
func intPtr(v int) *int { func intPtr(v int) *int {
return &v return &v
} }

View File

@@ -1,11 +1,8 @@
package usecase package usecase
import ( import (
"bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io"
"net/http" "net/http"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
@@ -13,10 +10,12 @@ import (
"go.maximumdirect.net/promptkit/internal/llm" "go.maximumdirect.net/promptkit/internal/llm"
) )
type authenticationProbe struct { type smokeAuthenticationClient interface {
client smokeHTTPGetter ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
} }
type authenticationProbe struct{ client smokeAuthenticationClient }
type authenticationProbeOutcome struct { type authenticationProbeOutcome struct {
supported bool supported bool
applicable *bool applicable *bool
@@ -26,16 +25,33 @@ type authenticationProbeOutcome struct {
failure domain.ExecutionFailureKind failure domain.ExecutionFailureKind
} }
func (p authenticationProbe) run( func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, target domain.SmokeTestTarget) (authenticationProbeOutcome, error) {
ctx context.Context,
backend domain.Backend,
target domain.SmokeTestTarget,
) (authenticationProbeOutcome, error) {
outcome, err := p.support(backend) outcome, err := p.support(backend)
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) { if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
return outcome, err return outcome, err
} }
return p.runOpenRouterKey(ctx, target) result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
outcome.evidence = result.HTTP
outcome.credentialsAccepted = result.Accepted
if result.Overflow {
outcome.reason = domain.SmokeTestReasonAuthenticationResponseLimitExceeded
outcome.failure = domain.ExecutionFailureOther
} else if result.Unrecognized {
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
outcome.failure = domain.ExecutionFailureMalformedResponse
} else if requestErr != nil {
if !result.HTTP.DispatchInvoked {
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
}
outcome.failure = httpProbeFailure(ctx, requestErr)
} else if result.HTTP.ResponseReceived {
if result.HTTP.HTTPStatus != nil && *result.HTTP.HTTPStatus != http.StatusOK && *result.HTTP.HTTPStatus != http.StatusUnauthorized {
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
}
} else {
outcome.failure = httpProbeFailure(ctx, nil)
}
return outcome, nil
} }
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) { func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
@@ -48,11 +64,7 @@ func (p authenticationProbe) support(backend domain.Backend) (authenticationProb
} }
switch *authentication.Kind { switch *authentication.Kind {
case "none": case "none":
return authenticationProbeOutcome{ return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil
supported: true,
applicable: boolPointer(false),
reason: domain.SmokeTestReasonAuthenticationNotApplicable,
}, nil
case "openrouter_key": case "openrouter_key":
if p.client == nil { if p.client == nil {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
@@ -62,129 +74,3 @@ func (p authenticationProbe) support(backend domain.Backend) (authenticationProb
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind) return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
} }
} }
func (p authenticationProbe) runOpenRouterKey(
ctx context.Context,
target domain.SmokeTestTarget,
) (authenticationProbeOutcome, error) {
endpoint, err := endpointWithoutUserInfo(target.Endpoint)
if err != nil {
return authenticationProbeOutcome{}, fmt.Errorf("prepare authentication endpoint: %w", err)
}
response, evidence, requestErr := p.client.GetWithAuthenticationEvidence(
ctx,
endpoint,
target.EffectiveSettings.TimeoutSeconds,
target.EffectiveSettings,
)
if response != nil && response.Body != nil {
defer response.Body.Close()
}
outcome := authenticationProbeOutcome{
supported: true,
applicable: boolPointer(true),
evidence: evidence,
}
if evidence.ResponseReceived || response != nil {
status := evidence.HTTPStatus
if status == nil && response != nil {
status = &response.StatusCode
}
if status == nil {
outcome.failure = domain.ExecutionFailureMalformedResponse
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
return outcome, nil
}
switch *status {
case http.StatusUnauthorized:
outcome.credentialsAccepted = boolPointer(false)
case http.StatusOK:
if response == nil || response.Body == nil {
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
outcome.failure = domain.ExecutionFailureMalformedResponse
return outcome, nil
}
recognized, bodyErr := recognizedOpenRouterKeyResponse(response.Body)
if bodyErr != nil {
outcome.failure = httpProbeFailure(ctx, bodyErr)
return outcome, nil
}
if recognized {
outcome.credentialsAccepted = boolPointer(true)
} else {
outcome.reason = domain.SmokeTestReasonAuthenticationResponseUnrecognized
outcome.failure = domain.ExecutionFailureMalformedResponse
}
default:
outcome.failure = generation.ClassifyFailure(llm.ErrUnexpectedStatus)
}
return outcome, nil
}
if evidence.DispatchInvoked {
if requestErr != nil {
outcome.failure = httpProbeFailure(ctx, requestErr)
} else {
outcome.failure = httpProbeFailure(ctx, nil)
}
return outcome, nil
}
if requestErr != nil {
return outcome, fmt.Errorf("execute authentication request: %w", requestErr)
}
outcome.failure = generation.ClassifyFailure(llm.ErrTransport)
return outcome, nil
}
func recognizedOpenRouterKeyResponse(body io.Reader) (bool, error) {
contents, err := io.ReadAll(body)
if err != nil {
return false, err
}
var envelope map[string]json.RawMessage
if err := json.Unmarshal(contents, &envelope); err != nil || envelope == nil {
return false, nil
}
dataJSON, ok := envelope["data"]
if !ok || !isJSONObject(dataJSON) {
return false, nil
}
var data map[string]json.RawMessage
if err := json.Unmarshal(dataJSON, &data); err != nil || data == nil {
return false, nil
}
label, labelOK := data["label"]
freeTier, freeTierOK := data["is_free_tier"]
usage, usageOK := data["usage"]
if !labelOK || !isJSONString(label) || !freeTierOK || !isJSONBoolean(freeTier) || !usageOK {
return false, nil
}
var number any
usageDecoder := json.NewDecoder(bytes.NewReader(usage))
usageDecoder.UseNumber()
if usageDecoder.Decode(&number) != nil {
return false, nil
}
_, ok = number.(json.Number)
return ok, nil
}
func isJSONObject(value json.RawMessage) bool {
value = bytes.TrimSpace(value)
return len(value) > 0 && value[0] == '{'
}
func isJSONString(value json.RawMessage) bool {
value = bytes.TrimSpace(value)
if len(value) == 0 || value[0] != '"' {
return false
}
var parsed string
return json.Unmarshal(value, &parsed) == nil
}
func isJSONBoolean(value json.RawMessage) bool {
value = bytes.TrimSpace(value)
return bytes.Equal(value, []byte("true")) || bytes.Equal(value, []byte("false"))
}

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"net/http" "net/http"
"strings" "strings"
"testing" "testing"
@@ -11,6 +12,58 @@ import (
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
) )
type measuredAuthenticationBody struct {
reader io.Reader
read int
closed bool
}
func (b *measuredAuthenticationBody) Read(p []byte) (int, error) {
n, err := b.reader.Read(p)
b.read += n
return n, err
}
func (b *measuredAuthenticationBody) Close() error { b.closed = true; return nil }
func TestAuthenticationResponseRecognitionBound(t *testing.T) {
const maxAuthenticationResponseBytes = 1 << 20
valid := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
for _, test := range []struct {
name string
size int
declared int64
status int
overflow bool
}{
{"exact limit", maxAuthenticationResponseBytes, -1, 200, false},
{"overflow unknown length", maxAuthenticationResponseBytes + 10, -1, 200, true},
{"overflow understated length", maxAuthenticationResponseBytes + 10, 1, 200, true},
{"unauthorized does not read", maxAuthenticationResponseBytes + 10, -1, 401, false},
} {
t.Run(test.name, func(t *testing.T) {
body := &measuredAuthenticationBody{reader: strings.NewReader(valid + strings.Repeat(" ", test.size-len(valid)))}
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: test.status, Header: make(http.Header), Body: body, ContentLength: test.declared}, nil
})
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"})
if err != nil || !body.closed {
t.Fatalf("outcome = %+v, err = %v, closed = %v", outcome, err, body.closed)
}
if test.overflow {
if outcome.reason != domain.SmokeTestReasonAuthenticationResponseLimitExceeded || outcome.failure != domain.ExecutionFailureOther || outcome.credentialsAccepted != nil || body.read != maxAuthenticationResponseBytes+1 {
t.Fatalf("overflow = %+v, read = %d", outcome, body.read)
}
} else if test.status == 401 {
if body.read != 0 || outcome.credentialsAccepted == nil || *outcome.credentialsAccepted {
t.Fatalf("unauthorized = %+v, read = %d", outcome, body.read)
}
} else if outcome.credentialsAccepted == nil || !*outcome.credentialsAccepted {
t.Fatalf("exact limit = %+v", outcome)
}
})
}
}
func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) { func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
client := reachabilityClient(t, func(*http.Request) (*http.Response, error) { client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
t.Fatal("authentication applicability unexpectedly dispatched") t.Fatal("authentication applicability unexpectedly dispatched")
@@ -159,21 +212,26 @@ func TestAuthenticationProbeInterpretsOpenRouterResponses(t *testing.T) {
} }
func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) { func TestAuthenticationProbeRecognizesOnlyOneCompleteJSONObject(t *testing.T) {
for _, body := range []string{ for _, tc := range []struct {
`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`, body string
`{"data":[]}`, accepted bool
`{"data":{"label":"member","is_free_tier":false,"usage":null}}`, }{
`{"data":{"label":"member","is_free_tier":0,"usage":1}}`, {`{"data":{"label":"member","is_free_tier":false,"usage":1}} {}`, false},
`{"data":{"is_free_tier":false,"usage":1}}`, {`{"data":[]}`, false},
`{"data":{"label":null,"is_free_tier":false,"usage":1}}`, {`{"data":{"label":"member","is_free_tier":false,"usage":null}}`, false},
{`{"data":{"label":"member","is_free_tier":0,"usage":1}}`, false},
{`{"data":{"is_free_tier":false,"usage":1}}`, false},
{`{"data":{"label":null,"is_free_tier":false,"usage":1}}`, false},
{`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`, true},
} { } {
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(body)); err != nil || recognized { client := reachabilityClient(t, func(*http.Request) (*http.Response, error) {
t.Errorf("recognized invalid account response %s", body) return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: ioNopCloser(tc.body)}, nil
})
outcome, err := (authenticationProbe{client: client}).run(context.Background(), backendWithAuthenticationKind("openrouter_key"), domain.SmokeTestTarget{Endpoint: "https://example.invalid/key"})
if err != nil || (outcome.credentialsAccepted != nil && *outcome.credentialsAccepted) != tc.accepted {
t.Errorf("account response %s: outcome = %+v, error = %v", tc.body, outcome, err)
} }
} }
if recognized, err := recognizedOpenRouterKeyResponse(strings.NewReader(`{"data":{"label":"","is_free_tier":false,"usage":1e2,"ignored":true},"ignored":true}`)); err != nil || !recognized {
t.Fatal("valid minimal identifying response was rejected")
}
} }
func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) { func TestAuthenticationProbeTreatsInterruptedResponseBodyAsTransportFailure(t *testing.T) {

View File

@@ -25,7 +25,7 @@ type smokeInferenceCase struct {
kind domain.SmokeTestKind kind domain.SmokeTestKind
request domain.GenerateRequest request domain.GenerateRequest
expectedContent string expectedContent string
jsonValidator validate.Validator jsonValidation validate.PreparedValidation
schemaValidation validate.PreparedValidation schemaValidation validate.PreparedValidation
} }
@@ -52,11 +52,13 @@ func prepareSmokeInferenceCase(
}, nil }, nil
case domain.SmokeTestStructuredOutput: case domain.SmokeTestStructuredOutput:
validator := validate.NewFSValidator(smokeSchemaFS, ".") validator := validate.NewFSValidator(smokeSchemaFS, ".")
preparer, ok := validator.(validate.ValidationPreparer) jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
if !ok { Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
return nil, fmt.Errorf("%w: private validator does not support prepared validation", ErrValidation) })
if err != nil {
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
} }
plan, err := preparer.PrepareValidation(ctx, domain.OutputContract{ plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
Format: domain.FormatJSON, Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSONSchema, ValidationMode: domain.ValidationJSONSchema,
SchemaPath: smokeStructuredSchemaPath, SchemaPath: smokeStructuredSchemaPath,
@@ -68,7 +70,7 @@ func prepareSmokeInferenceCase(
return &smokeInferenceCase{ return &smokeInferenceCase{
kind: kind, kind: kind,
expectedContent: smokeStructuredExpectedContent, expectedContent: smokeStructuredExpectedContent,
jsonValidator: validate.NewStandardValidator(""), jsonValidation: jsonPlan,
schemaValidation: plan, schemaValidation: plan,
request: newGenerationRequest( request: newGenerationRequest(
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}}, domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
@@ -101,10 +103,7 @@ func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.Gene
return facts, nil return facts, nil
case domain.SmokeTestStructuredOutput: case domain.SmokeTestStructuredOutput:
artifact := domain.Artifact{Body: []byte(response.Content)} artifact := domain.Artifact{Body: []byte(response.Content)}
jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{ jsonResult, err := c.jsonValidation.Validate(ctx, &artifact)
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSON,
})
if err != nil { if err != nil {
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err) return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
} }

View File

@@ -2,6 +2,7 @@ package usecase
import ( import (
"context" "context"
"errors"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
@@ -102,6 +103,7 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
}{ }{
{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)}, {name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, {name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
{name: "scalar JSON misses schema", content: `42`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)}, {name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil}, {name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
{name: "empty content", content: "", wantSchemaMatched: nil}, {name: "empty content", content: "", wantSchemaMatched: nil},
@@ -125,3 +127,19 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
t.Fatalf("facts were populated without generated content: %+v", noResponseFacts) t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
} }
} }
func TestSmokeStructuredInferenceCaseHonorsCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := prepareSmokeInferenceCase(ctx, domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0); !errors.Is(err, context.Canceled) {
t.Fatalf("prepare error = %v, want cancellation", err)
}
caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0)
if err != nil {
t.Fatal(err)
}
facts, err := caseValue.evaluate(ctx, &domain.GenerateResponse{Content: `{"status":"ok"}`})
if !errors.Is(err, context.Canceled) || facts.ValidJSON != nil || facts.SchemaMatched != nil {
t.Fatalf("evaluation facts = %+v, error = %v", facts, err)
}
}

View File

@@ -14,7 +14,6 @@ import (
type smokeHTTPGetter interface { type smokeHTTPGetter interface {
GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error) GetWithEvidence(context.Context, string, int) (*http.Response, llm.HTTPExecutionEvidence, error)
GetWithAuthenticationEvidence(context.Context, string, int, domain.ExecutionTarget) (*http.Response, llm.HTTPExecutionEvidence, error)
} }
type reachabilityProbe struct { type reachabilityProbe struct {
@@ -22,12 +21,11 @@ type reachabilityProbe struct {
} }
type reachabilityProbeOutcome struct { type reachabilityProbeOutcome struct {
supported bool supported bool
reason domain.SmokeTestReason reason domain.SmokeTestReason
initialEndpoint string reachable *bool
reachable *bool evidence llm.HTTPExecutionEvidence
evidence llm.HTTPExecutionEvidence failure domain.ExecutionFailureKind
failure domain.ExecutionFailureKind
} }
func (p reachabilityProbe) run( func (p reachabilityProbe) run(
@@ -45,7 +43,7 @@ func (p reachabilityProbe) run(
if err != nil { if err != nil {
return reachabilityProbeOutcome{}, fmt.Errorf("prepare reachability endpoint: %w", err) return reachabilityProbeOutcome{}, fmt.Errorf("prepare reachability endpoint: %w", err)
} }
result := reachabilityProbeOutcome{supported: true, initialEndpoint: endpoint} result := reachabilityProbeOutcome{supported: true}
response, evidence, requestErr := p.client.GetWithEvidence(ctx, endpoint, timeoutSeconds) response, evidence, requestErr := p.client.GetWithEvidence(ctx, endpoint, timeoutSeconds)
if response != nil && response.Body != nil { if response != nil && response.Body != nil {
_ = response.Body.Close() _ = response.Body.Close()

View File

@@ -62,7 +62,7 @@ func TestReachabilityProbeRemovesURLCredentials(t *testing.T) {
if observed == nil || observed.URL.User != nil || observed.Header.Get("Authorization") != "" { if observed == nil || observed.URL.User != nil || observed.Header.Get("Authorization") != "" {
t.Fatalf("probe request retained URL credentials: %+v", observed) t.Fatalf("probe request retained URL credentials: %+v", observed)
} }
if result.initialEndpoint != "https://example.invalid/health?check=1" || result.evidence.CredentialsSupplied { if result.evidence.Endpoint != "https://example.invalid/health?check=1" || result.evidence.CredentialsSupplied {
t.Fatalf("credential-free evidence = %+v", result) t.Fatalf("credential-free evidence = %+v", result)
} }
} }
@@ -111,12 +111,9 @@ func TestReachabilityProbeIsUnsupportedWithoutHTTPClient(t *testing.T) {
func reachabilityClient(t *testing.T, roundTrip func(*http.Request) (*http.Response, error)) *llm.OpenAICompatibleClient { func reachabilityClient(t *testing.T, roundTrip func(*http.Request) (*http.Response, error)) *llm.OpenAICompatibleClient {
t.Helper() t.Helper()
client, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ client := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
HTTPClient: &http.Client{Transport: reachabilityProbeTransport(roundTrip)}, HTTPClient: &http.Client{Transport: reachabilityProbeTransport(roundTrip)},
}) })
if err != nil {
t.Fatal(err)
}
return client return client
} }

View File

@@ -3,10 +3,10 @@ package usecase
import ( import (
"context" "context"
"fmt" "fmt"
"net/url"
"go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/llm"
) )
const ( const (
@@ -112,7 +112,9 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
settings.BackendID = selection.backend.ID settings.BackendID = selection.backend.ID
settings.Endpoint = selection.backend.Endpoint settings.Endpoint = selection.backend.Endpoint
settings.APIKeyEnv = selection.backend.APIKeyEnv settings.APIKeyEnv = selection.backend.APIKeyEnv
settings.ExtraParams = copyExtraParams(selection.backend.ExtraParams) if len(selection.backend.ExtraParams) > 0 {
settings.ExtraParams = selection.backend.ExtraParams
}
presence := domain.ExecutionTargetPresence{} presence := domain.ExecutionTargetPresence{}
if req.Execution != nil { if req.Execution != nil {
settings, presence = mergeExecutionTargetOverride(settings, presence, *req.Execution) settings, presence = mergeExecutionTargetOverride(settings, presence, *req.Execution)
@@ -144,7 +146,7 @@ func (r *Runner) resolveSmokeBackend(id string) (domain.Backend, error) {
} }
backend, err := lookup.GetBackend(id) backend, err := lookup.GetBackend(id)
if err != nil { if err != nil {
return domain.Backend{}, fmt.Errorf("%w: backend %q: %v", ErrProfileLoad, id, err) return domain.Backend{}, &OperationError{Category: ErrProfileLoad, Cause: fmt.Errorf("backend %q: %w", id, err)}
} }
return backend, nil return backend, nil
} }
@@ -176,7 +178,7 @@ func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, sm
return *authentication.Endpoint, nil return *authentication.Endpoint, nil
} }
if authentication.Kind != nil && *authentication.Kind == "openrouter_key" { if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
return url.JoinPath(baseEndpoint, "key") return llm.OpenRouterKeyEndpoint(baseEndpoint)
} }
} }
} }

View File

@@ -9,6 +9,7 @@ import (
"go.maximumdirect.net/promptkit/internal/configuration" "go.maximumdirect.net/promptkit/internal/configuration"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/profile" "go.maximumdirect.net/promptkit/internal/profile"
"go.maximumdirect.net/promptkit/internal/validate"
) )
type selectionTestCollaborators struct { type selectionTestCollaborators struct {
@@ -41,9 +42,9 @@ func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*do
type selectionTestValidator struct{ state *selectionTestCollaborators } type selectionTestValidator struct{ state *selectionTestCollaborators }
func (v selectionTestValidator) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) { func (v selectionTestValidator) PrepareValidation(context.Context, domain.OutputContract) (validate.PreparedValidation, error) {
v.state.validationCalls++ v.state.validationCalls++
return domain.ValidationResult{}, errors.New("validation must not occur during target selection") return nil, errors.New("validation must not occur during target selection")
} }
type selectionTestRepositories struct { type selectionTestRepositories struct {
@@ -57,6 +58,10 @@ func (r *selectionTestRepositories) GetProfile(ctx context.Context, id string) (
return r.profiles.GetProfile(ctx, id) return r.profiles.GetProfile(ctx, id)
} }
func (r *selectionTestRepositories) GetProfileIdentity(ctx context.Context, id string) (string, error) {
return r.profiles.GetProfileIdentity(ctx, id)
}
func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) { func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) {
return r.backends.GetBackend(id) return r.backends.GetBackend(id)
} }

View File

@@ -17,6 +17,7 @@ import (
type SmokeTester struct { type SmokeTester struct {
runner *Runner runner *Runner
httpClient smokeHTTPGetter httpClient smokeHTTPGetter
authClient smokeAuthenticationClient
now func() time.Time now func() time.Time
} }
@@ -24,18 +25,22 @@ type SmokeTester struct {
// leaves HTTP operations unsupported rather than falling back to another client. // leaves HTTP operations unsupported rather than falling back to another client.
func NewSmokeTester(runner *Runner, httpClient *llm.OpenAICompatibleClient) *SmokeTester { func NewSmokeTester(runner *Runner, httpClient *llm.OpenAICompatibleClient) *SmokeTester {
var getter smokeHTTPGetter var getter smokeHTTPGetter
var authentication smokeAuthenticationClient
if httpClient != nil { if httpClient != nil {
getter = httpClient getter = httpClient
authentication = httpClient
} }
return &SmokeTester{runner: runner, httpClient: getter, now: time.Now} return &SmokeTester{runner: runner, httpClient: getter, authClient: authentication, now: time.Now}
} }
func newSmokeTesterWithClock(runner *Runner, httpClient *llm.OpenAICompatibleClient, now func() time.Time) *SmokeTester { func newSmokeTesterWithClock(runner *Runner, httpClient *llm.OpenAICompatibleClient, now func() time.Time) *SmokeTester {
var getter smokeHTTPGetter var getter smokeHTTPGetter
var authentication smokeAuthenticationClient
if httpClient != nil { if httpClient != nil {
getter = httpClient getter = httpClient
authentication = httpClient
} }
return &SmokeTester{runner: runner, httpClient: getter, now: now} return &SmokeTester{runner: runner, httpClient: getter, authClient: authentication, now: now}
} }
// Run selects and executes one diagnostic, retaining only factual results. // Run selects and executes one diagnostic, retaining only factual results.
@@ -126,7 +131,7 @@ func (t *SmokeTester) Run(ctx context.Context, request domain.SmokeTestRequest)
return t.finishHTTP(ctx, report, finish) return t.finishHTTP(ctx, report, finish)
case domain.SmokeTestAuthentication: case domain.SmokeTestAuthentication:
requestStartedAt := now().UTC() requestStartedAt := now().UTC()
outcome, err := (authenticationProbe{client: t.httpClient}).run(ctx, selection.backend, selection.target) outcome, err := (authenticationProbe{client: t.authClient}).run(ctx, selection.backend, selection.target)
requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt)) requestDuration := nonnegativeDuration(now().UTC().Sub(requestStartedAt))
if err != nil { if err != nil {
if contextErr := ctx.Err(); contextErr != nil { if contextErr := ctx.Err(); contextErr != nil {
@@ -210,6 +215,9 @@ func (t *SmokeTester) runInference(
report.Failure = generation.ClassifyFailure(contextErr) report.Failure = generation.ClassifyFailure(contextErr)
return finish(contextErr) return finish(contextErr)
} }
if evidence.PreparationFailed && generationErr != nil {
return nil, &OperationError{Category: ErrInvalidRequest, Cause: generationErr}
}
if generationErr != nil { if generationErr != nil {
report.Failure = generation.ClassifyFailure(generationErr) report.Failure = generation.ClassifyFailure(generationErr)
report.Completed = true report.Completed = true
@@ -346,7 +354,7 @@ func (t *SmokeTester) setSupport(report *domain.SmokeTestReport, selection *smok
report.Applicable = boolPointer(true) report.Applicable = boolPointer(true)
return nil return nil
case domain.SmokeTestAuthentication: case domain.SmokeTestAuthentication:
outcome, err := (authenticationProbe{client: t.httpClient}).support(selection.backend) outcome, err := (authenticationProbe{client: t.authClient}).support(selection.backend)
if err != nil { if err != nil {
return err return err
} }
@@ -430,12 +438,8 @@ func httpExecutionFacts(evidence llm.HTTPExecutionEvidence) domain.SmokeTestFact
} }
func smokeReportTarget(target domain.SmokeTestTarget) (domain.SmokeTestTarget, error) { func smokeReportTarget(target domain.SmokeTestTarget) (domain.SmokeTestTarget, error) {
cloned, err := domain.CloneSmokeTestReport(domain.SmokeTestReport{Target: target})
if err != nil {
return domain.SmokeTestTarget{}, fmt.Errorf("clone smoke-test target settings: %w", err)
}
target = cloned.Target
target.EffectiveSettings.APIKey = "" target.EffectiveSettings.APIKey = ""
var err error
target.Endpoint, err = endpointWithoutUserInfo(target.Endpoint) target.Endpoint, err = endpointWithoutUserInfo(target.Endpoint)
if err != nil { if err != nil {
return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved diagnostic endpoint", ErrInvalidRequest) return domain.SmokeTestTarget{}, fmt.Errorf("%w: invalid resolved diagnostic endpoint", ErrInvalidRequest)

View File

@@ -393,3 +393,22 @@ func (b cancelOnCloseSmokeBody) Close() error {
b.cancel() b.cancel()
return b.ReadCloser.Close() return b.ReadCloser.Close()
} }
func TestSmokeReportTargetRedactsCredentialsAndSanitizesEndpoints(t *testing.T) {
target := domain.SmokeTestTarget{
Endpoint: "https://user:password@example.invalid/key",
EffectiveSettings: domain.ExecutionTarget{
Endpoint: "https://user:password@example.invalid/v1",
APIKey: "secret",
ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
},
}
got, err := smokeReportTarget(target)
if err != nil {
t.Fatal(err)
}
if got.EffectiveSettings.ExtraParams["nested"].(map[string]any)["value"] != "original" ||
got.EffectiveSettings.APIKey != "" || got.Endpoint != "https://example.invalid/key" || got.EffectiveSettings.Endpoint != "https://example.invalid/v1" {
t.Fatalf("report target not redacted: %+v", got)
}
}

View File

@@ -19,7 +19,7 @@ func TestValidationCancellationBeforeWorkDoesNotOpenSchemaSource(t *testing.T) {
source := &countingSchemaFS{FS: fstest.MapFS{ source := &countingSchemaFS{FS: fstest.MapFS{
"schema.json": {Data: []byte(`{"type":"object"}`)}, "schema.json": {Data: []byte(`{"type":"object"}`)},
}} }}
validator := NewFSValidator(source, ".").(ValidationPreparer) validator := NewFSValidator(source, ".")
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
@@ -35,17 +35,16 @@ func TestValidationCancellationBeforeWorkDoesNotOpenSchemaSource(t *testing.T) {
} }
} }
func TestValidationCancellationBetweenReferencedSchemaReadChunks(t *testing.T) { func TestValidationCancellationBetweenRootSchemaReadChunks(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
source := &controlledSchemaFS{ source := &controlledSchemaFS{
FS: fstest.MapFS{ FS: fstest.MapFS{
"root.json": {Data: []byte(`{"$ref":"child.json"}`)}, "root.json": {Data: []byte(`{"type":"string"}` + strings.Repeat(" ", schemaReadChunkSize*2))},
"child.json": {Data: []byte(`{"type":"string"}` + strings.Repeat(" ", schemaReadChunkSize*2))},
}, },
target: "child.json", target: "root.json",
cancel: cancel, cancel: cancel,
} }
validator := NewFSValidator(source, ".").(ValidationPreparer) validator := NewFSValidator(source, ".")
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{ plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema, ValidationMode: domain.ValidationJSONSchema,
@@ -57,8 +56,8 @@ func TestValidationCancellationBetweenReferencedSchemaReadChunks(t *testing.T) {
if reads := source.reads.Load(); reads != 1 { if reads := source.reads.Load(); reads != 1 {
t.Fatalf("controlled child reads = %d, want 1", reads) t.Fatalf("controlled child reads = %d, want 1", reads)
} }
if closes := source.closes.Load(); closes != 1 { if closes := source.closes.Load(); closes != 2 {
t.Fatalf("controlled child closes = %d, want 1", closes) t.Fatalf("controlled root closes = %d, want stat and read handles closed", closes)
} }
} }
@@ -113,7 +112,7 @@ func TestCancellationDoesNotDetachBlockedSchemaRead(t *testing.T) {
started: started, started: started,
release: release, release: release,
} }
validator := NewFSValidator(source, ".").(ValidationPreparer) validator := NewFSValidator(source, ".")
type outcome struct { type outcome struct {
plan PreparedValidation plan PreparedValidation
err error err error

View File

@@ -0,0 +1,91 @@
package validate
import (
"context"
"fmt"
"net/url"
"strconv"
"strings"
"github.com/santhosh-tekuri/jsonschema/v6"
)
// These schema-bearing positions mirror the inherited subschema tables in
// jsonschema/v6 v6.0.2 draft.go. Annotation objects are deliberately excluded.
func checkSchemaLocations(ctx context.Context, compiler *jsonschema.Compiler, resourceURL string, document any, pointer string, compiled *jsonschema.Schema) error {
if err := ctx.Err(); err != nil {
return err
}
object, ok := document.(map[string]any)
if !ok {
return nil
}
version := compiled.DraftVersion
mapKeys := []string{"definitions", "properties", "patternProperties", "dependencies"}
singleKeys := []string{"not", "additionalProperties", "items", "additionalItems"}
arrayKeys := []string{"allOf", "anyOf", "oneOf", "items"}
if version >= 6 {
singleKeys = append(singleKeys, "propertyNames", "contains")
}
if version >= 7 {
singleKeys = append(singleKeys, "if", "then", "else")
}
if version >= 2019 {
mapKeys = append(mapKeys, "$defs", "dependentSchemas")
singleKeys = append(singleKeys, "unevaluatedProperties", "unevaluatedItems", "contentSchema")
}
if version >= 2020 {
arrayKeys = append(arrayKeys, "prefixItems")
}
visit := func(value any, segments ...string) error {
switch value.(type) {
case map[string]any, bool:
default:
return nil
}
childPointer := pointer
for _, segment := range segments {
childPointer += "/" + strings.ReplaceAll(strings.ReplaceAll(segment, "~", "~0"), "/", "~1")
}
parsed, err := url.Parse(resourceURL)
if err != nil {
return err
}
parsed.Fragment = childPointer
child, err := compileJSONSchema(ctx, parsed.String(), compiler.Compile)
if err != nil {
return fmt.Errorf("compile declared schema %q: %w", childPointer, err)
}
return checkSchemaLocations(ctx, compiler, resourceURL, value, childPointer, child)
}
for _, key := range mapKeys {
entries, ok := object[key].(map[string]any)
if !ok {
continue
}
for name, value := range entries {
if err := visit(value, key, name); err != nil {
return err
}
}
}
for _, key := range singleKeys {
if value, ok := object[key]; ok {
if err := visit(value, key); err != nil {
return err
}
}
}
for _, key := range arrayKeys {
entries, ok := object[key].([]any)
if !ok {
continue
}
for i, value := range entries {
if err := visit(value, key, strconv.Itoa(i)); err != nil {
return err
}
}
}
return ctx.Err()
}

View File

@@ -0,0 +1,60 @@
package validate
import (
"context"
"strings"
"testing"
)
func TestSchemaLocationsCheckUnusedDeclarations(t *testing.T) {
for _, test := range []struct {
name, schema string
wantError bool
}{
{"unused external", `{"$defs":{"unused":{"$ref":"other.json"}},"type":"string"}`, true},
{"annotation is data", `{"examples":[{"$ref":"other.json"}],"default":{"$ref":"other.json"},"type":"string"}`, false},
{"internal escaped key", `{"$defs":{"a/b~c":{"type":"string"}},"$ref":"#/$defs/a~1b~0c"}`, false},
{"URI escaped key", `{"$defs":{"a # %/雪":{"type":"string"}}}`, false},
{"internal anchor", `{"$defs":{"value":{"$anchor":"item","type":"string"}},"$ref":"#item"}`, false},
{"dynamic reference", `{"$dynamicAnchor":"node","type":"object","properties":{"child":{"$dynamicRef":"#node"}}}`, false},
{"recursive", `{"$id":"https://example.invalid/root","$defs":{"node":{"$anchor":"node","anyOf":[{"type":"null"},{"$ref":"#node"}]}},"$ref":"#/$defs/node"}`, false},
{"embedded old dialect annotation", `{"$defs":{"old":{"$id":"old","$schema":"http://json-schema.org/draft-07/schema#","$defs":{"unused":{"$ref":"other.json"}},"type":"string"}}}`, false},
{"embedded old dialect definition", `{"$defs":{"old":{"$id":"old","$schema":"http://json-schema.org/draft-07/schema#","definitions":{"unused":{"$ref":"other.json"}}}}}`, true},
} {
t.Run(test.name, func(t *testing.T) {
doc, err := decodeJSONValue(context.Background(), []byte(test.schema))
if err != nil {
t.Fatal(err)
}
_, err = compileSchemaDocument(context.Background(), "promptkit-schema:///root.json", doc)
if test.wantError && (err == nil || !strings.Contains(err.Error(), "external schema resource")) {
t.Fatalf("expected external resource error, got %v", err)
}
if !test.wantError && err != nil {
t.Fatal(err)
}
})
}
}
func TestSchemaLocationsPreserveRootValidation(t *testing.T) {
doc, err := decodeJSONValue(context.Background(), []byte(`{"$defs":{"number":{"type":"integer","minimum":2}},"$ref":"#/$defs/number"}`))
if err != nil {
t.Fatal(err)
}
schema, err := compileSchemaDocument(context.Background(), "promptkit-schema:///root.json", doc)
if err != nil {
t.Fatal(err)
}
if err := schema.Validate(3); err != nil {
t.Fatalf("valid instance: %v", err)
}
if err := schema.Validate("not a number"); err == nil {
t.Fatal("invalid instance was accepted")
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := compileSchemaDocument(ctx, "promptkit-schema:///root.json", doc); err != context.Canceled {
t.Fatalf("canceled compilation = %v", err)
}
}

View File

@@ -0,0 +1,46 @@
package validate
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"testing/fstest"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestSchemaPreparationDoesNotDependOnSourceName(t *testing.T) {
for _, test := range []struct {
name, schema string
rejected bool
}{
{"source name", `{"$defs":{"v":{"type":"integer"}},"$ref":"schema.json#/$defs/v"}`, true},
{"unused source name", `{"$defs":{"v":{"$ref":"schema.json"}}}`, true},
{"fragment", `{"$defs":{"v":{"type":"integer"}},"$ref":"#/$defs/v"}`, false},
{"explicit ID", `{"$id":"https://example.invalid/schema.json","$defs":{"v":{"type":"integer"}},"$ref":"schema.json#/$defs/v"}`, false},
{"embedded ID", `{"$defs":{"v":{"$id":"child","type":"integer"}},"$ref":"child"}`, false},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "schema.json"), []byte(test.schema), 0600); err != nil {
t.Fatal(err)
}
fsys := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(test.schema)}}
for i, validator := range []ValidationPreparer{NewStandardValidator(dir), NewFSValidator(fsys, ".")} {
t.Run(fmt.Sprint(i), func(t *testing.T) {
_, err := validator.PrepareValidation(context.Background(), domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json"})
if test.rejected {
if err == nil || !strings.Contains(err.Error(), "source location") {
t.Fatalf("expected source-independent rejection, got %v", err)
}
} else if err != nil {
t.Fatal(err)
}
})
}
})
}
}

Some files were not shown because too many files have changed in this diff Show More