Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 207677e994 | |||
| 20c5ea47af | |||
| d4509b163e | |||
| ba4de9d11e | |||
| d77e3e2f83 | |||
| 81a2a7591d | |||
| cd0e1f5d39 | |||
| 6ed0e037c4 | |||
| 3d82cad2c9 | |||
| db1cad6d2d | |||
| c3257b63f0 | |||
| ca8cf1b827 | |||
| adc700f96e | |||
| 6056f93e55 | |||
| f2b0ed8d6c | |||
| ea7cc170ad | |||
| f762c0358e | |||
| a47df4ec20 | |||
| fed9ad082d | |||
| 4ee64b6eda | |||
| 440d9a2c88 | |||
| 23ad9e2a26 | |||
| 80649c579d | |||
| 0254eb1b5b | |||
| a10d1a2245 | |||
| 0564c8086c | |||
| 03209947f6 | |||
| 1070be8d89 |
@@ -45,6 +45,12 @@ boundary and constraints that framework work must preserve.
|
||||
|
||||
## Release Guidance
|
||||
|
||||
The [v0.25.0 release notes](docs/releases/v0.25.0.md) cover invocation-scoped
|
||||
observation, provider conventions, reasoning and cache usage, and optional USD
|
||||
cost reporting. Upgrading consumers should review provider selection,
|
||||
authentication diagnostic defaults, and the corrected OpenRouter cache-write
|
||||
mapping.
|
||||
|
||||
The [v0.24.0 release notes](docs/releases/v0.24.0.md) cover application-owned
|
||||
configuration loading, prompt-owned schemas, default output repair, and the
|
||||
prepared message-content hash. Upgrading consumers should review the breaking
|
||||
|
||||
10
backends.go
10
backends.go
@@ -26,6 +26,10 @@ type BackendConfig struct {
|
||||
ID string
|
||||
// Extends names a sibling backend to inherit through ConfigKit.
|
||||
Extends *string
|
||||
// Provider selects the backend's supported API conventions.
|
||||
Provider *string
|
||||
// ZeroCost enables a zero-cost fallback when no usable charge is reported.
|
||||
ZeroCost *bool
|
||||
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
|
||||
Endpoint *string
|
||||
// APIKeyEnv optionally names an environment variable, not a credential.
|
||||
@@ -55,6 +59,12 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
|
||||
if config.Extends != nil {
|
||||
fields["extends"] = map[string]any{"id": *config.Extends}
|
||||
}
|
||||
if config.Provider != nil {
|
||||
fields["provider"] = *config.Provider
|
||||
}
|
||||
if config.ZeroCost != nil {
|
||||
fields["zero_cost"] = *config.ZeroCost
|
||||
}
|
||||
if config.Endpoint != nil {
|
||||
fields["endpoint"] = *config.Endpoint
|
||||
}
|
||||
|
||||
70
cache_hit_contract_test.go
Normal file
70
cache_hit_contract_test.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestDuplicateContextAndRequestGroupEnrollment(t *testing.T) {
|
||||
charge := .5
|
||||
engine := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil
|
||||
})))
|
||||
var group promptkit.ExecutionGroup
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||||
if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := group.Snapshot()
|
||||
if len(got.Members) != 1 || got.Attempts != 1 || got.CostUSD.Value != .5 {
|
||||
t.Fatalf("summary=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) {
|
||||
known := func(v int) promptkit.ObservedTokenCount {
|
||||
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
input, reads promptkit.ObservedTokenCount
|
||||
ratio float64
|
||||
ok bool
|
||||
}{
|
||||
{name: "fraction", input: known(8), reads: known(3), ratio: .375, ok: true},
|
||||
{name: "zero reads", input: known(8), reads: known(0), ok: true},
|
||||
{name: "zero input", input: known(0), reads: known(0)},
|
||||
{name: "negative reads", input: known(8), reads: known(-1)},
|
||||
{name: "reads exceed input", input: known(8), reads: known(9)},
|
||||
{name: "unknown", input: known(8)},
|
||||
{name: "incomplete", input: known(8), reads: promptkit.ObservedTokenCount{Value: 3, Known: true}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
usage := promptkit.ObservedTokenUsage{PromptTokens: tc.input, CachedTokens: tc.reads}
|
||||
got, ok := usage.CacheHitRatio()
|
||||
if ok != tc.ok || math.Abs(got-tc.ratio) > 1e-12 {
|
||||
t.Fatalf("ratio=%v available=%t", got, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupCacheRatioUsesSummedTokensAndDeduplicatesTerminals(t *testing.T) {
|
||||
known := func(v int) promptkit.ObservedTokenCount {
|
||||
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
|
||||
}
|
||||
var group promptkit.ExecutionGroup
|
||||
first := promptkit.ExecutionEvent{RunID: "a", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(10), CachedTokens: known(10), ReasoningTokens: known(1)}}
|
||||
second := promptkit.ExecutionEvent{RunID: "b", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(90), CachedTokens: known(0), ReasoningTokens: known(2)}}
|
||||
group.Observe(first)
|
||||
group.Observe(first)
|
||||
group.Observe(second)
|
||||
snapshot := group.Snapshot()
|
||||
ratio, ok := snapshot.Usage.CacheHitRatio()
|
||||
if !ok || math.Abs(ratio-.1) > 1e-12 || snapshot.Usage.ReasoningTokens != known(3) {
|
||||
t.Fatalf("summary=%+v ratio=%v available=%t", snapshot, ratio, ok)
|
||||
}
|
||||
}
|
||||
@@ -134,6 +134,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||
InputHashes: copyStringMap(result.InputHashes),
|
||||
Usage: fromDomainTokenUsage(result.Usage),
|
||||
CostUSD: fromDomainCost(result.CostUSD),
|
||||
StartTime: result.StartTime,
|
||||
EndTime: result.EndTime,
|
||||
Duration: result.Duration,
|
||||
@@ -158,6 +159,7 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
|
||||
Content: resp.Content,
|
||||
Usage: toDomainTokenUsage(resp.Usage),
|
||||
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
|
||||
CostUSD: copyFloat64Ptr(resp.CostUSD),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +170,7 @@ func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsage
|
||||
return &domain.TokenUsagePresence{
|
||||
PromptTokens: presence.PromptTokens,
|
||||
CompletionTokens: presence.CompletionTokens,
|
||||
ReasoningTokens: presence.ReasoningTokens,
|
||||
TotalTokens: presence.TotalTokens,
|
||||
CachedTokens: presence.CachedTokens,
|
||||
CacheWriteTokens: presence.CacheWriteTokens,
|
||||
@@ -265,7 +268,7 @@ func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
|
||||
}
|
||||
|
||||
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
||||
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, Provider: copyStringPtr(v.Provider), ZeroCost: copyBoolPtr(v.ZeroCost), APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||
if v.ExtraParams != nil {
|
||||
data := copyAnyMap(v.ExtraParams)
|
||||
out.ExtraParams = &data
|
||||
@@ -356,6 +359,7 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
|
||||
return TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
ReasoningTokens: usage.ReasoningTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
@@ -366,6 +370,7 @@ func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: usage.PromptTokens,
|
||||
CompletionTokens: usage.CompletionTokens,
|
||||
ReasoningTokens: usage.ReasoningTokens,
|
||||
TotalTokens: usage.TotalTokens,
|
||||
CachedTokens: usage.CachedTokens,
|
||||
CacheWriteTokens: usage.CacheWriteTokens,
|
||||
|
||||
@@ -3,6 +3,7 @@ package promptkit_test
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
@@ -10,6 +11,31 @@ import (
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestCoordinateObserverOrderWithContext(t *testing.T) {
|
||||
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
||||
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var order []string
|
||||
observer := func(name string) promptkit.ExecutionObserver {
|
||||
return func(promptkit.ExecutionEvent) { order = append(order, name) }
|
||||
}
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), observer("context"))
|
||||
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: observer("coordinator"), Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.RemainingTurns == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &promptkit.RunRequest{PromptID: "prompt", Observer: observer("request")}, nil
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(order, []string{"request", "coordinator", "context", "request", "coordinator", "context", "request", "coordinator", "context"}) {
|
||||
t.Fatalf("observer order = %v", order)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
|
||||
42
cost.go
Normal file
42
cost.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package promptkit
|
||||
|
||||
import "go.maximumdirect.net/promptkit/internal/domain"
|
||||
|
||||
// CostSource identifies how an observed USD cost was supplied.
|
||||
type CostSource string
|
||||
|
||||
const (
|
||||
// CostSourceNone denotes no known sourced contribution.
|
||||
CostSourceNone CostSource = ""
|
||||
// CostSourceProvider denotes a reported provider or injected-client amount.
|
||||
CostSourceProvider CostSource = "provider"
|
||||
// CostSourceConfiguredZero denotes an explicitly configured zero fallback.
|
||||
CostSourceConfiguredZero CostSource = "configured_zero"
|
||||
// CostSourceMixed denotes both provider and configured-zero contributions.
|
||||
CostSourceMixed CostSource = "mixed"
|
||||
)
|
||||
|
||||
// ObservedCost reports an available USD subtotal and its coverage and source.
|
||||
// Known means at least one contribution is available. Complete means all
|
||||
// included dispatched attempts have an amount under the selected policy; it
|
||||
// does not verify billing. A completed run with no dispatched attempts, or an
|
||||
// empty group, has known complete zero with no source. A group with members but
|
||||
// no terminals has unknown cost. Amounts are unrounded binary floating point.
|
||||
type ObservedCost struct {
|
||||
// Value is the known USD subtotal.
|
||||
Value float64 `json:"value"`
|
||||
// Known reports whether a subtotal is available.
|
||||
Known bool `json:"known"`
|
||||
// Complete reports whether all included dispatched attempts are covered.
|
||||
Complete bool `json:"complete"`
|
||||
// Source summarizes the known contributions.
|
||||
Source CostSource `json:"source"`
|
||||
}
|
||||
|
||||
func fromDomainCost(value domain.ObservedCost) ObservedCost {
|
||||
return ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: CostSource(value.Source)}
|
||||
}
|
||||
|
||||
func toDomainCost(value ObservedCost) domain.ObservedCost {
|
||||
return domain.ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: domain.CostSource(value.Source)}
|
||||
}
|
||||
131
cost_prepared_contract_test.go
Normal file
131
cost_prepared_contract_test.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestPreparedCostAndRejectedClaims(t *testing.T) {
|
||||
calls := 0
|
||||
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return &promptkit.GenerateResponse{Content: "bad"}, nil
|
||||
}
|
||||
return nil, errors.New("repair failed")
|
||||
})))
|
||||
ctx := context.Background()
|
||||
var events []promptkit.ExecutionEvent
|
||||
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
|
||||
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
|
||||
if err != nil || len(events) != 0 {
|
||||
t.Fatalf("prepare error=%v events=%+v", err, events)
|
||||
}
|
||||
result, err := engine.RunPrepared(ctx, handle)
|
||||
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
|
||||
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
|
||||
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
|
||||
}
|
||||
events = nil
|
||||
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
|
||||
t.Fatalf("reused err=%v events=%+v", err, events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
|
||||
childCalls := 0
|
||||
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
childCalls++
|
||||
charge := .2
|
||||
if childCalls == 2 {
|
||||
charge = .3
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||||
})))
|
||||
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var childRun, childPrepared *promptkit.RunResult
|
||||
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
var err error
|
||||
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
childPrepared, err = child.RunPrepared(ctx, handle)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
charge := 1.0
|
||||
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||
})))
|
||||
var group promptkit.ExecutionGroup
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||||
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
|
||||
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||||
t.Fatalf("summary=%+v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
|
||||
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
charge := 2.0
|
||||
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
|
||||
})))
|
||||
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
charge := 1.0
|
||||
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||
})))
|
||||
var group promptkit.ExecutionGroup
|
||||
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
|
||||
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
|
||||
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
charge := .5
|
||||
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||||
})))
|
||||
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
charge := 1.0
|
||||
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||
})))
|
||||
var childResult *promptkit.RunResult
|
||||
var childErr error
|
||||
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
|
||||
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var group promptkit.ExecutionGroup
|
||||
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
|
||||
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
|
||||
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||||
t.Fatalf("summary=%+v", summary)
|
||||
}
|
||||
}
|
||||
127
cost_run_contract_test.go
Normal file
127
cost_run_contract_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type costRoundTrip func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||
|
||||
func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}},
|
||||
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}
|
||||
options = append(options, promptkit.WithBundledOpenRouter(false))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
|
||||
for _, observed := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) {
|
||||
calls := 0
|
||||
transport := costRoundTrip(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||
})
|
||||
engine := costEngine(t, false, &http.Client{Transport: transport})
|
||||
var events []promptkit.ExecutionEvent
|
||||
req := promptkit.RunRequest{PromptID: "prompt"}
|
||||
if observed {
|
||||
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
|
||||
}
|
||||
result, err := engine.Run(context.Background(), req)
|
||||
want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider}
|
||||
if err != nil || result == nil || result.CostUSD != want || calls != 2 {
|
||||
t.Fatalf("result=%+v calls=%d err=%v", result, calls, err)
|
||||
}
|
||||
if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
|
||||
charge := .125
|
||||
invalid := math.Inf(1)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
response *promptkit.GenerateResponse
|
||||
err error
|
||||
want promptkit.ObservedCost
|
||||
}{
|
||||
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
|
||||
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
|
||||
{"nonfinite report falls back", &promptkit.GenerateResponse{Content: "ok", CostUSD: &invalid}, nil, promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return tc.response, tc.err
|
||||
})))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) {
|
||||
if e.Kind == promptkit.ExecutionFinished {
|
||||
terminal = e
|
||||
}
|
||||
}})
|
||||
if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) {
|
||||
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) {
|
||||
charge := .1
|
||||
calls := 0
|
||||
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return &promptkit.GenerateResponse{Content: "invalid"}, nil
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil
|
||||
})))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) {
|
||||
if e.Kind == promptkit.ExecutionFinished {
|
||||
terminal = e
|
||||
}
|
||||
}})
|
||||
want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed}
|
||||
if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want {
|
||||
t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) {
|
||||
engine := costEngine(t, true, nil)
|
||||
var terminal promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) {
|
||||
if e.Kind == promptkit.ExecutionFinished {
|
||||
terminal = e
|
||||
}
|
||||
}})
|
||||
if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
|
||||
}
|
||||
}
|
||||
@@ -728,6 +728,40 @@ counts, but its zero counts remain unknown to observations; supply explicit
|
||||
presence when measured zeros matter. Existing `RunResult.Usage` sums returned
|
||||
values independently of observation presence.
|
||||
|
||||
For an application-owned invocation spanning several calls or engines, attach
|
||||
an observer once to a context and propagate it to those calls:
|
||||
|
||||
```go
|
||||
var group promptkit.ExecutionGroup
|
||||
ctx = promptkit.WithExecutionObserver(ctx, group.Observe)
|
||||
result, err := engine.Run(ctx, request)
|
||||
// After joining all application-owned work, inspect group.Snapshot(), even on error.
|
||||
```
|
||||
|
||||
Explicit request observers run first, followed by contextual observers in
|
||||
registration order. Coordinated continuations pick up the context through
|
||||
`Run`; prepared handles retain the request observer but use the execution
|
||||
context's observers, not the preparation context's. Each nested call has its
|
||||
own run ID and accounting. Duplicate registration can deliver duplicate events;
|
||||
groups count each run's first terminal once.
|
||||
|
||||
Reasoning tokens are a subset of completion tokens; cache reads and writes are
|
||||
input details, not extra tokens. `ObservedTokenUsage.CacheHitRatio()` derives
|
||||
the cache-read fraction only when input and read totals are known, complete,
|
||||
positive-denominator, and consistent. It divides aggregate counts rather than
|
||||
averaging per-run ratios and is independent of cache writes.
|
||||
|
||||
`RunResult.CostUSD` is available even without an observer; attempt and terminal
|
||||
events and group snapshots also report cost. `Known` denotes an available
|
||||
subtotal, `Complete` covers dispatched attempts, and `Source` distinguishes
|
||||
provider reports, an explicitly configured zero fallback, and mixed sources.
|
||||
Reported cost, including zero, takes precedence over fallback. Unknown failed
|
||||
attempts make a subtotal incomplete; configured zero may cover them. Terminal
|
||||
events can report cost when a failed operation returns no result. Neither
|
||||
complete reporting nor provider attribution verifies an invoice; consumers
|
||||
choose presentation precision. See [backend definitions](../formats.md#backend-definitions)
|
||||
and the [integration contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||
|
||||
Callbacks run synchronously, in order for one operation, and can overlap for
|
||||
different operations. Synchronize shared callback state and avoid blocking on
|
||||
work that needs the same run's admission slot. Callback panics propagate;
|
||||
@@ -879,15 +913,15 @@ outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||
})
|
||||
```
|
||||
|
||||
`assess` is application code and explicitly selects its observer on any nested
|
||||
run. A completed turn releases its admission lease before the callback, so that
|
||||
`assess` is application code and can select its observer explicitly or inherit
|
||||
one from the context. A completed turn releases its admission lease before the callback, so that
|
||||
assessment may use the same backend. `stopped` and `exhausted` describe control
|
||||
flow, not content acceptance. Cancellation before callback delivery can return one
|
||||
completed response in `Undelivered`; errors retain their ordinary identities.
|
||||
|
||||
### Collect Selected Execution Facts
|
||||
|
||||
Groups are independent of continuation and enrollment is explicit:
|
||||
Groups are independent of continuation and can be enrolled explicitly:
|
||||
|
||||
```go
|
||||
var group, validatorGroup promptkit.ExecutionGroup
|
||||
@@ -904,7 +938,7 @@ _, _ = summary, validatorSummary
|
||||
```
|
||||
|
||||
Only the first terminal event per run contributes counts and usage; coverage of
|
||||
currently observed members is separate from token-usage completeness. Empty
|
||||
currently observed members is separate from token and cost completeness. Empty
|
||||
groups report complete, known zero totals, while members without terminals do
|
||||
not invent totals. Overlapping groups are not generally additive. See the
|
||||
[offline example](../../examples/go-library/observe/main.go) for complete setup.
|
||||
|
||||
@@ -223,7 +223,7 @@ Backend definitions are supplied through caller-selected configuration
|
||||
sources. The conventional backend IDs are names only; PromptKit does not
|
||||
install definitions for them.
|
||||
|
||||
A backend definition may have `endpoint`, `api_key_env`,
|
||||
A backend definition may have `provider`, `zero_cost`, `endpoint`, `api_key_env`,
|
||||
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
|
||||
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
|
||||
user information, query, or fragment. An omitted or null API-key environment
|
||||
@@ -237,6 +237,17 @@ unrecognized backend field is application data and is not automatically redacted
|
||||
by Promptkit. Applications own sensitivity declarations and disclosure policy
|
||||
for their extension fields.
|
||||
|
||||
`provider` selects response and diagnostic conventions independently of the
|
||||
backend ID, endpoint, or model. Its final value is trimmed and lowercased;
|
||||
omitted, null, or blank selects `openai-compatible` at runtime. Supported
|
||||
nonblank values are `openai-compatible`, `openrouter`, and `vllm`. `zero_cost`
|
||||
is an optional Boolean: true supplies a zero-cost fallback when a dispatched
|
||||
attempt has no usable reported cost; omitted, null, and false disable it.
|
||||
Neither setting is sent to the provider. They follow normal layering and
|
||||
inheritance, and endpoint overrides retain the selected backend's policy.
|
||||
Resource inspection shows declared normalized values without inventing runtime
|
||||
defaults; configuration inspection retains source history.
|
||||
|
||||
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
||||
It may contain `default_profile`, `reachability`, and `authentication`:
|
||||
|
||||
@@ -253,7 +264,7 @@ backends:
|
||||
endpoint: https://openrouter.ai/api/v1/key
|
||||
```
|
||||
|
||||
An omitted or null `smoke_tests` block disables these metadata defaults.
|
||||
An omitted or null `smoke_tests` block removes its declared metadata.
|
||||
`default_profile` is preserved as an exact, nonblank profile ID; its existence
|
||||
is checked only if an inference diagnostic selects that default. Omitted/null
|
||||
`default_profile` means there is no inference default. Omitted/null
|
||||
@@ -262,12 +273,22 @@ reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
|
||||
host; unlike the backend base endpoint, its path and query are allowed. Null
|
||||
means fallback, while an explicit blank value is invalid.
|
||||
|
||||
Omitted/null `authentication` means no authentication adapter is declared. A
|
||||
non-null object requires `kind`, which is trimmed and case-normalized; the
|
||||
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
|
||||
Omitted/null `authentication` leaves the provider's diagnostic default in
|
||||
effect; `openrouter` defaults to `openrouter_key`, `vllm` defaults to
|
||||
`vllm_models`, and the generic provider has no default. A non-null object's
|
||||
optional `kind` is trimmed and case-normalized; accepted
|
||||
values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint
|
||||
uses the adapter's backend-relative destination (`openrouter_key` appends
|
||||
`/key` to the effective backend base); an explicit endpoint follows the same
|
||||
full-URL rules as reachability. `none` declares authentication inapplicable.
|
||||
Without a selected adapter, authentication diagnostics report an unsupported
|
||||
selection. An endpoint-only authentication
|
||||
block overrides the destination of a provider default; `kind: none` suppresses
|
||||
that default.
|
||||
Previously, a supplied authentication object required `kind`. Endpoint-only
|
||||
objects now use a provider default where available; clearing the object
|
||||
restores that default rather than suppressing it. Use `kind: none` to suppress
|
||||
authentication testing.
|
||||
These values are configuration only: engine construction does not contact a
|
||||
backend.
|
||||
|
||||
@@ -358,7 +379,7 @@ does not disable the caller context or HTTP transport timeout.
|
||||
|
||||
### Bundled OpenRouter Configuration
|
||||
|
||||
The default bundle installs root backend `openrouter`, endpoint
|
||||
The default bundle installs root backend `openrouter`, provider `openrouter`, endpoint
|
||||
`https://openrouter.ai/api/v1`, optional credential environment name
|
||||
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
|
||||
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
|
||||
|
||||
@@ -58,7 +58,7 @@ report facts, and injected-client behavior.
|
||||
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
|
||||
one unauthenticated `GET` to the selected diagnostic URL; an observed response
|
||||
counts as reachable even when its status is non-2xx. Authentication behavior
|
||||
is selected from backend metadata, not inferred from the host. The
|
||||
is selected from backend metadata and provider defaults, not inferred from the host. The
|
||||
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
|
||||
backend base, unless configuration supplies a full diagnostic endpoint (see
|
||||
the [backend format reference](../formats.md#backend-definitions)). It may send
|
||||
@@ -73,6 +73,15 @@ not validation of the full account document. Other statuses and unrecognized
|
||||
recognition allowance reports `authentication_response_limit_exceeded`, leaving
|
||||
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
|
||||
|
||||
The `vllm_models` adapter issues one `GET` to `/models` relative to the
|
||||
configured base unless an explicit diagnostic endpoint is supplied. It sends
|
||||
an available credential but does not require one. A 200 response is recognized
|
||||
when it is a JSON object with `object: "list"` and an array-valued `data`;
|
||||
empty arrays are accepted. A 401 means rejection; other responses do not
|
||||
establish acceptance. Recognition uses the same bounded response handling as
|
||||
the key probe. Acceptance does not establish that authentication is enforced or
|
||||
that inference is authorized.
|
||||
|
||||
Diagnostic HTTP evidence describes the last request observed at Promptkit's
|
||||
RoundTripper boundary and any response associated with it. It includes the
|
||||
selected destination without URL user information, response status, and
|
||||
@@ -154,7 +163,22 @@ data, a second JSON value, absent choices, missing content, `null` content,
|
||||
non-string content, and size overflow are malformed responses and return no
|
||||
partial result.
|
||||
|
||||
Each of the five token fields retains whether the provider reported a
|
||||
For OpenRouter and vLLM, the client also reads reasoning tokens from
|
||||
`usage.completion_tokens_details.reasoning_tokens`. OpenRouter cache writes
|
||||
come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
|
||||
`usage.prompt_tokens_details.created_cache_tokens`. The generic provider keeps
|
||||
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
|
||||
fractional, or unrepresentable new details remain unknown without invalidating
|
||||
otherwise usable content. Existing common count decoding remains strict.
|
||||
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
|
||||
amounts are ignored without invalidating content. The generic and vLLM
|
||||
providers do not interpret that field.
|
||||
vLLM prompt-token details depend on server configuration, including
|
||||
`--enable-prompt-tokens-details`; reasoning details depend on a configured
|
||||
reasoning parser. Servers without these details remain usable. vLLM cache
|
||||
writes describe local prefix-cache creation, not a billable write charge.
|
||||
|
||||
Each common token field retains whether the provider reported a
|
||||
non-null integer, including zero. Omitted and `null` fields are unknown for
|
||||
observations. A strictly decoded successful envelope can yield observed usage
|
||||
even when a later missing choice or content makes the attempt malformed.
|
||||
|
||||
@@ -79,7 +79,7 @@ other backend IDs.
|
||||
The coordinator does not retry injected clients; it normalizes a nil successful
|
||||
response as malformed generation. The built-in HTTP client remains responsible
|
||||
for provider transport behavior. The runner, rather than the
|
||||
capacity layer, sums all five usage fields from the initial response and every
|
||||
capacity layer, sums usage fields from the initial response and every
|
||||
completed repair response into the successful run result.
|
||||
|
||||
An invocation-local observer receives attempt events after active permit
|
||||
|
||||
@@ -35,6 +35,12 @@ unknown usage. Transport, timeout, malformed, and noneligible status failures
|
||||
stop the sequence without retry exhaustion. Context is checked before each
|
||||
wait, acquisition, and dispatch.
|
||||
|
||||
At every dispatched completion the coordinator resolves any reported cost
|
||||
against the frozen backend zero-cost policy and adds it once to the run's
|
||||
[cost accumulator](observation.md), independently of observer selection.
|
||||
Injected responses may retain reported cost alongside an error. Undispatched
|
||||
acquisitions and retry waits contribute no cost.
|
||||
|
||||
The public client adapter translates explicitly supplied public transport and
|
||||
malformed-response identities into internal categories while retaining the
|
||||
original error chain. This keeps injected attempt classifications consistent
|
||||
|
||||
@@ -142,6 +142,14 @@ known. Strictly decoded usage can still be observed if later choice/content
|
||||
checks fail; invalid framing or a non-success error body contributes none.
|
||||
Existing successful-result accounting remains value-based.
|
||||
|
||||
The captured provider selects additional response details and authentication
|
||||
diagnostic defaults. Optional new reasoning, cache-write, and OpenRouter cost
|
||||
fields are decoded independently, without relaxing common-field or framing
|
||||
validation. Cost remains an optional raw amount at this boundary; shared
|
||||
accounting applies configured-zero fallback after dispatch. The integration
|
||||
contract specifies the [wire mappings](../integrations/openai-compatible-chat.md#response-handling)
|
||||
and [probe recognition](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||
|
||||
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
||||
only normalized detail from the bounded recognized envelope. It retains
|
||||
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
||||
|
||||
@@ -18,11 +18,14 @@ Timeout attribution is finalized and the child scope released before terminal
|
||||
delivery, so observer latency cannot change the already selected outcome.
|
||||
Prepared-content notification is a separate opt-in callback, not an event.
|
||||
|
||||
The facade masks inherited recorder and prepared-claim state when entering
|
||||
The facade masks inherited recorder, cost, and prepared-claim state when entering
|
||||
preparation or execution. Delegated calls from an injected collaborator retain
|
||||
the caller's cancellation, deadline, and unrelated context values, but do not
|
||||
reuse the enclosing invocation's accounting. Only the new request's observer
|
||||
(or the claimed handle's retained observer) enables its observation lifecycle.
|
||||
reuse the enclosing invocation's accounting. Ordinary runs compose the request
|
||||
observer before any observers selected by the invocation context; contextual
|
||||
selection survives this boundary. Prepared executions compose the retained
|
||||
request observer with selection from the execution context after a successful
|
||||
claim; the preparation context does not select execution observers.
|
||||
|
||||
A recorder keeps only counters, resolved identity fields, and the selected
|
||||
callback. It has no global history, queue, goroutine, or mutex held across a
|
||||
@@ -53,12 +56,24 @@ attempts above the first in a generation, and repair generations with at least
|
||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||
attempt events.
|
||||
|
||||
An execution group is a zero-ready synchronized collector of explicitly selected
|
||||
Ordinary runs and claimed prepared executions install invocation-local cost
|
||||
accounting even without an observer.
|
||||
The generation coordinator resolves cost once per dispatched attempt and adds
|
||||
it to that state independently of event delivery. The result and recorder read
|
||||
the same accumulator; groups merge first terminal costs independently of
|
||||
terminal coverage.
|
||||
|
||||
Reasoning counts use the same per-field rules and are a subset of completion
|
||||
tokens. The public cache-hit helper derives a ratio from complete, consistent
|
||||
summed input and cache-read counts; it does not aggregate ratios or use writes.
|
||||
|
||||
An execution group is a zero-ready synchronized collector of selected
|
||||
events. It enrolls by run ID and aggregates each run's first terminal counters and
|
||||
presence-aware usage exactly once. Snapshots copy members and totals under one
|
||||
lock; terminal coverage and token completeness are distinct. Observer fan-out
|
||||
copies the callback slice and invokes callbacks synchronously without holding a
|
||||
group lock. No group history, registry, or automatic nested membership exists.
|
||||
group lock. No group history or registry exists; contextual selection can enroll
|
||||
nested ordinary runs that use the scoped context.
|
||||
|
||||
Smoke diagnostics mask inherited execution-recorder state while invoking
|
||||
generation or HTTP collaborators. Their private attempt evidence is assembled
|
||||
|
||||
@@ -21,7 +21,7 @@ contributor workflow and validation.
|
||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||
| `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded retries, per-attempt cost, ordinary observations, and separate diagnostic evidence. | [Internal generation coordination](generation.md) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
@@ -30,8 +30,8 @@ contributor workflow and validation.
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible requests, provider-specific response interpretation and authentication probes, bounded decoding, deadlines, and diagnostic evidence. | [Internal model client](llm.md) |
|
||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware token and cost accounting, contextual observer selection, synchronous event delivery, and terminal-based group aggregation; smoke diagnostics retain separate evidence. | [Internal execution observation](observation.md) |
|
||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||
|
||||
The internal engine assembles these internal components without exposing their
|
||||
|
||||
@@ -216,10 +216,16 @@ ordinary `Run` reuses its run ID and carries it to the generation coordinator
|
||||
through context. Prompt and profile identities are enriched only after their
|
||||
respective resolutions succeed, before admission. The facade starts observation
|
||||
before request conversion and emits its terminal event after public error
|
||||
mapping and after the runner's deferred admission release. Preparation and
|
||||
prepared execution follows the same terminal mapping after a successful claim;
|
||||
mapping and after the runner's deferred admission release. Preparation emits no
|
||||
events; prepared execution follows the same terminal mapping after a successful claim;
|
||||
invalid or discarded handles do not acquire an observer.
|
||||
|
||||
Ordinary runs establish fresh private cost state before observation; a claimed
|
||||
prepared execution establishes its own state after claim. Contextual observers
|
||||
remain available at public boundaries, while enclosing recorder and cost state
|
||||
are masked. Prepared handles retain the request observer and frozen provider
|
||||
policy, not preparation-context selection or an accumulator.
|
||||
|
||||
The facade establishes an optional operation scope for ordinary runs before
|
||||
observation and preparation. A claimed prepared execution establishes its own
|
||||
scope from the retained duration; preparation and idle time do not consume it.
|
||||
@@ -273,11 +279,13 @@ run identifier, and UTC timing. The same effective session reaches initial
|
||||
generation and any repair attempt through the rendered prompt. The same
|
||||
effective target and presence metadata, including backend identity and direct
|
||||
credential during execution, reaches generation and every repair attempt.
|
||||
Result usage is the field-wise sum of all five usage values from the initial
|
||||
Result usage is the field-wise sum of usage values from the initial
|
||||
response and every completed repair response. Final raw output, artifact, and
|
||||
validation state still come from the last candidate. A repair error returns no
|
||||
partial run result or partial result usage, although the invocation-local
|
||||
observer retains completed-attempt usage and reports it in the terminal event.
|
||||
Result cost instead snapshots the shared accumulator across all dispatched
|
||||
attempts, including retries and repairs, whether or not observation is enabled.
|
||||
|
||||
## Failure Categories
|
||||
|
||||
|
||||
88
docs/releases/v0.25.0.md
Normal file
88
docs/releases/v0.25.0.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# Promptkit v0.25.0
|
||||
|
||||
This minor release adds invocation-scoped execution observation, provider-specific
|
||||
reasoning and cache usage, and optional USD cost reporting. Backend definitions
|
||||
can now select API conventions independently of their names and endpoints.
|
||||
Consumers upgrading from v0.24.x should review the configuration and accounting
|
||||
changes below.
|
||||
|
||||
## Added And Changed
|
||||
|
||||
- `WithExecutionObserver` selects observation through a caller-owned context
|
||||
across nested, concurrent, coordinated, and multi-engine calls. Request
|
||||
observers run before contextual observers. Prepared handles retain their
|
||||
explicit request observer but select contextual observers when executed,
|
||||
not when prepared. Each execution retains independent accounting; groups
|
||||
count each run's first terminal once. See the
|
||||
[observation guide](../consumers/pkg-promptkit.md#observe-an-ordinary-run)
|
||||
and [offline example](../../examples/go-library/observe/main.go).
|
||||
- Backend `provider` selects `openai-compatible`, `openrouter`, or `vllm`
|
||||
conventions. Omitted, null, or blank selects the generic OpenAI-compatible
|
||||
behavior. The bundled OpenRouter backend explicitly selects `openrouter`.
|
||||
`InspectBackend` exposes declared provider and `zero_cost` settings without
|
||||
manufacturing runtime defaults. See the
|
||||
[backend reference](../formats.md#backend-definitions).
|
||||
- Usage values, presence indicators, observations, and group summaries include
|
||||
`ReasoningTokens`. OpenRouter and vLLM decoding includes their respective
|
||||
reasoning, cache-read, and cache-write fields. Reasoning is a subset of
|
||||
completion tokens; cache activity does not add to reported token totals.
|
||||
`ObservedTokenUsage.CacheHitRatio` derives a ratio from complete, consistent
|
||||
input and cache-read totals. See the
|
||||
[response contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||
- `RunResult.CostUSD`, execution events, and group summaries report a USD
|
||||
subtotal with availability, completeness, and source metadata. OpenRouter
|
||||
supplies optional `usage.cost`; injected clients can supply
|
||||
`GenerateResponse.CostUSD`. Explicit backend `zero_cost: true` supplies a
|
||||
zero fallback when no usable amount is reported. A reported amount, including
|
||||
zero, takes precedence. See the [cost GoDoc](../../cost.go).
|
||||
- Authentication diagnostics can use provider defaults: `openrouter_key` for
|
||||
OpenRouter and the new `vllm_models` adapter for vLLM. The latter probes
|
||||
base-relative `/models` and reports recognized response facts without
|
||||
asserting authentication enforcement or inference authorization. See the
|
||||
[diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||
|
||||
## Compatibility And Upgrade
|
||||
|
||||
- For application-defined OpenRouter backends, set `provider: openrouter` to
|
||||
enable OpenRouter-specific usage and cost decoding and its authentication
|
||||
default. Backend IDs and endpoint hostnames do not select conventions.
|
||||
Generic backends retain their existing request and token conventions and
|
||||
do not decode monetary cost. Unsupported nonblank provider names are
|
||||
configuration errors; supplied names are trimmed and lowercased.
|
||||
- OpenRouter cache-write reporting now reads
|
||||
`usage.prompt_tokens_details.cache_write_tokens`, not the generic top-level
|
||||
`usage.cache_write_tokens` field. vLLM uses
|
||||
`usage.prompt_tokens_details.created_cache_tokens`. Missing or unusable new
|
||||
details remain unknown rather than invalidating otherwise usable content.
|
||||
Availability depends on the provider and server settings; consult the
|
||||
[response contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||
- A supplied authentication object no longer requires `kind`: an omitted or
|
||||
null kind uses the provider default. Clearing the authentication block also
|
||||
restores that default. Set `kind: none` to declare authentication testing
|
||||
inapplicable. The generic provider has no authentication default; explicit
|
||||
adapters remain supported independently of provider selection. Probes still
|
||||
run only when explicitly requested. See the
|
||||
[backend reference](../formats.md#backend-definitions).
|
||||
- Cost includes all dispatched attempts, including transient retries and
|
||||
structural repairs, and is available on successful results without an
|
||||
observer. On operational failure, observations can retain cost even though
|
||||
no partial result is returned. Existing `RunResult.Usage` continues to sum
|
||||
successful generation responses; presence-aware observations cover all
|
||||
dispatched attempts. Do not add terminal totals to attempt totals.
|
||||
- Unknown cost is distinct from reported or configured zero. No backend is
|
||||
automatically declared zero-cost, and no token-rate estimation is performed.
|
||||
Group cost completeness describes included terminals, independently of
|
||||
terminal coverage. Consumers own presentation and interpretation; complete
|
||||
reporting does not verify billing.
|
||||
- Result JSON adds `cost_usd`, and token JSON adds `reasoning_tokens`.
|
||||
Update strict decoders if necessary. Older result JSON without cost decodes
|
||||
to unknown cost. Consumers using positional literals for expanded public
|
||||
structs may need to update them. See the [public types](../../types.go).
|
||||
|
||||
The Go consumer minimum remains 1.27.0; dependencies are unchanged from v0.24.0.
|
||||
|
||||
```sh
|
||||
go get go.maximumdirect.net/promptkit@v0.25.0
|
||||
```
|
||||
|
||||
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/docs/oss/attribution.md)
|
||||
24
engine.go
24
engine.go
@@ -535,7 +535,8 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr
|
||||
// returns no handle on error. A nil Engine returns an error matching
|
||||
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
||||
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
||||
// privately for RunPrepared; preparation itself emits no events.
|
||||
// privately for RunPrepared; preparation itself emits no events and does not
|
||||
// retain contextual observers.
|
||||
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
|
||||
// starting a timer or invoking the notification during preparation.
|
||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||
@@ -602,16 +603,24 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
||||
// result on error.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
ctx = observation.WithoutExecutionState(ctx)
|
||||
cost := &observation.CostAccumulator{}
|
||||
ctx = observation.WithCostAccumulator(ctx, cost)
|
||||
scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
|
||||
defer scope.Close()
|
||||
ctx = scope.Context
|
||||
if req.Observer == nil {
|
||||
var explicit domain.ExecutionObserver
|
||||
if req.Observer != nil {
|
||||
explicit = func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) }
|
||||
}
|
||||
observer := observation.Compose(explicit, observation.ObserverFromContext(ctx))
|
||||
if observer == nil {
|
||||
result, err := e.run(ctx, req)
|
||||
err = mapPublicError(scope.Finalize(err))
|
||||
scope.Close()
|
||||
return result, err
|
||||
}
|
||||
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
||||
recorder, idErr := observation.New(observer)
|
||||
recorder.SetCostAccumulator(cost)
|
||||
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
|
||||
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
|
||||
}
|
||||
@@ -682,10 +691,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
// content-validation rejection, including repair exhaustion, is returned in
|
||||
// RunResult, not as an operational
|
||||
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An
|
||||
// operational error returns no partial RunResult. A retained Observer begins
|
||||
// only after a successful atomic claim, with identities from the frozen
|
||||
// snapshot, and receives a terminal event on normal return after public error
|
||||
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
|
||||
// operational error returns no partial RunResult. The retained Observer and
|
||||
// execution-context observers begin only after a successful atomic claim, with
|
||||
// identities from the frozen snapshot, and receive a terminal event on normal
|
||||
// return after public error mapping. Invalid, foreign, discarded, or reused
|
||||
// handles emit no events.
|
||||
// Callbacks run outside the handle and capacity locks; panics propagate and
|
||||
// do not promise a terminal event.
|
||||
// The retained RunRequest.OperationTimeout starts after claim; the retained
|
||||
|
||||
@@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
||||
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
|
||||
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens},
|
||||
"usage": map[string]any{
|
||||
"prompt_tokens": promptTokens, "completion_tokens": completionTokens,
|
||||
"prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1},
|
||||
"completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01,
|
||||
},
|
||||
})
|
||||
return httpResponse(http.StatusOK, string(body))
|
||||
}
|
||||
@@ -80,6 +84,10 @@ type summary struct {
|
||||
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts uint64 `json:"group_attempts"`
|
||||
GroupCost promptkit.ObservedCost `json:"group_cost"`
|
||||
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
|
||||
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
|
||||
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
}
|
||||
@@ -97,6 +105,8 @@ func run() (summary, error) {
|
||||
filesystem := configkit.NewFilesystem("example schemas", assets)
|
||||
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||
backend.ID = backendID
|
||||
provider, zeroCost := "openrouter", true
|
||||
backend.Provider, backend.ZeroCost = &provider, &zeroCost
|
||||
inputs = append(inputs,
|
||||
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
|
||||
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||
@@ -117,8 +127,9 @@ func run() (summary, error) {
|
||||
|
||||
var proposalLog eventLog
|
||||
var executions promptkit.ExecutionGroup
|
||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe)
|
||||
proposal, err := engine.Run(ctx, promptkit.RunRequest{
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record,
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
@@ -133,8 +144,8 @@ func run() (summary, error) {
|
||||
var assessmentLog eventLog
|
||||
var preparedDetails promptkit.PreparedRun
|
||||
decision := "rejected"
|
||||
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
|
||||
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
|
||||
coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||
Start: proposal.Exchange, AdditionalTurns: 1,
|
||||
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result == nil {
|
||||
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
|
||||
@@ -142,7 +153,7 @@ func run() (summary, error) {
|
||||
if !turn.Result.Validation.IsValid {
|
||||
return nil, fmt.Errorf("follow-up failed structural validation")
|
||||
}
|
||||
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
|
||||
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -181,6 +192,8 @@ func run() (summary, error) {
|
||||
return summary{}, err
|
||||
}
|
||||
group := executions.Snapshot()
|
||||
_, ratioAvailable := group.Usage.CacheHitRatio()
|
||||
assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio()
|
||||
return summary{
|
||||
ProposalAttempts: proposalTerminal.Attempts,
|
||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||
@@ -192,6 +205,10 @@ func run() (summary, error) {
|
||||
ExhaustedOutcome: exhausted.Outcome,
|
||||
GroupMembers: len(group.Members),
|
||||
GroupAttempts: group.Attempts,
|
||||
GroupCost: group.CostUSD,
|
||||
GroupReasoning: group.Usage.ReasoningTokens,
|
||||
GroupCacheRatioAvailable: ratioAvailable,
|
||||
AssessmentCacheRatio: assessmentRatio,
|
||||
PreparedMessages: len(preparedDetails.Messages),
|
||||
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
|
||||
}, nil
|
||||
|
||||
@@ -3,7 +3,10 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestObserveExample(t *testing.T) {
|
||||
@@ -12,18 +15,22 @@ func TestObserveExample(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
ProposalAttempts *int `json:"proposal_attempts"`
|
||||
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||
Outcome string `json:"assessment_outcome"`
|
||||
CoordinationOutcome string `json:"coordination_outcome"`
|
||||
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts int `json:"group_attempts"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
ProposalAttempts *int `json:"proposal_attempts"`
|
||||
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||
Outcome string `json:"assessment_outcome"`
|
||||
CoordinationOutcome string `json:"coordination_outcome"`
|
||||
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts int `json:"group_attempts"`
|
||||
GroupCost promptkit.ObservedCost `json:"group_cost"`
|
||||
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
|
||||
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
|
||||
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
}
|
||||
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) {
|
||||
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
|
||||
t.Fatalf("unexpected observation summary: %+v", got)
|
||||
}
|
||||
if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 {
|
||||
t.Fatalf("unexpected usage summary: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/observation"
|
||||
)
|
||||
|
||||
// ExecutionGroup collects facts from explicitly selected execution events.
|
||||
// ExecutionGroup collects facts from selected execution events.
|
||||
// Its zero value is ready for concurrent use. Do not copy it after first use.
|
||||
type ExecutionGroup struct{ group observation.Group }
|
||||
|
||||
@@ -21,7 +21,8 @@ type ExecutionGroupMember struct {
|
||||
// member has a terminal, independently of usage completeness. An empty group
|
||||
// has complete coverage and known complete zero usage; a group with members
|
||||
// but no terminals has unknown, incomplete usage. Overlapping groups are not
|
||||
// generally additive.
|
||||
// generally additive. CostUSD covers included terminals independently of
|
||||
// terminal coverage, using the same known, complete, and source merge rules.
|
||||
type ExecutionGroupSummary struct {
|
||||
Members []ExecutionGroupMember `json:"members"`
|
||||
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||
@@ -29,6 +30,7 @@ type ExecutionGroupSummary struct {
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
RepairCalls uint64 `json:"repair_calls"`
|
||||
Usage ObservedTokenUsage `json:"usage"`
|
||||
CostUSD ObservedCost `json:"cost_usd"`
|
||||
}
|
||||
|
||||
// Observe enrolls an event's run, counting its first terminal event only.
|
||||
@@ -36,8 +38,8 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) {
|
||||
count := func(c ObservedTokenCount) domain.ObservedTokenCount {
|
||||
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
|
||||
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
|
||||
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, CostUSD: toDomainCost(event.CostUSD),
|
||||
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
|
||||
}
|
||||
|
||||
// Snapshot returns an internally consistent, caller-owned summary. Coverage
|
||||
@@ -51,6 +53,6 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
|
||||
for i, member := range value.Members {
|
||||
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
|
||||
}
|
||||
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
|
||||
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, CostUSD: fromDomainCost(value.CostUSD),
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
backends:
|
||||
- id: openrouter
|
||||
provider: openrouter
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
smoke_tests:
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := state.GetBackend("openrouter")
|
||||
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||
t.Fatalf("backend = %+v, %v", backend, err)
|
||||
}
|
||||
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
|
||||
@@ -40,6 +40,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
||||
}
|
||||
want := defaults.ExecutionTargetDefault()
|
||||
want.Provider = "openrouter"
|
||||
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
|
||||
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
||||
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
|
||||
|
||||
@@ -59,6 +59,35 @@ func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderPolicyCaptureAndInspection(t *testing.T) {
|
||||
state, err := load(mapping("provider policy", map[string]any{
|
||||
"collections": map[string]any{"team": map[string]any{
|
||||
"backends": []any{map[string]any{"id": "renamed", "endpoint": "https://example.invalid/v1", "provider": "OPENROUTER", "zero_cost": true}, map[string]any{"id": "child", "extends": map[string]any{"id": "renamed"}, "zero_cost": false}},
|
||||
"profiles": []any{map[string]any{"id": "selected", "backend": "team/child", "endpoint": "https://other.invalid/v1", "model": "m"}},
|
||||
}},
|
||||
"profiles": []any{map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1", "model": "m"}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := state.GetBackend("team/child")
|
||||
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost {
|
||||
t.Fatalf("backend: %+v, %v", backend, err)
|
||||
}
|
||||
profile, err := state.GetProfile(context.Background(), "team/selected")
|
||||
if err != nil || profile.ResolvedTarget.Provider != "openrouter" || profile.ResolvedTarget.ZeroCost || profile.ResolvedTarget.Endpoint != "https://other.invalid/v1" {
|
||||
t.Fatalf("profile: %+v, %v", profile, err)
|
||||
}
|
||||
profile, err = state.GetProfile(context.Background(), "generic")
|
||||
if err != nil || profile.ResolvedTarget.Provider != "openai-compatible" || profile.ResolvedTarget.ZeroCost {
|
||||
t.Fatalf("generic: %+v, %v", profile, err)
|
||||
}
|
||||
resource, err := state.InspectBackend(context.Background(), "team/child")
|
||||
if err != nil || resource.Provider == nil || *resource.Provider != "openrouter" || resource.ZeroCost == nil || *resource.ZeroCost {
|
||||
t.Fatalf("inspection: %+v, %v", resource, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
|
||||
definition := map[string]any{
|
||||
"id": "example", "inputs": []any{map[string]any{"name": " document "}},
|
||||
|
||||
@@ -291,6 +291,9 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
|
||||
if backend.Endpoint == nil || *backend.Endpoint == "" {
|
||||
validation.Error(base.Key("endpoint"), "endpoint is required")
|
||||
}
|
||||
if backend.Provider != nil && !llm.SupportedProvider(*backend.Provider) {
|
||||
validation.Error(base.Key("provider"), "unsupported provider")
|
||||
}
|
||||
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
|
||||
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
|
||||
}
|
||||
@@ -319,10 +322,8 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
|
||||
}
|
||||
if authentication := smokeTests.Authentication; authentication != nil {
|
||||
kindPath := smokePath.Key("authentication").Key("kind")
|
||||
if authentication.Kind == nil {
|
||||
validation.Error(kindPath, "authentication kind is required")
|
||||
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
|
||||
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
|
||||
if authentication.Kind != nil && *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" && *authentication.Kind != "vllm_models" {
|
||||
validation.Error(kindPath, "unsupported authentication kind")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -532,6 +532,54 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendProviderFinalConfiguration(t *testing.T) {
|
||||
underlays := []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": "bad", "zero_cost": true}},
|
||||
})}}}
|
||||
main := mapping("override", map[string]any{"backends": []any{
|
||||
map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": " OPENROUTER ", "zero_cost": false, "smoke_tests": map[string]any{"authentication": map[string]any{}}},
|
||||
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}},
|
||||
map[string]any{"id": "cleared", "extends": map[string]any{"id": "base"}, "provider": nil, "zero_cost": nil},
|
||||
map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1"},
|
||||
}})
|
||||
state, err := load(main, underlays)
|
||||
if err != nil {
|
||||
var detail *configkit.LoadError
|
||||
if errors.As(err, &detail) {
|
||||
t.Fatalf("%v: %+v", err, detail.Diagnostics())
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"base", "child"} {
|
||||
b := state.backendRecords[name]
|
||||
if b.Provider == nil || *b.Provider != "openrouter" || b.ZeroCost == nil || *b.ZeroCost {
|
||||
t.Fatalf("%s: %+v", name, b)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"cleared", "generic"} {
|
||||
b := state.backendRecords[name]
|
||||
if b.Provider != nil || b.ZeroCost != nil {
|
||||
t.Fatalf("%s: %+v", name, b)
|
||||
}
|
||||
}
|
||||
field, found := state.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("provider"))
|
||||
if !found || field.Origin.Operation != configkit.ChangeNormalization {
|
||||
t.Fatalf("provider normalization: %+v", field)
|
||||
}
|
||||
for _, provider := range []string{"openai-compatible", "openrouter", "vllm", " ", ""} {
|
||||
_, err := load(mapping("provider", map[string]any{"backends": []any{map[string]any{"id": "x", "endpoint": "https://example.invalid/v1", "provider": provider, "smoke_tests": map[string]any{"authentication": map[string]any{"kind": " VLLM_MODELS "}}}}}), nil)
|
||||
if err != nil {
|
||||
t.Errorf("provider %q: %v", provider, err)
|
||||
}
|
||||
}
|
||||
for _, fields := range []map[string]any{{"provider": "unsupported"}, {"zero_cost": "false"}} {
|
||||
fields["id"], fields["endpoint"] = "x", "https://example.invalid/v1"
|
||||
if _, err := load(mapping("invalid", map[string]any{"backends": []any{fields}}), nil); err == nil {
|
||||
t.Fatalf("accepted invalid fields: %v", fields)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
@@ -638,11 +686,6 @@ func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
|
||||
data: map[string]any{"default_profile": " \t "},
|
||||
path: "/promptkit/backends/0/smoke_tests/default_profile",
|
||||
},
|
||||
{
|
||||
name: "authentication kind required",
|
||||
data: map[string]any{"authentication": map[string]any{}},
|
||||
path: "/promptkit/backends/0/smoke_tests/authentication/kind",
|
||||
},
|
||||
{
|
||||
name: "unknown authentication kind",
|
||||
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},
|
||||
|
||||
@@ -21,6 +21,8 @@ func copyProviderMap(value map[string]any) map[string]any {
|
||||
}
|
||||
|
||||
func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings {
|
||||
in.Provider = copyPointer(in.Provider)
|
||||
in.ZeroCost = copyPointer(in.ZeroCost)
|
||||
in.Endpoint = copyPointer(in.Endpoint)
|
||||
in.APIKeyEnv = copyPointer(in.APIKeyEnv)
|
||||
in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit)
|
||||
|
||||
@@ -15,6 +15,9 @@ func resolveBackends(entries []backendConfig, base configkit.Path, resolution *c
|
||||
for i := range entries {
|
||||
entry := &entries[i]
|
||||
path := base.Index(i)
|
||||
if entry.Provider != nil {
|
||||
recordStringNormalization(&entry.Provider, path.Key("provider"), resolution, strings.ToLower(strings.TrimSpace(*entry.Provider)))
|
||||
}
|
||||
if entry.Endpoint != nil {
|
||||
if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil {
|
||||
recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized)
|
||||
|
||||
@@ -26,6 +26,8 @@ type ProfileResource struct {
|
||||
// Its fields borrow immutable configuration captured by the adapter.
|
||||
type BackendResource struct {
|
||||
ID, Endpoint string
|
||||
Provider *string
|
||||
ZeroCost *bool
|
||||
APIKeyEnv *string
|
||||
ConcurrencyLimit, QueueCapacity *int
|
||||
ExtraParams map[string]any
|
||||
@@ -131,7 +133,7 @@ func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendRes
|
||||
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
|
||||
}
|
||||
return &BackendResource{
|
||||
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv,
|
||||
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, Provider: config.Provider, ZeroCost: config.ZeroCost,
|
||||
ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity,
|
||||
ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests,
|
||||
}, nil
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||
"go.maximumdirect.net/promptkit/internal/llm"
|
||||
profilepkg "go.maximumdirect.net/promptkit/internal/profile"
|
||||
"go.maximumdirect.net/promptkit/internal/settings"
|
||||
)
|
||||
@@ -36,7 +37,7 @@ func (state *loaded) buildExecutionProfiles(config *settings.Settings) error {
|
||||
}
|
||||
|
||||
func captureBackend(id string, config backendConfig) domain.Backend {
|
||||
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
||||
definition := domain.Backend{ID: id, Provider: llm.EffectiveProvider(valueOrZero(config.Provider)), ZeroCost: valueOrZero(config.ZeroCost), ExtraParams: config.ExtraParams}
|
||||
if config.SmokeTests != nil {
|
||||
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
||||
if config.SmokeTests.Reachability != nil {
|
||||
@@ -67,6 +68,7 @@ func captureBackend(id string, config backendConfig) domain.Backend {
|
||||
|
||||
func captureExecutionProfile(id string, config profileConfig, backends map[string]domain.Backend) (domain.ExecutionProfile, error) {
|
||||
target := defaults.ExecutionTargetDefault()
|
||||
target.Provider = llm.EffectiveProvider("")
|
||||
if config.Backend != nil && *config.Backend != "" {
|
||||
// These defaults are capture-owned and immutable. Public and runtime
|
||||
// lookups still detach mutable values; profile overlays copy before merging.
|
||||
@@ -75,6 +77,8 @@ func captureExecutionProfile(id string, config profileConfig, backends map[strin
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w: %q", id, backend.ErrBackendNotFound, *config.Backend)
|
||||
}
|
||||
target.BackendID = selected.ID
|
||||
target.Provider = selected.Provider
|
||||
target.ZeroCost = selected.ZeroCost
|
||||
target.Endpoint = selected.Endpoint
|
||||
target.APIKeyEnv = selected.APIKeyEnv
|
||||
target.ExtraParams = selected.ExtraParams
|
||||
|
||||
17
internal/domain/cost.go
Normal file
17
internal/domain/cost.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package domain
|
||||
|
||||
type CostSource string
|
||||
|
||||
const (
|
||||
CostSourceNone CostSource = ""
|
||||
CostSourceProvider CostSource = "provider"
|
||||
CostSourceConfiguredZero CostSource = "configured_zero"
|
||||
CostSourceMixed CostSource = "mixed"
|
||||
)
|
||||
|
||||
type ObservedCost struct {
|
||||
Value float64
|
||||
Known bool
|
||||
Complete bool
|
||||
Source CostSource
|
||||
}
|
||||
@@ -99,6 +99,7 @@ type RunResult struct {
|
||||
EffectiveModelParams ExecutionTarget
|
||||
InputHashes map[string]string
|
||||
Usage TokenUsage
|
||||
CostUSD ObservedCost
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Duration time.Duration
|
||||
@@ -188,6 +189,8 @@ type PromptMessageTemplate struct {
|
||||
// diagnostic metadata.
|
||||
type Backend struct {
|
||||
ID string
|
||||
Provider string
|
||||
ZeroCost bool
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
@@ -236,6 +239,8 @@ type ExecutionTargetPresence struct {
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings for a run.
|
||||
type ExecutionTarget struct {
|
||||
Provider string `yaml:"-" json:"-"`
|
||||
ZeroCost bool `yaml:"-" json:"-"`
|
||||
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
@@ -324,12 +329,14 @@ type GenerateResponse struct {
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
UsagePresence *TokenUsagePresence
|
||||
CostUSD *float64
|
||||
}
|
||||
|
||||
// TokenUsagePresence records which token fields a provider reported.
|
||||
type TokenUsagePresence struct {
|
||||
PromptTokens bool
|
||||
CompletionTokens bool
|
||||
ReasoningTokens bool
|
||||
TotalTokens bool
|
||||
CachedTokens bool
|
||||
CacheWriteTokens bool
|
||||
@@ -339,6 +346,7 @@ type TokenUsagePresence struct {
|
||||
type TokenUsage struct {
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
ReasoningTokens int
|
||||
TotalTokens int
|
||||
CachedTokens int
|
||||
CacheWriteTokens int
|
||||
|
||||
@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
ReasoningTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
@@ -75,6 +76,7 @@ type ExecutionGroupSummary struct {
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only; it carries no mutable containers or
|
||||
@@ -100,6 +102,7 @@ type ExecutionEvent struct {
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
|
||||
@@ -130,7 +130,7 @@ func (c *Coordinator) generate(
|
||||
if result.Dispatched {
|
||||
attempts++
|
||||
}
|
||||
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
|
||||
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal)
|
||||
if result.Outcome.Err == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if captureEvidence {
|
||||
@@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
||||
generation = 1
|
||||
}
|
||||
result := c.executeInjected(ctx, req)
|
||||
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
|
||||
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1)
|
||||
if captureEvidence {
|
||||
evidence.Usage = usage.Observed()
|
||||
}
|
||||
@@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe
|
||||
requested := req.StructuredOutput != nil
|
||||
result.Outcome.StructuredOutputRequested = &requested
|
||||
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
|
||||
if result.Outcome.Response != nil {
|
||||
result.Outcome.CostUSD = result.Outcome.Response.CostUSD
|
||||
}
|
||||
result.RequestDuration = elapsed(start, c.now())
|
||||
if result.Outcome.Err == nil && result.Outcome.Response == nil {
|
||||
result.Outcome.Err = llm.ErrMalformedResponse
|
||||
@@ -240,8 +243,15 @@ type attemptResult struct {
|
||||
|
||||
// recordAttempt runs after the attempt has released its permit, including when
|
||||
// the recorder calls application code. Injected clients expose no HTTP facts.
|
||||
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
|
||||
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) {
|
||||
outcome := result.Outcome
|
||||
var resolved domain.ObservedCost
|
||||
if result.Dispatched {
|
||||
resolved = observation.ResolveCost(outcome.CostUSD, zeroCost)
|
||||
if cost != nil {
|
||||
cost.Add(resolved)
|
||||
}
|
||||
}
|
||||
failure := ClassifyFailure(outcome.Err)
|
||||
if capture {
|
||||
attempt := AttemptEvidence{
|
||||
@@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
|
||||
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
|
||||
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,89 @@ func TestObservedRetryAndExhaustion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatchedAttemptCostsAreCollectedWithoutObserver(t *testing.T) {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
var cost observation.CostAccumulator
|
||||
ctx := observation.WithCostAccumulator(context.Background(), &cost)
|
||||
req := testRequest(1)
|
||||
req.Target.Provider = "openrouter"
|
||||
if _, err := coordinator.Generate(ctx, req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := cost.Observed()
|
||||
if got.Value != .02 || !got.Known || got.Complete || got.Source != domain.CostSourceProvider {
|
||||
t.Fatalf("cost=%+v", got)
|
||||
}
|
||||
|
||||
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { return nil, errors.New("transport failed") })
|
||||
var zero observation.CostAccumulator
|
||||
req.Target.ZeroCost = true
|
||||
if _, err := configured.Generate(observation.WithCostAccumulator(context.Background(), &zero), req); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if got := zero.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
|
||||
t.Fatalf("fallback=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) {
|
||||
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
|
||||
charge := .25
|
||||
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{CostUSD: &charge}, err: errors.New("failed")})
|
||||
var cost observation.CostAccumulator
|
||||
var events []domain.ExecutionEvent
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder.SetCostAccumulator(&cost)
|
||||
ctx := observation.WithRecorder(observation.WithCostAccumulator(context.Background(), &cost), recorder)
|
||||
if _, err := coordinator.Generate(ctx, testRequest(0)); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
recorder.Finish(observation.FinishData{})
|
||||
want := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
|
||||
if len(events) != 2 || events[0].CostUSD != want || events[1].CostUSD != want || cost.Observed() != want {
|
||||
t.Fatalf("events=%+v cost=%+v", events, cost.Observed())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledDispatchCostFallbackAndUndispatchedEmptyCost(t *testing.T) {
|
||||
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
coordinator := NewInjected(configured.manager, cancelingCostClient{cancel: cancel})
|
||||
var cost observation.CostAccumulator
|
||||
req := testRequest(0)
|
||||
req.Target.ZeroCost = true
|
||||
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &cost), req); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("dispatch error=%v", err)
|
||||
}
|
||||
if got := cost.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
|
||||
t.Fatalf("dispatched=%+v", got)
|
||||
}
|
||||
var untouched observation.CostAccumulator
|
||||
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &untouched), req); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("pre-dispatch error=%v", err)
|
||||
}
|
||||
if got := untouched.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("undispatched=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type cancelingCostClient struct{ cancel context.CancelFunc }
|
||||
|
||||
func (c cancelingCostClient) Generate(ctx context.Context, _ domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||
c.cancel()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
@@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
// PreparedGeneration holds one encoded request and resolved authentication for
|
||||
// repeated attempts. Each execution creates a fresh HTTP request and context.
|
||||
type PreparedGeneration struct {
|
||||
provider string
|
||||
endpoint string
|
||||
payload []byte
|
||||
apiKey string
|
||||
@@ -104,6 +105,7 @@ type AttemptOutcome struct {
|
||||
StatusCode int
|
||||
Usage domain.TokenUsage
|
||||
UsagePresence *domain.TokenUsagePresence
|
||||
CostUSD *float64
|
||||
RetryNotBefore time.Time
|
||||
Evidence HTTPExecutionEvidence
|
||||
StructuredOutputRequested *bool
|
||||
@@ -156,7 +158,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
||||
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
|
||||
}
|
||||
return &PreparedGeneration{
|
||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||
endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider),
|
||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||
structuredOutputRequested: wirePayload["response_format"] != nil,
|
||||
}, nil
|
||||
@@ -266,16 +268,40 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
var writes, reasoning *int
|
||||
switch p.provider {
|
||||
case "openrouter", "vllm":
|
||||
var details map[string]json.RawMessage
|
||||
if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil {
|
||||
reasoning = optionalInteger(details["reasoning_tokens"])
|
||||
}
|
||||
if p.provider == "openrouter" {
|
||||
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens)
|
||||
} else {
|
||||
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens)
|
||||
}
|
||||
default:
|
||||
if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil {
|
||||
outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
|
||||
outcome.Err = contextErr
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
outcome.ResponseDecoded = true
|
||||
outcome.Usage = domain.TokenUsage{
|
||||
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
|
||||
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
|
||||
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
|
||||
CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning),
|
||||
}
|
||||
outcome.UsagePresence = &domain.TokenUsagePresence{
|
||||
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
|
||||
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
|
||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
|
||||
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
|
||||
}
|
||||
if p.provider == "openrouter" {
|
||||
outcome.CostUSD = optionalFloat(wireResp.Usage.Cost)
|
||||
}
|
||||
|
||||
if len(wireResp.Choices) == 0 {
|
||||
@@ -297,7 +323,7 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
|
||||
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: outcome.CostUSD}
|
||||
outcome.GenerationCompleted = true
|
||||
if captureEvidence {
|
||||
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
|
||||
@@ -540,12 +566,38 @@ type openAIChatResponse struct {
|
||||
CompletionTokens *int `json:"completion_tokens"`
|
||||
TotalTokens *int `json:"total_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
||||
CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
CacheWriteTokens *int `json:"cache_write_tokens"`
|
||||
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
|
||||
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
||||
Cost json.RawMessage `json:"cost"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
func optionalInteger(raw json.RawMessage) *int {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
var value *int
|
||||
if json.Unmarshal(raw, &value) != nil {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func optionalFloat(raw json.RawMessage) *float64 {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
var value *float64
|
||||
if json.Unmarshal(raw, &value) != nil || (value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0))) {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type openAIResponseFormat struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
||||
|
||||
@@ -882,6 +882,85 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSpecificUsageDetails(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, provider, usage string
|
||||
writes, reasoning int
|
||||
writesKnown, reasoningKnown bool
|
||||
}{
|
||||
{name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true},
|
||||
{name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true},
|
||||
{name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true},
|
||||
{name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`},
|
||||
{name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`},
|
||||
{name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`},
|
||||
{name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = tc.provider
|
||||
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
|
||||
if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown {
|
||||
t.Fatalf("response=%+v error=%v", response, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = "openrouter"
|
||||
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens {
|
||||
t.Fatalf("outcome=%+v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderReportedCostIsOptional(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, provider, cost string
|
||||
known bool
|
||||
value float64
|
||||
}{
|
||||
{"charge", "openrouter", `0.002`, true, .002},
|
||||
{"zero", "openrouter", `0`, true, 0},
|
||||
{"null", "openrouter", `null`, false, 0},
|
||||
{"string", "openrouter", `"0.2"`, false, 0},
|
||||
{"object", "openrouter", `{}`, false, 0},
|
||||
{"overflow", "openrouter", `1e9999`, false, 0},
|
||||
{"generic ignores cost", "openai-compatible", `0.02`, false, 0},
|
||||
{"vllm ignores cost", "vllm", `0.02`, false, 0},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{"cost":`+tc.cost+`},"choices":[{"message":{"content":"ok"}}]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = tc.provider
|
||||
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
|
||||
if err != nil || response == nil || (response.CostUSD != nil) != tc.known || (response.CostUSD != nil && *response.CostUSD != tc.value) {
|
||||
t.Fatalf("response=%+v err=%v", response, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{"cost":0.02},"choices":[]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = "openrouter"
|
||||
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.CostUSD == nil || *outcome.CostUSD != .02 {
|
||||
t.Fatalf("outcome=%+v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -26,8 +27,11 @@ func OpenRouterKeyEndpoint(base string) (string, error) {
|
||||
return url.JoinPath(base, "key")
|
||||
}
|
||||
|
||||
// ProbeOpenRouterKey executes and interprets an OpenRouter key request.
|
||||
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
|
||||
// ProbeAuthentication executes the selected authentication adapter.
|
||||
func (c *OpenAICompatibleClient) ProbeAuthentication(ctx context.Context, adapter, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
|
||||
if !SupportedAuthenticationAdapter(adapter) {
|
||||
return AuthenticationEvidence{}, fmt.Errorf("unsupported authentication adapter %q", adapter)
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return AuthenticationEvidence{}, err
|
||||
@@ -69,7 +73,14 @@ func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoin
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if recognizedOpenRouterKeyResponse(contents) {
|
||||
recognized := false
|
||||
switch adapter {
|
||||
case "openrouter_key":
|
||||
recognized = recognizedOpenRouterKeyResponse(contents)
|
||||
case "vllm_models":
|
||||
recognized = recognizedVLLMModelsResponse(contents)
|
||||
}
|
||||
if recognized {
|
||||
accepted := true
|
||||
result.Accepted = &accepted
|
||||
} else {
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestProbeOpenRouterKey(t *testing.T) {
|
||||
authorization = req.Header.Get("Authorization")
|
||||
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
|
||||
})}})
|
||||
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -67,7 +67,7 @@ func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
|
||||
})}})
|
||||
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
|
||||
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||
}
|
||||
@@ -105,7 +105,7 @@ func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
|
||||
})}})
|
||||
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
|
||||
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
|
||||
}
|
||||
@@ -127,7 +127,7 @@ func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
|
||||
})}})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
got, err := client.ProbeAuthentication(ctx, "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
|
||||
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
|
||||
t.Fatalf("evidence = %+v, error = %v", got, err)
|
||||
}
|
||||
|
||||
59
internal/llm/provider.go
Normal file
59
internal/llm/provider.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// EffectiveProvider resolves an omitted provider to the generic conventions.
|
||||
func EffectiveProvider(provider string) string {
|
||||
if provider == "" {
|
||||
return "openai-compatible"
|
||||
}
|
||||
return provider
|
||||
}
|
||||
|
||||
// SupportedProvider reports whether a normalized provider selection is supported.
|
||||
// An empty selection uses the generic OpenAI-compatible conventions.
|
||||
func SupportedProvider(provider string) bool {
|
||||
switch provider {
|
||||
case "", "openai-compatible", "openrouter", "vllm":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// AuthenticationAdapter selects declared diagnostic metadata or provider defaults.
|
||||
func AuthenticationAdapter(provider string, smoke *domain.BackendSmokeTestConfig) string {
|
||||
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Kind != nil {
|
||||
return *smoke.Authentication.Kind
|
||||
}
|
||||
switch provider {
|
||||
case "openrouter":
|
||||
return "openrouter_key"
|
||||
case "vllm":
|
||||
return "vllm_models"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// AuthenticationEndpoint resolves a diagnostic destination without changing configuration.
|
||||
func AuthenticationEndpoint(adapter, base string, smoke *domain.BackendSmokeTestConfig) (string, error) {
|
||||
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Endpoint != nil {
|
||||
return *smoke.Authentication.Endpoint, nil
|
||||
}
|
||||
if adapter == "openrouter_key" {
|
||||
return OpenRouterKeyEndpoint(base)
|
||||
}
|
||||
if adapter == "vllm_models" {
|
||||
return url.JoinPath(base, "models")
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// SupportedAuthenticationAdapter identifies implemented authentication probes.
|
||||
func SupportedAuthenticationAdapter(adapter string) bool {
|
||||
return adapter == "openrouter_key" || adapter == "vllm_models"
|
||||
}
|
||||
34
internal/llm/provider_test.go
Normal file
34
internal/llm/provider_test.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestAuthenticationSelection(t *testing.T) {
|
||||
none, explicit, models, endpoint := "none", "openrouter_key", "vllm_models", "https://diagnostics.example/probe"
|
||||
for _, tc := range []struct {
|
||||
name, provider, adapter, destination string
|
||||
metadata *domain.BackendSmokeTestConfig
|
||||
}{
|
||||
{name: "provider default", provider: "openrouter", adapter: "openrouter_key", destination: "https://example.test/v1/key"},
|
||||
{name: "cleared block", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
|
||||
{name: "endpoint only", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "openrouter_key", destination: endpoint},
|
||||
{name: "explicit adapter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &explicit}}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
|
||||
{name: "disabled", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
|
||||
{name: "generic unsupported", destination: "https://example.test/v1"},
|
||||
{name: "vllm default", provider: "vllm", adapter: "vllm_models", destination: "https://example.test/v1/models"},
|
||||
{name: "vllm override", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "vllm_models", destination: endpoint},
|
||||
{name: "explicit vllm on generic", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &models}}, adapter: "vllm_models", destination: "https://example.test/v1/models"},
|
||||
{name: "disabled vllm", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
kind := AuthenticationAdapter(tc.provider, tc.metadata)
|
||||
destination, err := AuthenticationEndpoint(kind, "https://example.test/v1", tc.metadata)
|
||||
if err != nil || kind != tc.adapter || destination != tc.destination {
|
||||
t.Fatalf("selection=%q destination=%q error=%v", kind, destination, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
16
internal/llm/vllm_authentication.go
Normal file
16
internal/llm/vllm_authentication.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package llm
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
func recognizedVLLMModelsResponse(contents []byte) bool {
|
||||
var envelope map[string]json.RawMessage
|
||||
if json.Unmarshal(contents, &envelope) != nil || envelope == nil {
|
||||
return false
|
||||
}
|
||||
var object string
|
||||
if json.Unmarshal(envelope["object"], &object) != nil || object != "list" {
|
||||
return false
|
||||
}
|
||||
var data []json.RawMessage
|
||||
return json.Unmarshal(envelope["data"], &data) == nil && data != nil
|
||||
}
|
||||
51
internal/llm/vllm_authentication_test.go
Normal file
51
internal/llm/vllm_authentication_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestVLLMModelsAuthentication(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, body, key string
|
||||
status int
|
||||
accepted *bool
|
||||
unrecognized, overflow bool
|
||||
}{
|
||||
{name: "empty list", status: 200, body: `{"object":"list","data":[]}`, accepted: boolPtr(true)},
|
||||
{name: "nonempty list", status: 200, body: `{"object":"list","data":[{"id":"model"}]}`, key: "key", accepted: boolPtr(true)},
|
||||
{name: "wrong object", status: 200, body: `{"object":"model","data":[]}`, unrecognized: true},
|
||||
{name: "null data", status: 200, body: `{"object":"list","data":null}`, unrecognized: true},
|
||||
{name: "object data", status: 200, body: `{"object":"list","data":{}}`, unrecognized: true},
|
||||
{name: "invalid framing", status: 200, body: `{"object":"list","data":[]} {}`, unrecognized: true},
|
||||
{name: "overflow", status: 200, body: strings.Repeat(" ", maxAuthenticationResponseBytes+1), overflow: true},
|
||||
{name: "unauthorized", status: 401, accepted: boolPtr(false)},
|
||||
{name: "other status", status: 503},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var credential string
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||
credential = req.Header.Get("Authorization")
|
||||
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
|
||||
})}})
|
||||
got, err := client.ProbeAuthentication(context.Background(), "vllm_models", "https://example.invalid/v1/models", 0, domain.ExecutionTarget{APIKey: tc.key})
|
||||
if err != nil || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow || (got.Accepted == nil) != (tc.accepted == nil) || (got.Accepted != nil && *got.Accepted != *tc.accepted) {
|
||||
t.Fatalf("result=%+v err=%v", got, err)
|
||||
}
|
||||
want := ""
|
||||
if tc.key != "" {
|
||||
want = "Bearer " + tc.key
|
||||
}
|
||||
if credential != want {
|
||||
t.Fatalf("authorization=%q", credential)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func boolPtr(value bool) *bool { return &value }
|
||||
67
internal/observation/cost.go
Normal file
67
internal/observation/cost.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
type costContextKey struct{}
|
||||
|
||||
// WithCostAccumulator carries one invocation's cost state.
|
||||
func WithCostAccumulator(ctx context.Context, accumulator *CostAccumulator) context.Context {
|
||||
return context.WithValue(ctx, costContextKey{}, accumulator)
|
||||
}
|
||||
|
||||
// CostFromContext returns the current invocation's cost state, if installed.
|
||||
func CostFromContext(ctx context.Context) *CostAccumulator {
|
||||
value, _ := ctx.Value(costContextKey{}).(*CostAccumulator)
|
||||
return value
|
||||
}
|
||||
|
||||
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
|
||||
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
|
||||
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {
|
||||
return domain.ObservedCost{Value: *reported, Known: true, Complete: true, Source: domain.CostSourceProvider}
|
||||
}
|
||||
if zeroCost {
|
||||
return domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}
|
||||
}
|
||||
return domain.ObservedCost{}
|
||||
}
|
||||
|
||||
// CostAccumulator merges dispatched attempts or already aggregated costs.
|
||||
type CostAccumulator struct {
|
||||
value domain.ObservedCost
|
||||
started bool
|
||||
}
|
||||
|
||||
// Add incorporates one contribution.
|
||||
func (c *CostAccumulator) Add(next domain.ObservedCost) {
|
||||
if !c.started {
|
||||
c.started = true
|
||||
c.value = next
|
||||
return
|
||||
}
|
||||
if next.Known {
|
||||
c.value.Value += next.Value
|
||||
if next.Source != domain.CostSourceNone {
|
||||
if c.value.Source == domain.CostSourceNone {
|
||||
c.value.Source = next.Source
|
||||
} else if c.value.Source != next.Source {
|
||||
c.value.Source = domain.CostSourceMixed
|
||||
}
|
||||
}
|
||||
}
|
||||
c.value.Known = c.value.Known || next.Known
|
||||
c.value.Complete = c.value.Complete && next.Complete
|
||||
}
|
||||
|
||||
// Observed returns known complete zero when no contribution has been added.
|
||||
func (c CostAccumulator) Observed() domain.ObservedCost {
|
||||
if !c.started {
|
||||
return domain.ObservedCost{Known: true, Complete: true}
|
||||
}
|
||||
return c.value
|
||||
}
|
||||
63
internal/observation/cost_test.go
Normal file
63
internal/observation/cost_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestResolveCostPrecedence(t *testing.T) {
|
||||
zero, negative, invalid := 0.0, -1.5, math.Inf(1)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
reported *float64
|
||||
fallback bool
|
||||
want domain.ObservedCost
|
||||
}{
|
||||
{"unknown", nil, false, domain.ObservedCost{}},
|
||||
{"configured zero", nil, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
|
||||
{"reported zero", &zero, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceProvider}},
|
||||
{"negative", &negative, false, domain.ObservedCost{Value: -1.5, Known: true, Complete: true, Source: domain.CostSourceProvider}},
|
||||
{"invalid fallback", &invalid, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
|
||||
{"invalid unknown", &invalid, false, domain.ObservedCost{}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := ResolveCost(tc.reported, tc.fallback); got != tc.want {
|
||||
t.Fatalf("got %+v want %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostAccumulatorMergesSourcesAndCoverage(t *testing.T) {
|
||||
var acc CostAccumulator
|
||||
if got := acc.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("empty: %+v", got)
|
||||
}
|
||||
acc.Add(domain.ObservedCost{})
|
||||
if got := acc.Observed(); got != (domain.ObservedCost{}) {
|
||||
t.Fatalf("unknown: %+v", got)
|
||||
}
|
||||
a, b := .1, .2
|
||||
acc.Add(ResolveCost(&a, false))
|
||||
acc.Add(ResolveCost(&b, false))
|
||||
acc.Add(ResolveCost(nil, true))
|
||||
got := acc.Observed()
|
||||
if math.Abs(got.Value-.3) > 1e-12 || !got.Known || got.Complete || got.Source != domain.CostSourceMixed {
|
||||
t.Fatalf("merged: %+v", got)
|
||||
}
|
||||
var zeros CostAccumulator
|
||||
zero := 0.0
|
||||
zeros.Add(ResolveCost(&zero, false))
|
||||
zeros.Add(ResolveCost(nil, true))
|
||||
if got := zeros.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceMixed}) {
|
||||
t.Fatalf("zeros: %+v", got)
|
||||
}
|
||||
var aggregate CostAccumulator
|
||||
aggregate.Add(domain.ObservedCost{Known: true, Complete: true})
|
||||
aggregate.Add(got)
|
||||
if aggregate.Observed() != got {
|
||||
t.Fatalf("aggregate: %+v", aggregate.Observed())
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ type Group struct {
|
||||
terminals int
|
||||
attempts, retries, repairs uint64
|
||||
usage domain.ObservedTokenUsage
|
||||
cost CostAccumulator
|
||||
}
|
||||
|
||||
// Observe enrolls a run and incorporates its first terminal event only.
|
||||
@@ -36,9 +37,10 @@ func (g *Group) Observe(event domain.ExecutionEvent) {
|
||||
g.repairs += event.RepairCalls
|
||||
if g.terminals == 0 {
|
||||
complete := domain.ObservedTokenCount{Complete: true}
|
||||
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||
}
|
||||
mergeGroupUsage(&g.usage, event.Usage)
|
||||
g.cost.Add(event.CostUSD)
|
||||
g.terminals++
|
||||
}
|
||||
|
||||
@@ -47,12 +49,15 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage}
|
||||
if g.terminals > 0 || len(g.members) == 0 {
|
||||
result.CostUSD = g.cost.Observed()
|
||||
}
|
||||
for id, terminal := range g.members {
|
||||
result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal})
|
||||
}
|
||||
if len(g.members) == 0 {
|
||||
zero := domain.ObservedTokenCount{Known: true, Complete: true}
|
||||
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
|
||||
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, ReasoningTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -67,6 +72,7 @@ func mergeGroupUsage(total *domain.ObservedTokenUsage, next domain.ObservedToken
|
||||
}
|
||||
merge(&total.PromptTokens, next.PromptTokens)
|
||||
merge(&total.CompletionTokens, next.CompletionTokens)
|
||||
merge(&total.ReasoningTokens, next.ReasoningTokens)
|
||||
merge(&total.TotalTokens, next.TotalTokens)
|
||||
merge(&total.CachedTokens, next.CachedTokens)
|
||||
merge(&total.CacheWriteTokens, next.CacheWriteTokens)
|
||||
|
||||
@@ -41,6 +41,25 @@ func TestGroupTerminalAccounting(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupCostCoverageIsIndependentOfTerminalCoverage(t *testing.T) {
|
||||
var group Group
|
||||
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("empty=%+v", got)
|
||||
}
|
||||
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionStarted, RunID: "pending"})
|
||||
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{}) {
|
||||
t.Fatalf("pending=%+v", got)
|
||||
}
|
||||
charge := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
|
||||
terminal := domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "done", CostUSD: charge}
|
||||
group.Observe(terminal)
|
||||
group.Observe(terminal)
|
||||
got := group.Snapshot()
|
||||
if got.TerminalCoverageComplete || got.CostUSD != charge {
|
||||
t.Fatalf("snapshot=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupConcurrentSnapshots(t *testing.T) {
|
||||
var group Group
|
||||
var wg sync.WaitGroup
|
||||
|
||||
@@ -30,6 +30,7 @@ type Recorder struct {
|
||||
generationDuration time.Duration
|
||||
retryDelay time.Duration
|
||||
usage UsageAccumulator
|
||||
cost *CostAccumulator
|
||||
}
|
||||
|
||||
// Selection contains resolved identities only. Unresolved fields stay empty.
|
||||
@@ -53,6 +54,7 @@ type AttemptData struct {
|
||||
StatusCode int
|
||||
Usage domain.TokenUsage
|
||||
UsagePresence *domain.TokenUsagePresence
|
||||
CostUSD domain.ObservedCost
|
||||
}
|
||||
|
||||
// DelayData describes one completed or interrupted retry wait.
|
||||
@@ -99,6 +101,9 @@ func (r *Recorder) RunID() string { return r.runID }
|
||||
// SetParentRunID records the immediate source identity before start delivery.
|
||||
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
|
||||
|
||||
// SetCostAccumulator selects the invocation-owned cost state for terminal reporting.
|
||||
func (r *Recorder) SetCostAccumulator(cost *CostAccumulator) { r.cost = cost }
|
||||
|
||||
// Start delivers the opening event with only identities already resolved.
|
||||
func (r *Recorder) Start() {
|
||||
event := r.event(domain.ExecutionStarted)
|
||||
@@ -138,6 +143,7 @@ func (r *Recorder) Attempt(data AttemptData) {
|
||||
event.StatusCode = data.StatusCode
|
||||
if data.Dispatched {
|
||||
event.Usage = observedUsage(data.Usage, data.UsagePresence)
|
||||
event.CostUSD = data.CostUSD
|
||||
}
|
||||
r.deliver(event)
|
||||
}
|
||||
@@ -176,6 +182,9 @@ func (r *Recorder) Finish(data FinishData) {
|
||||
event.Duration = 0
|
||||
}
|
||||
event.Usage = r.usage.Observed()
|
||||
if r.cost != nil {
|
||||
event.CostUSD = r.cost.Observed()
|
||||
}
|
||||
r.deliver(event)
|
||||
}
|
||||
|
||||
@@ -195,6 +204,22 @@ func (r *Recorder) deliver(event domain.ExecutionEvent) {
|
||||
|
||||
type contextKey struct{}
|
||||
|
||||
type observerKey struct{}
|
||||
|
||||
// WithObserver adds an observer to the inherited contextual selection.
|
||||
func WithObserver(ctx context.Context, observer domain.ExecutionObserver) context.Context {
|
||||
if observer == nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, observerKey{}, Compose(ObserverFromContext(ctx), observer))
|
||||
}
|
||||
|
||||
// ObserverFromContext returns the contextual observer selection, if any.
|
||||
func ObserverFromContext(ctx context.Context) domain.ExecutionObserver {
|
||||
observer, _ := ctx.Value(observerKey{}).(domain.ExecutionObserver)
|
||||
return observer
|
||||
}
|
||||
|
||||
// WithoutExecutionState starts a public operation without inheriting another
|
||||
// invocation's recorder or prepared-claim sink. Other context values and all
|
||||
// cancellation/deadline behavior remain with the caller's context.
|
||||
@@ -206,7 +231,7 @@ type executionBoundary struct{ context.Context }
|
||||
|
||||
func (c executionBoundary) Value(key any) any {
|
||||
switch key.(type) {
|
||||
case contextKey, claimSinkKey:
|
||||
case contextKey, claimSinkKey, costContextKey:
|
||||
return nil
|
||||
default:
|
||||
return c.Context.Value(key)
|
||||
@@ -244,7 +269,7 @@ type usageField struct {
|
||||
known, complete bool
|
||||
}
|
||||
type usageTotals struct {
|
||||
prompt, completion, total, cached, cacheWrite usageField
|
||||
prompt, completion, reasoning, total, cached, cacheWrite usageField
|
||||
}
|
||||
|
||||
// UsageAccumulator combines presence-aware usage across dispatched attempts.
|
||||
@@ -270,7 +295,7 @@ func (u UsageAccumulator) Observed() domain.ObservedTokenUsage {
|
||||
}
|
||||
|
||||
func completeUsageTotals() usageTotals {
|
||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
|
||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, reasoning: usageField{complete: true},
|
||||
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
|
||||
}
|
||||
|
||||
@@ -278,6 +303,7 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
|
||||
known := usagePresence(usage, presence)
|
||||
u.prompt.add(usage.PromptTokens, known.PromptTokens)
|
||||
u.completion.add(usage.CompletionTokens, known.CompletionTokens)
|
||||
u.reasoning.add(usage.ReasoningTokens, known.ReasoningTokens)
|
||||
u.total.add(usage.TotalTokens, known.TotalTokens)
|
||||
u.cached.add(usage.CachedTokens, known.CachedTokens)
|
||||
u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens)
|
||||
@@ -286,7 +312,8 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
|
||||
func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage {
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts),
|
||||
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
|
||||
ReasoningTokens: u.reasoning.snapshot(attempts),
|
||||
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
|
||||
CacheWriteTokens: u.cacheWrite.snapshot(attempts),
|
||||
}
|
||||
}
|
||||
@@ -312,6 +339,7 @@ func observedUsage(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens),
|
||||
CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens),
|
||||
ReasoningTokens: observedCount(usage.ReasoningTokens, known.ReasoningTokens),
|
||||
TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens),
|
||||
CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens),
|
||||
CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens),
|
||||
@@ -324,7 +352,8 @@ func usagePresence(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
|
||||
}
|
||||
return domain.TokenUsagePresence{
|
||||
PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0,
|
||||
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
|
||||
ReasoningTokens: usage.ReasoningTokens != 0,
|
||||
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
|
||||
CacheWriteTokens: usage.CacheWriteTokens != 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,10 +19,15 @@ func TestExecutionBoundaryPreservesCallerContext(t *testing.T) {
|
||||
recorder := newRecorder("outer", time.Now, nil)
|
||||
sink := recorder
|
||||
parent = WithClaimSink(WithRecorder(parent, recorder), &sink)
|
||||
var cost CostAccumulator
|
||||
parent = WithCostAccumulator(parent, &cost)
|
||||
child := WithoutExecutionState(parent)
|
||||
if FromContext(child) != nil || FromContext(parent) != recorder {
|
||||
t.Fatal("execution boundary did not isolate recorder state")
|
||||
}
|
||||
if CostFromContext(child) != nil || CostFromContext(parent) != &cost {
|
||||
t.Fatal("execution boundary did not isolate cost state")
|
||||
}
|
||||
RegisterClaim(child, nil)
|
||||
if sink != recorder {
|
||||
t.Fatal("execution boundary retained the parent claim sink")
|
||||
|
||||
@@ -25,3 +25,18 @@ func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) {
|
||||
t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReasoningUsagePresence(t *testing.T) {
|
||||
var usage UsageAccumulator
|
||||
usage.Add(domain.TokenUsage{}, &domain.TokenUsagePresence{ReasoningTokens: true})
|
||||
usage.Add(domain.TokenUsage{ReasoningTokens: 4}, nil)
|
||||
got := usage.Observed().ReasoningTokens
|
||||
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: true}) {
|
||||
t.Fatalf("usage=%+v", got)
|
||||
}
|
||||
usage.Add(domain.TokenUsage{}, nil)
|
||||
got = usage.Observed().ReasoningTokens
|
||||
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}) {
|
||||
t.Fatalf("partial usage=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,10 @@ type CollectionSettings struct {
|
||||
type BackendSettings struct {
|
||||
// ID identifies the backend in its resource list.
|
||||
ID string `yaml:"id"`
|
||||
// Provider selects response and diagnostic conventions when supplied.
|
||||
Provider *string `yaml:"provider"`
|
||||
// ZeroCost enables a configured-zero fallback when no cost is reported.
|
||||
ZeroCost *bool `yaml:"zero_cost"`
|
||||
// Endpoint is the backend's absolute base URL.
|
||||
Endpoint *string `yaml:"endpoint"`
|
||||
// APIKeyEnv names an environment variable containing a credential.
|
||||
@@ -163,7 +167,6 @@ type BackendSmokeTestConfig struct {
|
||||
// diagnostics. A nil endpoint uses the effective backend endpoint.
|
||||
Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"`
|
||||
// Authentication optionally declares a named authentication diagnostic.
|
||||
// A supplied object requires Kind.
|
||||
Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"`
|
||||
}
|
||||
|
||||
@@ -178,7 +181,8 @@ type ReachabilityProbeConfig struct {
|
||||
// AuthenticationProbeConfig configures an optional named authentication diagnostic.
|
||||
type AuthenticationProbeConfig struct {
|
||||
// Kind selects the authentication adapter. Supported values are "none" and
|
||||
// "openrouter_key"; surrounding whitespace and case are normalized.
|
||||
// "openrouter_key" and "vllm_models"; surrounding whitespace and case are
|
||||
// normalized. Nil leaves adapter selection to the provider default.
|
||||
Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"`
|
||||
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
|
||||
// Nil selects the adapter's backend-relative destination; an explicit blank
|
||||
|
||||
@@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
||||
return nil, err
|
||||
}
|
||||
defer payload.clear()
|
||||
cost := &observation.CostAccumulator{}
|
||||
ctx = observation.WithCostAccumulator(ctx, cost)
|
||||
scope := NewOperationScope(ctx, payload.operationTimeout)
|
||||
defer scope.Close()
|
||||
defer func() { err = scope.Finalize(err) }()
|
||||
@@ -117,9 +119,11 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
||||
|
||||
var recorder *observation.Recorder
|
||||
var runID string
|
||||
if payload.observer != nil {
|
||||
observer := observation.Compose(payload.observer, observation.ObserverFromContext(ctx))
|
||||
if observer != nil {
|
||||
var idErr error
|
||||
recorder, idErr = observation.New(payload.observer)
|
||||
recorder, idErr = observation.New(observer)
|
||||
recorder.SetCostAccumulator(cost)
|
||||
recorder.SetParentRunID(payload.parent.RunID())
|
||||
observation.RegisterClaim(ctx, recorder)
|
||||
ctx = observation.WithRecorder(ctx, recorder)
|
||||
|
||||
@@ -95,6 +95,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
||||
}
|
||||
admitter := &fakeRunAdmitter{}
|
||||
profile := defaultExecutionProfile()
|
||||
profile.ResolvedTarget.Provider = "vllm"
|
||||
profile.ResolvedTarget.ZeroCost = true
|
||||
profile.ResolvedTarget.ExtraParams = map[string]any{
|
||||
"metadata": map[string]any{"source": "original"},
|
||||
}
|
||||
@@ -144,9 +146,11 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
||||
}
|
||||
|
||||
renderer.rendered.Messages[0].Content = "source mutation"
|
||||
profile.ResolvedTarget.Provider = "openrouter"
|
||||
profile.ResolvedTarget.ZeroCost = false
|
||||
|
||||
second := prepared.Details()
|
||||
if second != first || second.Messages[0].Content != "original message" ||
|
||||
if second != first || second.Messages[0].Content != "original message" || second.EffectiveModelParams.Provider != "vllm" || !second.EffectiveModelParams.ZeroCost ||
|
||||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||
t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
|
||||
@@ -313,7 +317,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
responses: []*domain.GenerateResponse{{
|
||||
Content: `{"repaired":true}`,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5,
|
||||
PromptTokens: 2, CompletionTokens: 3, ReasoningTokens: 2, TotalTokens: 5,
|
||||
CachedTokens: 7, CacheWriteTokens: 11,
|
||||
},
|
||||
}},
|
||||
@@ -324,7 +328,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
client := &sequenceLLM{responses: []*domain.GenerateResponse{{
|
||||
Content: `{"broken":true}`,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
|
||||
PromptTokens: 13, CompletionTokens: 17, ReasoningTokens: 4, TotalTokens: 19,
|
||||
CachedTokens: 23, CacheWriteTokens: 29,
|
||||
},
|
||||
}}}
|
||||
@@ -366,7 +370,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
||||
}
|
||||
wantUsage := domain.TokenUsage{
|
||||
PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24,
|
||||
PromptTokens: 15, CompletionTokens: 20, ReasoningTokens: 6, TotalTokens: 24,
|
||||
CachedTokens: 30, CacheWriteTokens: 40,
|
||||
}
|
||||
if result.Usage != wantUsage {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
|
||||
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
||||
Provider: "openrouter", ZeroCost: true,
|
||||
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
||||
Temperature: 0, ExtraParams: map[string]any{"configured": true},
|
||||
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
|
||||
@@ -35,7 +36,7 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
|
||||
t.Fatalf("request did not complete partial target: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Endpoint != "https://runtime.example/v1" ||
|
||||
prepared.EffectiveModelParams.Model != "runtime-model" || !prepared.TargetPresence.Temperature ||
|
||||
prepared.EffectiveModelParams.Model != "runtime-model" || prepared.EffectiveModelParams.Provider != "openrouter" || !prepared.EffectiveModelParams.ZeroCost || !prepared.TargetPresence.Temperature ||
|
||||
prepared.EffectiveModelParams.ExtraParams["request"] != true {
|
||||
t.Fatalf("request overrides were not applied: %+v", prepared)
|
||||
}
|
||||
|
||||
@@ -137,6 +137,9 @@ func NewRunnerWithRepairer(
|
||||
}
|
||||
|
||||
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
|
||||
if observation.CostFromContext(ctx) == nil {
|
||||
ctx = observation.WithCostAccumulator(ctx, &observation.CostAccumulator{})
|
||||
}
|
||||
recorder := observation.FromContext(ctx)
|
||||
runID := ""
|
||||
if recorder != nil {
|
||||
@@ -274,7 +277,7 @@ func (r *Runner) executePreparedRun(
|
||||
exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content)
|
||||
}
|
||||
|
||||
return &domain.RunResult{
|
||||
result := &domain.RunResult{
|
||||
Exchange: exchange,
|
||||
ParentRunID: prepared.ParentRunID,
|
||||
RunID: runID,
|
||||
@@ -295,7 +298,11 @@ func (r *Runner) executePreparedRun(
|
||||
StartTime: start,
|
||||
EndTime: end,
|
||||
Duration: end.Sub(start),
|
||||
}, nil
|
||||
}
|
||||
if cost := observation.CostFromContext(ctx); cost != nil {
|
||||
result.CostUSD = cost.Observed()
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func wrapGenerationError(err error) error {
|
||||
@@ -309,6 +316,7 @@ func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: total.PromptTokens + next.PromptTokens,
|
||||
CompletionTokens: total.CompletionTokens + next.CompletionTokens,
|
||||
ReasoningTokens: total.ReasoningTokens + next.ReasoningTokens,
|
||||
TotalTokens: total.TotalTokens + next.TotalTokens,
|
||||
CachedTokens: total.CachedTokens + next.CachedTokens,
|
||||
CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens,
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
type smokeAuthenticationClient interface {
|
||||
ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
|
||||
ProbeAuthentication(context.Context, string, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
|
||||
}
|
||||
|
||||
type authenticationProbe struct{ client smokeAuthenticationClient }
|
||||
@@ -30,7 +30,8 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
|
||||
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
|
||||
return outcome, err
|
||||
}
|
||||
result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
|
||||
adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
|
||||
result, requestErr := p.client.ProbeAuthentication(ctx, adapter, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
|
||||
outcome.evidence = result.HTTP
|
||||
outcome.credentialsAccepted = result.Accepted
|
||||
if result.Overflow {
|
||||
@@ -55,22 +56,18 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
|
||||
}
|
||||
|
||||
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
|
||||
if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil {
|
||||
adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
|
||||
if adapter == "" {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
|
||||
}
|
||||
authentication := backend.SmokeTests.Authentication
|
||||
if authentication.Kind == nil {
|
||||
return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest)
|
||||
}
|
||||
switch *authentication.Kind {
|
||||
case "none":
|
||||
if adapter == "none" {
|
||||
return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil
|
||||
case "openrouter_key":
|
||||
if p.client == nil {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
||||
}
|
||||
return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil
|
||||
default:
|
||||
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
|
||||
}
|
||||
if !llm.SupportedAuthenticationAdapter(adapter) {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
|
||||
}
|
||||
if p.client == nil {
|
||||
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
|
||||
}
|
||||
return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil
|
||||
}
|
||||
|
||||
@@ -95,6 +95,11 @@ func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
|
||||
backend: backendWithAuthenticationKind("openrouter_key"),
|
||||
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
|
||||
},
|
||||
{
|
||||
name: "vllm HTTP adapter unavailable",
|
||||
backend: domain.Backend{Provider: "vllm"},
|
||||
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
probe := authenticationProbe{}
|
||||
|
||||
@@ -110,6 +110,8 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
|
||||
} else {
|
||||
settings := defaults.ExecutionTargetDefault()
|
||||
settings.BackendID = selection.backend.ID
|
||||
settings.Provider = selection.backend.Provider
|
||||
settings.ZeroCost = selection.backend.ZeroCost
|
||||
settings.Endpoint = selection.backend.Endpoint
|
||||
settings.APIKeyEnv = selection.backend.APIKeyEnv
|
||||
if len(selection.backend.ExtraParams) > 0 {
|
||||
@@ -132,7 +134,7 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
|
||||
if promptInspection != nil {
|
||||
selection.target.PromptID = promptInspection.PromptID
|
||||
}
|
||||
selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend.SmokeTests)
|
||||
selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
@@ -165,22 +167,15 @@ func isInferenceSmokeTest(kind domain.SmokeTestKind) bool {
|
||||
return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput
|
||||
}
|
||||
|
||||
func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, smokeTests *domain.BackendSmokeTestConfig) (string, error) {
|
||||
func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, backend domain.Backend) (string, error) {
|
||||
smokeTests := backend.SmokeTests
|
||||
switch kind {
|
||||
case domain.SmokeTestReachability:
|
||||
if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil {
|
||||
return *smokeTests.Reachability.Endpoint, nil
|
||||
}
|
||||
case domain.SmokeTestAuthentication:
|
||||
if smokeTests != nil && smokeTests.Authentication != nil {
|
||||
authentication := smokeTests.Authentication
|
||||
if authentication.Endpoint != nil {
|
||||
return *authentication.Endpoint, nil
|
||||
}
|
||||
if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
|
||||
return llm.OpenRouterKeyEndpoint(baseEndpoint)
|
||||
}
|
||||
}
|
||||
return llm.AuthenticationEndpoint(llm.AuthenticationAdapter(backend.Provider, smokeTests), baseEndpoint, smokeTests)
|
||||
}
|
||||
return baseEndpoint, nil
|
||||
}
|
||||
|
||||
127
invocation_workflow_contract_test.go
Normal file
127
invocation_workflow_contract_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) {
|
||||
pendingStarted, release := make(chan struct{}), make(chan struct{})
|
||||
releasePending := sync.OnceFunc(func() { close(release) })
|
||||
defer releasePending()
|
||||
var retryCalls atomic.Int32
|
||||
transport := costRoundTrip(func(req *http.Request) (*http.Response, error) {
|
||||
var body struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status, payload := 200, ""
|
||||
switch body.Model {
|
||||
case "router":
|
||||
if retryCalls.Add(1) == 1 {
|
||||
status, payload = 503, `{"error":{}}`
|
||||
} else {
|
||||
payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`
|
||||
}
|
||||
case "failure":
|
||||
payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}`
|
||||
case "vllm":
|
||||
payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}`
|
||||
case "pending":
|
||||
close(pendingStarted)
|
||||
<-release
|
||||
payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}`
|
||||
default:
|
||||
t.Errorf("unexpected model %q", body.Model)
|
||||
}
|
||||
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil
|
||||
})
|
||||
engine := func(provider, model string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}},
|
||||
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}
|
||||
result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
router, vllm := engine("openrouter", "router"), engine("vllm", "vllm")
|
||||
var group promptkit.ExecutionGroup
|
||||
if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("empty=%+v", empty)
|
||||
}
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||||
results := make(chan error, 4)
|
||||
var completed sync.WaitGroup
|
||||
var pendingDone sync.WaitGroup
|
||||
start := func(e *promptkit.Engine, model string, fail bool) {
|
||||
completed.Add(1)
|
||||
go func() {
|
||||
defer completed.Done()
|
||||
req := promptkit.RunRequest{PromptID: "prompt"}
|
||||
if model != "" {
|
||||
req.Execution = &promptkit.ExecutionTargetOverride{Model: model}
|
||||
}
|
||||
_, err := e.Run(ctx, req)
|
||||
if fail && err != nil {
|
||||
err = nil
|
||||
} else if fail {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
pendingDone.Add(1)
|
||||
go func() {
|
||||
defer pendingDone.Done()
|
||||
_, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}})
|
||||
results <- err
|
||||
}()
|
||||
<-pendingStarted
|
||||
if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete {
|
||||
t.Fatalf("pending=%+v", pending)
|
||||
}
|
||||
start(router, "", false)
|
||||
start(router, "failure", true)
|
||||
start(vllm, "", false)
|
||||
completed.Wait()
|
||||
for range 3 {
|
||||
if err := <-results; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
snapshot := group.Snapshot()
|
||||
if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 {
|
||||
t.Fatalf("snapshot=%+v", snapshot)
|
||||
}
|
||||
if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete {
|
||||
t.Fatalf("usage=%+v", snapshot.Usage)
|
||||
}
|
||||
if _, ok := snapshot.Usage.CacheHitRatio(); ok {
|
||||
t.Fatal("ratio available despite an unreported retry")
|
||||
}
|
||||
releasePending()
|
||||
if err := <-results; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pendingDone.Wait()
|
||||
if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 {
|
||||
t.Fatalf("final=%+v", final)
|
||||
}
|
||||
}
|
||||
@@ -19,9 +19,9 @@ func stringPointerForTest(value string) *string {
|
||||
func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
||||
value := promptkit.GenerateResponse{
|
||||
Content: "ok",
|
||||
Usage: promptkit.TokenUsage{TotalTokens: 5},
|
||||
Usage: promptkit.TokenUsage{TotalTokens: 5, ReasoningTokens: 2},
|
||||
UsagePresence: &promptkit.TokenUsagePresence{
|
||||
PromptTokens: true, CompletionTokens: true, TotalTokens: true,
|
||||
PromptTokens: true, CompletionTokens: true, TotalTokens: true, ReasoningTokens: true,
|
||||
CachedTokens: true, CacheWriteTokens: true,
|
||||
},
|
||||
}
|
||||
@@ -30,6 +30,9 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
object := decodeJSONObject(t, payload)
|
||||
if string(decodeJSONObject(t, object["usage"])["reasoning_tokens"]) != "2" {
|
||||
t.Fatalf("reasoning field missing: %s", payload)
|
||||
}
|
||||
if _, ok := object["usage_presence"]; !ok {
|
||||
t.Fatalf("missing usage_presence: %s", payload)
|
||||
}
|
||||
@@ -46,6 +49,34 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostJSONPreservesUnknownAndReportedZero(t *testing.T) {
|
||||
var old promptkit.RunResult
|
||||
if err := json.Unmarshal([]byte(`{"prompt_id":"old"}`), &old); err != nil || old.CostUSD != (promptkit.ObservedCost{}) {
|
||||
t.Fatalf("old result=%+v err=%v", old, err)
|
||||
}
|
||||
value := promptkit.RunResult{CostUSD: promptkit.ObservedCost{Value: .125, Known: true, Complete: false, Source: promptkit.CostSourceProvider}}
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded promptkit.RunResult
|
||||
if err := json.Unmarshal(data, &decoded); err != nil || decoded.CostUSD != value.CostUSD {
|
||||
t.Fatalf("round trip=%+v err=%v", decoded, err)
|
||||
}
|
||||
zero := 0.0
|
||||
data, err = json.Marshal(promptkit.GenerateResponse{CostUSD: &zero})
|
||||
if err != nil || string(decodeJSONObject(t, data)["cost_usd"]) != "0" {
|
||||
t.Fatalf("reported zero=%s err=%v", data, err)
|
||||
}
|
||||
data, err = json.Marshal(promptkit.GenerateResponse{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, present := decodeJSONObject(t, data)["cost_usd"]; present {
|
||||
t.Fatalf("unknown cost=%s", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovedPromptVersionIsAbsentFromJSON(t *testing.T) {
|
||||
values := map[string]any{
|
||||
"prepared": promptkit.PreparedRun{PromptID: "prompt"},
|
||||
@@ -173,6 +204,7 @@ func TestRunResultJSONContractRoundTripsAllFields(t *testing.T) {
|
||||
"effective_model_params",
|
||||
"input_hashes",
|
||||
"usage",
|
||||
"cost_usd",
|
||||
"start_time",
|
||||
"end_time",
|
||||
"duration_ms",
|
||||
|
||||
@@ -50,10 +50,12 @@ func TestPublicLLMClientAdapterGivesClientOwnedNestedValues(t *testing.T) {
|
||||
|
||||
func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
|
||||
presence := &TokenUsagePresence{PromptTokens: true}
|
||||
cost := .25
|
||||
client := &usagePresenceClient{response: &GenerateResponse{
|
||||
Content: "ok",
|
||||
Usage: TokenUsage{PromptTokens: 0, TotalTokens: 5},
|
||||
UsagePresence: presence,
|
||||
CostUSD: &cost,
|
||||
}}
|
||||
adapter := publicLLMClientAdapter{client: client}
|
||||
got, err := adapter.Generate(context.Background(), domain.GenerateRequest{})
|
||||
@@ -61,17 +63,23 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
presence.PromptTokens = false
|
||||
cost = .5
|
||||
if got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.UsagePresence.TotalTokens || got.Usage.TotalTokens != 5 {
|
||||
t.Fatalf("copied response = %+v", got)
|
||||
}
|
||||
if got.CostUSD == nil || *got.CostUSD != .25 {
|
||||
t.Fatalf("copied cost=%v", got.CostUSD)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {
|
||||
presence := &TokenUsagePresence{PromptTokens: true}
|
||||
cost := .125
|
||||
source := &GenerateResponse{
|
||||
Content: "returned with error",
|
||||
Usage: TokenUsage{PromptTokens: 6},
|
||||
UsagePresence: presence,
|
||||
CostUSD: &cost,
|
||||
}
|
||||
marker := errors.New("injected failure")
|
||||
adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}}
|
||||
@@ -82,7 +90,8 @@ func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T
|
||||
}
|
||||
source.Content = "mutated after return"
|
||||
presence.PromptTokens = false
|
||||
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens {
|
||||
cost = .5
|
||||
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.CostUSD == nil || *got.CostUSD != .125 {
|
||||
t.Fatalf("adapter response was not owned: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -11,6 +12,66 @@ import (
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestContextualObserversSelectIndependentRuns(t *testing.T) {
|
||||
var order []string
|
||||
var events []promptkit.ExecutionEvent
|
||||
observer := func(name string) promptkit.ExecutionObserver {
|
||||
return func(event promptkit.ExecutionEvent) {
|
||||
order = append(order, name)
|
||||
if name == "inherited" {
|
||||
events = append(events, event)
|
||||
}
|
||||
}
|
||||
}
|
||||
base := context.Background()
|
||||
if promptkit.WithExecutionObserver(base, nil) != base {
|
||||
t.Fatal("nil observer changed context")
|
||||
}
|
||||
ctx := promptkit.WithExecutionObserver(base, observer("inherited"))
|
||||
ctx = promptkit.WithExecutionObserver(ctx, observer("new"))
|
||||
ctx = promptkit.WithExecutionObserver(ctx, nil)
|
||||
child := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: "child"}, nil
|
||||
}))
|
||||
var childResult *promptkit.RunResult
|
||||
outer := observationEngine(t, observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
var err error
|
||||
childResult, err = child.Run(callCtx, promptkit.RunRequest{PromptID: "prompt", Observer: observer("child explicit")})
|
||||
return &promptkit.GenerateResponse{Content: "outer"}, err
|
||||
}))
|
||||
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: observer("outer explicit")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"outer explicit", "inherited", "new", "child explicit", "inherited", "new", "child explicit", "inherited", "new", "child explicit", "inherited", "new", "outer explicit", "inherited", "new", "outer explicit", "inherited", "new"}
|
||||
if !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("observer order = %v", order)
|
||||
}
|
||||
if childResult.RunID == "" || result.RunID == "" || childResult.RunID == result.RunID {
|
||||
t.Fatalf("run IDs: outer=%q child=%q", result.RunID, childResult.RunID)
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, event := range events {
|
||||
counts[event.RunID]++
|
||||
}
|
||||
if counts[result.RunID] != 3 || counts[childResult.RunID] != 3 || len(counts) != 2 {
|
||||
t.Fatalf("events = %+v", events)
|
||||
}
|
||||
order = nil
|
||||
if _, err := child.Run(base, promptkit.RunRequest{PromptID: "prompt"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(order) != 0 {
|
||||
t.Fatalf("unrelated context selected observer: %v", order)
|
||||
}
|
||||
if _, err := child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(order, []string{"inherited", "new", "inherited", "new", "inherited", "new"}) {
|
||||
t.Fatalf("context-only observer order = %v", order)
|
||||
}
|
||||
}
|
||||
|
||||
type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
|
||||
|
||||
func (f observationClientFunc) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
|
||||
@@ -16,6 +16,19 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi
|
||||
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
|
||||
}
|
||||
|
||||
func TestInjectedReasoningPresenceReachesObservation(t *testing.T) {
|
||||
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok", UsagePresence: &promptkit.TokenUsagePresence{ReasoningTokens: true}}})
|
||||
var events []promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
||||
if err != nil || result == nil || len(events) != 3 {
|
||||
t.Fatalf("result=%+v events=%+v error=%v", result, events, err)
|
||||
}
|
||||
want := promptkit.ObservedTokenCount{Known: true, Complete: true}
|
||||
if events[1].Usage.ReasoningTokens != want || events[2].Usage.ReasoningTokens != want {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinuationObservationAncestry(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}}
|
||||
engine := observationEngine(t, client)
|
||||
|
||||
@@ -17,6 +17,20 @@ import (
|
||||
// event, and Promptkit keeps no asynchronous queue or durable history.
|
||||
type ExecutionObserver func(ExecutionEvent)
|
||||
|
||||
// WithExecutionObserver adds an observer to executions using the returned
|
||||
// context. Request observers run first, then inherited contextual observers
|
||||
// in registration order. A nil observer leaves the context unchanged.
|
||||
// Prepared executions use their execution context, not their preparation
|
||||
// context. Callers own context propagation and synchronization of observers.
|
||||
func WithExecutionObserver(ctx context.Context, observer ExecutionObserver) context.Context {
|
||||
if observer == nil {
|
||||
return ctx
|
||||
}
|
||||
return observation.WithObserver(ctx, func(event domain.ExecutionEvent) {
|
||||
observer(fromDomainExecutionEvent(event))
|
||||
})
|
||||
}
|
||||
|
||||
// ComposeObservers returns a synchronous ordered fan-out of non-nil observers.
|
||||
// It copies the input slice; a panic propagates and prevents later callbacks.
|
||||
func ComposeObservers(observers ...ExecutionObserver) ExecutionObserver {
|
||||
@@ -83,11 +97,24 @@ type ObservedTokenCount struct {
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
ReasoningTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
}
|
||||
|
||||
// CacheHitRatio returns the fraction of input tokens served from cache when
|
||||
// both counts are known, complete, and consistent, with positive input and
|
||||
// reads between zero and input inclusive. Otherwise it returns (0, false).
|
||||
// Cache writes are independent; aggregate ratios use summed counts.
|
||||
func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) {
|
||||
input, reads := u.PromptTokens, u.CachedTokens
|
||||
if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value {
|
||||
return 0, false
|
||||
}
|
||||
return float64(reads.Value) / float64(input.Value), true
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only, not provider text, request content,
|
||||
// credentials, URLs, headers, or error objects. It has no stable JSON contract.
|
||||
// Identifiers themselves may be sensitive. Nonterminal aggregate counters are
|
||||
@@ -127,6 +154,7 @@ type ExecutionEvent struct {
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
@@ -144,8 +172,9 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
|
||||
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
|
||||
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
|
||||
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
|
||||
CostUSD: fromDomainCost(value.CostUSD),
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
|
||||
TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
|
||||
TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
|
||||
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@ const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
|
||||
// A successful PrepareExecution may retain an Observer function in this
|
||||
// handle's private payload. The callback and other private execution settings
|
||||
// are cleared on claim completion or Discard. Details never exposes them.
|
||||
// Contextual observers are selected from the RunPrepared context, not retained
|
||||
// from preparation.
|
||||
// It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for
|
||||
// execution of the frozen snapshot. Only RunPrepared can produce its completed
|
||||
// [Exchange]; preparation alone does not.
|
||||
|
||||
@@ -3,6 +3,7 @@ package promptkit_test
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
@@ -10,6 +11,50 @@ import (
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestPreparedObservationUsesExecutionContext(t *testing.T) {
|
||||
engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
|
||||
foreign := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
|
||||
var order []string
|
||||
observe := func(name string) promptkit.ExecutionObserver {
|
||||
return func(promptkit.ExecutionEvent) { order = append(order, name) }
|
||||
}
|
||||
prepareCtx := promptkit.WithExecutionObserver(context.Background(), observe("preparation"))
|
||||
executeCtx := promptkit.WithExecutionObserver(context.Background(), observe("execution"))
|
||||
handle, err := engine.PrepareExecution(prepareCtx, promptkit.RunRequest{PromptID: "prepared", Observer: observe("explicit")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(order) != 0 {
|
||||
t.Fatalf("preparation emitted events: %v", order)
|
||||
}
|
||||
if _, err := foreign.RunPrepared(executeCtx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
|
||||
t.Fatalf("foreign claim: %v, %v", err, order)
|
||||
}
|
||||
if _, err := engine.RunPrepared(executeCtx, handle); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(order, []string{"explicit", "execution", "explicit", "execution", "explicit", "execution"}) {
|
||||
t.Fatalf("observer order: %v", order)
|
||||
}
|
||||
order = nil
|
||||
if _, err := engine.RunPrepared(executeCtx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
|
||||
t.Fatalf("reused claim: %v, %v", err, order)
|
||||
}
|
||||
if _, err := engine.RunPrepared(executeCtx, nil); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
|
||||
t.Fatalf("nil claim: %v, %v", err, order)
|
||||
}
|
||||
handle, err = engine.PrepareExecution(prepareCtx, promptkit.RunRequest{PromptID: "prepared"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := engine.RunPrepared(executeCtx, handle); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(order, []string{"execution", "execution", "execution"}) {
|
||||
t.Fatalf("context-only order: %v", order)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) {
|
||||
engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
|
||||
var events []promptkit.ExecutionEvent
|
||||
|
||||
@@ -28,7 +28,11 @@ type ProfileInspection struct {
|
||||
// caller-owned. Nil pointers denote absent fields; non-nil pointers preserve
|
||||
// explicit zero values and empty collections in JSON and YAML.
|
||||
type BackendInspection struct {
|
||||
BackendID string `json:"id" yaml:"id"`
|
||||
BackendID string `json:"id" yaml:"id"`
|
||||
// Provider is the normalized declaration, not a manufactured runtime default.
|
||||
Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"`
|
||||
// ZeroCost preserves an explicitly declared false as well as true.
|
||||
ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"`
|
||||
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
||||
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
|
||||
ConcurrencyLimit *int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit,omitempty"`
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
|
||||
"backends": []any{map[string]any{"id": "local", "provider": " VLLM ", "zero_cost": false, "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
|
||||
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}}})
|
||||
@@ -77,7 +77,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
b := decode(backend)
|
||||
if b["id"] != "local" || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
|
||||
if b["id"] != "local" || b["provider"] != "vllm" || b["zero_cost"] != false || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
|
||||
t.Errorf("backend = %#v", b)
|
||||
}
|
||||
params := b["extra_params"].(map[string]any)
|
||||
@@ -92,7 +92,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||
t.Errorf("reachability = %#v", reach)
|
||||
}
|
||||
absent(b, "extends")
|
||||
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit")
|
||||
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit", "provider", "zero_cost")
|
||||
p := decode(profile)
|
||||
if p["id"] != "profile" || p["backend"] != "local" || p["model"] != "model" || p["endpoint"] != "" || p["api_key_env"] != "" || p["service_tier"] != "" || p["reasoning_effort"] != "" || p["api_key_required"] != false || fmt.Sprint(p["temperature"]) != "0" || fmt.Sprint(p["top_p"]) != "0" || fmt.Sprint(p["max_tokens"]) != "0" || fmt.Sprint(p["timeout_seconds"]) != "0" || len(p["extra_params"].(map[string]any)) != 0 {
|
||||
t.Errorf("profile = %#v", p)
|
||||
|
||||
@@ -65,3 +65,21 @@ func TestSharedSmokeSettingsWorkWithBackendSource(t *testing.T) {
|
||||
t.Fatalf("backend source = %+v, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendSourcePreservesProviderFieldPresence(t *testing.T) {
|
||||
blank, disabled := "", false
|
||||
result, err := configkit.Load[promptkit.Settings](promptkit.BackendSource("backends", []promptkit.BackendConfig{{ID: "omitted"}, {ID: "supplied", Provider: &blank, ZeroCost: &disabled}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Config.Backends[0].Provider != nil || result.Config.Backends[0].ZeroCost != nil {
|
||||
t.Fatalf("omitted values: %+v", result.Config.Backends[0])
|
||||
}
|
||||
got := result.Config.Backends[1]
|
||||
if got.Provider == nil || *got.Provider != "" || got.ZeroCost == nil || *got.ZeroCost {
|
||||
t.Fatalf("supplied values: %+v", got)
|
||||
}
|
||||
if local := promptkit.LocalBackend("https://example.invalid/v1", 0); local.Provider != nil || local.ZeroCost != nil {
|
||||
t.Fatalf("local defaults: %+v", local)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,6 +368,7 @@ func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedToken
|
||||
return ObservedTokenUsage{
|
||||
PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens),
|
||||
CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens),
|
||||
ReasoningTokens: fromDomainObservedTokenCount(usage.ReasoningTokens),
|
||||
TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens),
|
||||
CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens),
|
||||
CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens),
|
||||
|
||||
@@ -5,13 +5,40 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestSmokeAuthenticationUsesProviderDefault(t *testing.T) {
|
||||
var path string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path = r.URL.Path
|
||||
fmt.Fprint(w, `{"data":{"label":"member","is_free_tier":false,"usage":0}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("diagnostic", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "renamed", "provider": "openrouter", "endpoint": server.URL + "/v1"}},
|
||||
})}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inspection, err := engine.InspectBackend(context.Background(), "renamed")
|
||||
if err != nil || inspection.SmokeTests != nil {
|
||||
t.Fatalf("declared metadata: %+v, %v", inspection, err)
|
||||
}
|
||||
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "renamed"})
|
||||
if err != nil || !report.Supported || !report.Completed || path != "/v1/key" || report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted {
|
||||
t.Fatalf("report=%+v path=%q err=%v", report, path, err)
|
||||
}
|
||||
}
|
||||
|
||||
type gatedSmokeClient struct {
|
||||
started chan promptkit.GenerateRequest
|
||||
release <-chan struct{}
|
||||
|
||||
13
types.go
13
types.go
@@ -298,6 +298,8 @@ type RunResult struct {
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
// Usage is the token accounting reported by the LLM client.
|
||||
Usage TokenUsage `json:"usage"`
|
||||
// CostUSD is the observed charge subtotal across dispatched attempts.
|
||||
CostUSD ObservedCost `json:"cost_usd"`
|
||||
// StartTime is the UTC time immediately before ordinary Run preparation or
|
||||
// after RunPrepared claims its handle.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
@@ -568,9 +570,11 @@ type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// CompletionTokens is the reported generated-token count.
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
// ReasoningTokens is the reported reasoning subset of completion tokens.
|
||||
ReasoningTokens int `json:"reasoning_tokens"`
|
||||
// TotalTokens is the reported total-token count.
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
// CachedTokens is the reported cached-input-token count.
|
||||
// CachedTokens is the reported input-token cache-read count.
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
// CacheWriteTokens is the reported cache-write-token count.
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
@@ -656,7 +660,9 @@ type StructuredOutputJSONSpec struct {
|
||||
// treated as known and zero counts as unknown for observation only; the legacy
|
||||
// interface cannot distinguish omitted zero from measured zero. An explicit
|
||||
// UsagePresence can report known zero counts. Result usage sums the supplied
|
||||
// Usage values independently of presence.
|
||||
// Usage values independently of presence. A non-nil CostUSD is copied and
|
||||
// retained for attempt accounting even when returned alongside an error;
|
||||
// non-finite values are treated as unknown before configured-zero fallback.
|
||||
type LLMClient interface {
|
||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||
}
|
||||
@@ -690,6 +696,8 @@ type GenerateResponse struct {
|
||||
// UsagePresence states which usage fields were reported, including zeros.
|
||||
// Nil uses the legacy observation fallback described by LLMClient.
|
||||
UsagePresence *TokenUsagePresence `json:"usage_presence,omitempty"`
|
||||
// CostUSD is an optional reported charge; a pointer to zero reports zero.
|
||||
CostUSD *float64 `json:"cost_usd,omitempty"`
|
||||
}
|
||||
|
||||
// TokenUsagePresence identifies provider-reported token counts for observation.
|
||||
@@ -697,6 +705,7 @@ type GenerateResponse struct {
|
||||
type TokenUsagePresence struct {
|
||||
PromptTokens bool `json:"prompt_tokens"`
|
||||
CompletionTokens bool `json:"completion_tokens"`
|
||||
ReasoningTokens bool `json:"reasoning_tokens"`
|
||||
TotalTokens bool `json:"total_tokens"`
|
||||
CachedTokens bool `json:"cached_tokens"`
|
||||
CacheWriteTokens bool `json:"cache_write_tokens"`
|
||||
|
||||
Reference in New Issue
Block a user