28 Commits

Author SHA1 Message Date
207677e994 Prepare the v0.25.0 release documentation 2026-11-22 19:08:29 +00:00
20c5ea47af Document invocation observation and cost reporting 2026-11-22 18:38:11 +00:00
d4509b163e Cover concurrent multi-provider invocation accounting 2026-11-22 18:32:29 +00:00
ba4de9d11e Isolate cost accounting for prepared executions 2026-11-22 18:30:02 +00:00
d77e3e2f83 Report costs for ordinary runs 2026-11-22 18:28:39 +00:00
81a2a7591d Collect costs for dispatched model attempts 2026-11-22 18:27:01 +00:00
cd0e1f5d39 Capture optional provider reported costs 2026-11-22 18:25:32 +00:00
6ed0e037c4 Define observed cost values and aggregation 2026-11-22 18:24:33 +00:00
3d82cad2c9 Aggregate reasoning usage and derive cache hit ratios 2026-11-22 18:22:07 +00:00
db1cad6d2d Decode provider reasoning and cache usage details 2026-11-22 18:20:11 +00:00
c3257b63f0 Add vLLM models authentication diagnostics 2026-11-22 18:17:30 +00:00
ca8cf1b827 Resolve authentication diagnostics by provider 2026-11-22 18:16:13 +00:00
adc700f96e Capture provider policy in backend execution targets 2026-11-22 18:13:17 +00:00
6056f93e55 Add backend provider and cost configuration 2026-11-22 18:11:10 +00:00
f2b0ed8d6c Select prepared execution observers at run time 2026-11-22 18:08:28 +00:00
ea7cc170ad Select execution observers from context 2026-11-22 18:07:17 +00:00
f762c0358e Plan invocation observation and usage implementation 2026-11-22 18:01:31 +00:00
a47df4ec20 Consolidate the observation roadmap around the agreed target state 2026-11-22 17:52:58 +00:00
fed9ad082d Clarify inspection and diagnostic accounting roadmap 2026-11-22 17:16:14 +00:00
4ee64b6eda Add explicit zero-cost declarations and cost-source reporting to the roadmap 2026-11-22 17:11:57 +00:00
440d9a2c88 Add vLLM provider conventions to the observation roadmap 2026-11-22 15:57:37 +00:00
23ad9e2a26 Add reasoning and cache usage reporting to the roadmap 2026-11-22 15:54:02 +00:00
80649c579d Define the default OpenAI-compatible provider in the roadmap 2026-11-22 15:49:17 +00:00
0254eb1b5b Clarify provider selection and diagnostic defaults in the roadmap 2026-11-22 15:41:20 +00:00
a10d1a2245 Select provider conventions through backend configuration 2026-11-22 15:27:12 +00:00
0564c8086c Clarify observation accounting and implementation planning 2026-11-22 10:02:22 +00:00
03209947f6 Add optional OpenRouter cost reporting to the observation roadmap 2026-11-22 09:48:38 +00:00
1070be8d89 Define invocation-scoped observation roadmap 2026-11-22 09:39:27 +00:00
73 changed files with 1908 additions and 140 deletions

View File

@@ -45,6 +45,12 @@ boundary and constraints that framework work must preserve.
## Release Guidance ## Release Guidance
The [v0.25.0 release notes](docs/releases/v0.25.0.md) cover invocation-scoped
observation, provider conventions, reasoning and cache usage, and optional USD
cost reporting. Upgrading consumers should review provider selection,
authentication diagnostic defaults, and the corrected OpenRouter cache-write
mapping.
The [v0.24.0 release notes](docs/releases/v0.24.0.md) cover application-owned The [v0.24.0 release notes](docs/releases/v0.24.0.md) cover application-owned
configuration loading, prompt-owned schemas, default output repair, and the configuration loading, prompt-owned schemas, default output repair, and the
prepared message-content hash. Upgrading consumers should review the breaking prepared message-content hash. Upgrading consumers should review the breaking

View File

@@ -26,6 +26,10 @@ type BackendConfig struct {
ID string ID string
// Extends names a sibling backend to inherit through ConfigKit. // Extends names a sibling backend to inherit through ConfigKit.
Extends *string Extends *string
// Provider selects the backend's supported API conventions.
Provider *string
// ZeroCost enables a zero-cost fallback when no usable charge is reported.
ZeroCost *bool
// Endpoint is the required absolute HTTP(S) base URL after all layers merge. // Endpoint is the required absolute HTTP(S) base URL after all layers merge.
Endpoint *string Endpoint *string
// APIKeyEnv optionally names an environment variable, not a credential. // APIKeyEnv optionally names an environment variable, not a credential.
@@ -55,6 +59,12 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
if config.Extends != nil { if config.Extends != nil {
fields["extends"] = map[string]any{"id": *config.Extends} fields["extends"] = map[string]any{"id": *config.Extends}
} }
if config.Provider != nil {
fields["provider"] = *config.Provider
}
if config.ZeroCost != nil {
fields["zero_cost"] = *config.ZeroCost
}
if config.Endpoint != nil { if config.Endpoint != nil {
fields["endpoint"] = *config.Endpoint fields["endpoint"] = *config.Endpoint
} }

View File

@@ -0,0 +1,70 @@
package promptkit_test
import (
"context"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestDuplicateContextAndRequestGroupEnrollment(t *testing.T) {
charge := .5
engine := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe}); err != nil {
t.Fatal(err)
}
got := group.Snapshot()
if len(got.Members) != 1 || got.Attempts != 1 || got.CostUSD.Value != .5 {
t.Fatalf("summary=%+v", got)
}
}
func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) {
known := func(v int) promptkit.ObservedTokenCount {
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
}
for _, tc := range []struct {
name string
input, reads promptkit.ObservedTokenCount
ratio float64
ok bool
}{
{name: "fraction", input: known(8), reads: known(3), ratio: .375, ok: true},
{name: "zero reads", input: known(8), reads: known(0), ok: true},
{name: "zero input", input: known(0), reads: known(0)},
{name: "negative reads", input: known(8), reads: known(-1)},
{name: "reads exceed input", input: known(8), reads: known(9)},
{name: "unknown", input: known(8)},
{name: "incomplete", input: known(8), reads: promptkit.ObservedTokenCount{Value: 3, Known: true}},
} {
t.Run(tc.name, func(t *testing.T) {
usage := promptkit.ObservedTokenUsage{PromptTokens: tc.input, CachedTokens: tc.reads}
got, ok := usage.CacheHitRatio()
if ok != tc.ok || math.Abs(got-tc.ratio) > 1e-12 {
t.Fatalf("ratio=%v available=%t", got, ok)
}
})
}
}
func TestGroupCacheRatioUsesSummedTokensAndDeduplicatesTerminals(t *testing.T) {
known := func(v int) promptkit.ObservedTokenCount {
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
}
var group promptkit.ExecutionGroup
first := promptkit.ExecutionEvent{RunID: "a", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(10), CachedTokens: known(10), ReasoningTokens: known(1)}}
second := promptkit.ExecutionEvent{RunID: "b", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(90), CachedTokens: known(0), ReasoningTokens: known(2)}}
group.Observe(first)
group.Observe(first)
group.Observe(second)
snapshot := group.Snapshot()
ratio, ok := snapshot.Usage.CacheHitRatio()
if !ok || math.Abs(ratio-.1) > 1e-12 || snapshot.Usage.ReasoningTokens != known(3) {
t.Fatalf("summary=%+v ratio=%v available=%t", snapshot, ratio, ok)
}
}

View File

@@ -134,6 +134,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
InputHashes: copyStringMap(result.InputHashes), InputHashes: copyStringMap(result.InputHashes),
Usage: fromDomainTokenUsage(result.Usage), Usage: fromDomainTokenUsage(result.Usage),
CostUSD: fromDomainCost(result.CostUSD),
StartTime: result.StartTime, StartTime: result.StartTime,
EndTime: result.EndTime, EndTime: result.EndTime,
Duration: result.Duration, Duration: result.Duration,
@@ -158,6 +159,7 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
Content: resp.Content, Content: resp.Content,
Usage: toDomainTokenUsage(resp.Usage), Usage: toDomainTokenUsage(resp.Usage),
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence), UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
CostUSD: copyFloat64Ptr(resp.CostUSD),
} }
} }
@@ -168,6 +170,7 @@ func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsage
return &domain.TokenUsagePresence{ return &domain.TokenUsagePresence{
PromptTokens: presence.PromptTokens, PromptTokens: presence.PromptTokens,
CompletionTokens: presence.CompletionTokens, CompletionTokens: presence.CompletionTokens,
ReasoningTokens: presence.ReasoningTokens,
TotalTokens: presence.TotalTokens, TotalTokens: presence.TotalTokens,
CachedTokens: presence.CachedTokens, CachedTokens: presence.CachedTokens,
CacheWriteTokens: presence.CacheWriteTokens, CacheWriteTokens: presence.CacheWriteTokens,
@@ -265,7 +268,7 @@ func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
} }
func fromBackendResource(v *configuration.BackendResource) *BackendInspection { func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)} out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, Provider: copyStringPtr(v.Provider), ZeroCost: copyBoolPtr(v.ZeroCost), APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
if v.ExtraParams != nil { if v.ExtraParams != nil {
data := copyAnyMap(v.ExtraParams) data := copyAnyMap(v.ExtraParams)
out.ExtraParams = &data out.ExtraParams = &data
@@ -356,6 +359,7 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
return TokenUsage{ return TokenUsage{
PromptTokens: usage.PromptTokens, PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens, CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens, TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens, CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens, CacheWriteTokens: usage.CacheWriteTokens,
@@ -366,6 +370,7 @@ func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
return domain.TokenUsage{ return domain.TokenUsage{
PromptTokens: usage.PromptTokens, PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens, CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens, TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens, CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens, CacheWriteTokens: usage.CacheWriteTokens,

View File

@@ -3,6 +3,7 @@ package promptkit_test
import ( import (
"context" "context"
"errors" "errors"
"reflect"
"testing" "testing"
"testing/synctest" "testing/synctest"
"time" "time"
@@ -10,6 +11,31 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestCoordinateObserverOrderWithContext(t *testing.T) {
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
if err != nil {
t.Fatal(err)
}
var order []string
observer := func(name string) promptkit.ExecutionObserver {
return func(promptkit.ExecutionEvent) { order = append(order, name) }
}
ctx := promptkit.WithExecutionObserver(context.Background(), observer("context"))
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: observer("coordinator"), Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
if turn.RemainingTurns == 0 {
return nil, nil
}
return &promptkit.RunRequest{PromptID: "prompt", Observer: observer("request")}, nil
}})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"request", "coordinator", "context", "request", "coordinator", "context", "request", "coordinator", "context"}) {
t.Fatalf("observer order = %v", order)
}
}
func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) { func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) {
synctest.Test(t, func(t *testing.T) { synctest.Test(t, func(t *testing.T) {
engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {

42
cost.go Normal file
View File

@@ -0,0 +1,42 @@
package promptkit
import "go.maximumdirect.net/promptkit/internal/domain"
// CostSource identifies how an observed USD cost was supplied.
type CostSource string
const (
// CostSourceNone denotes no known sourced contribution.
CostSourceNone CostSource = ""
// CostSourceProvider denotes a reported provider or injected-client amount.
CostSourceProvider CostSource = "provider"
// CostSourceConfiguredZero denotes an explicitly configured zero fallback.
CostSourceConfiguredZero CostSource = "configured_zero"
// CostSourceMixed denotes both provider and configured-zero contributions.
CostSourceMixed CostSource = "mixed"
)
// ObservedCost reports an available USD subtotal and its coverage and source.
// Known means at least one contribution is available. Complete means all
// included dispatched attempts have an amount under the selected policy; it
// does not verify billing. A completed run with no dispatched attempts, or an
// empty group, has known complete zero with no source. A group with members but
// no terminals has unknown cost. Amounts are unrounded binary floating point.
type ObservedCost struct {
// Value is the known USD subtotal.
Value float64 `json:"value"`
// Known reports whether a subtotal is available.
Known bool `json:"known"`
// Complete reports whether all included dispatched attempts are covered.
Complete bool `json:"complete"`
// Source summarizes the known contributions.
Source CostSource `json:"source"`
}
func fromDomainCost(value domain.ObservedCost) ObservedCost {
return ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: CostSource(value.Source)}
}
func toDomainCost(value ObservedCost) domain.ObservedCost {
return domain.ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: domain.CostSource(value.Source)}
}

View File

@@ -0,0 +1,131 @@
package promptkit_test
import (
"context"
"errors"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestPreparedCostAndRejectedClaims(t *testing.T) {
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "bad"}, nil
}
return nil, errors.New("repair failed")
})))
ctx := context.Background()
var events []promptkit.ExecutionEvent
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
if err != nil || len(events) != 0 {
t.Fatalf("prepare error=%v events=%+v", err, events)
}
result, err := engine.RunPrepared(ctx, handle)
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
}
events = nil
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
t.Fatalf("reused err=%v events=%+v", err, events)
}
}
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
childCalls := 0
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
childCalls++
charge := .2
if childCalls == 2 {
charge = .3
}
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
var childRun, childPrepared *promptkit.RunResult
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
var err error
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
return nil, err
}
childPrepared, err = child.RunPrepared(ctx, handle)
if err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
}
summary := group.Snapshot()
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 2.0
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
})))
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
summary := group.Snapshot()
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
}
}
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := .5
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var childResult *promptkit.RunResult
var childErr error
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
}})
if err != nil {
t.Fatal(err)
}
var group promptkit.ExecutionGroup
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
}
summary := group.Snapshot()
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}

127
cost_run_contract_test.go Normal file
View File

@@ -0,0 +1,127 @@
package promptkit_test
import (
"context"
"errors"
"io"
"math"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type costRoundTrip func(*http.Request) (*http.Response, error)
func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
options = append(options, promptkit.WithBundledOpenRouter(false))
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...)
if err != nil {
t.Fatal(err)
}
return engine
}
func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
for _, observed := range []bool{false, true} {
t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) {
calls := 0
transport := costRoundTrip(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil
})
engine := costEngine(t, false, &http.Client{Transport: transport})
var events []promptkit.ExecutionEvent
req := promptkit.RunRequest{PromptID: "prompt"}
if observed {
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
}
result, err := engine.Run(context.Background(), req)
want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider}
if err != nil || result == nil || result.CostUSD != want || calls != 2 {
t.Fatalf("result=%+v calls=%d err=%v", result, calls, err)
}
if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) {
t.Fatalf("events=%+v", events)
}
})
}
}
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
charge := .125
invalid := math.Inf(1)
for _, tc := range []struct {
name string
response *promptkit.GenerateResponse
err error
want promptkit.ObservedCost
}{
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
{"nonfinite report falls back", &promptkit.GenerateResponse{Content: "ok", CostUSD: &invalid}, nil, promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
} {
t.Run(tc.name, func(t *testing.T) {
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return tc.response, tc.err
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
})
}
}
func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) {
charge := .1
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "invalid"}, nil
}
return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed}
if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want {
t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err)
}
}
func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) {
engine := costEngine(t, true, nil)
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
}

View File

@@ -728,6 +728,40 @@ counts, but its zero counts remain unknown to observations; supply explicit
presence when measured zeros matter. Existing `RunResult.Usage` sums returned presence when measured zeros matter. Existing `RunResult.Usage` sums returned
values independently of observation presence. values independently of observation presence.
For an application-owned invocation spanning several calls or engines, attach
an observer once to a context and propagate it to those calls:
```go
var group promptkit.ExecutionGroup
ctx = promptkit.WithExecutionObserver(ctx, group.Observe)
result, err := engine.Run(ctx, request)
// After joining all application-owned work, inspect group.Snapshot(), even on error.
```
Explicit request observers run first, followed by contextual observers in
registration order. Coordinated continuations pick up the context through
`Run`; prepared handles retain the request observer but use the execution
context's observers, not the preparation context's. Each nested call has its
own run ID and accounting. Duplicate registration can deliver duplicate events;
groups count each run's first terminal once.
Reasoning tokens are a subset of completion tokens; cache reads and writes are
input details, not extra tokens. `ObservedTokenUsage.CacheHitRatio()` derives
the cache-read fraction only when input and read totals are known, complete,
positive-denominator, and consistent. It divides aggregate counts rather than
averaging per-run ratios and is independent of cache writes.
`RunResult.CostUSD` is available even without an observer; attempt and terminal
events and group snapshots also report cost. `Known` denotes an available
subtotal, `Complete` covers dispatched attempts, and `Source` distinguishes
provider reports, an explicitly configured zero fallback, and mixed sources.
Reported cost, including zero, takes precedence over fallback. Unknown failed
attempts make a subtotal incomplete; configured zero may cover them. Terminal
events can report cost when a failed operation returns no result. Neither
complete reporting nor provider attribution verifies an invoice; consumers
choose presentation precision. See [backend definitions](../formats.md#backend-definitions)
and the [integration contract](../integrations/openai-compatible-chat.md#response-handling).
Callbacks run synchronously, in order for one operation, and can overlap for Callbacks run synchronously, in order for one operation, and can overlap for
different operations. Synchronize shared callback state and avoid blocking on different operations. Synchronize shared callback state and avoid blocking on
work that needs the same run's admission slot. Callback panics propagate; work that needs the same run's admission slot. Callback panics propagate;
@@ -879,15 +913,15 @@ outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
}) })
``` ```
`assess` is application code and explicitly selects its observer on any nested `assess` is application code and can select its observer explicitly or inherit
run. A completed turn releases its admission lease before the callback, so that one from the context. A completed turn releases its admission lease before the callback, so that
assessment may use the same backend. `stopped` and `exhausted` describe control assessment may use the same backend. `stopped` and `exhausted` describe control
flow, not content acceptance. Cancellation before callback delivery can return one flow, not content acceptance. Cancellation before callback delivery can return one
completed response in `Undelivered`; errors retain their ordinary identities. completed response in `Undelivered`; errors retain their ordinary identities.
### Collect Selected Execution Facts ### Collect Selected Execution Facts
Groups are independent of continuation and enrollment is explicit: Groups are independent of continuation and can be enrolled explicitly:
```go ```go
var group, validatorGroup promptkit.ExecutionGroup var group, validatorGroup promptkit.ExecutionGroup
@@ -904,7 +938,7 @@ _, _ = summary, validatorSummary
``` ```
Only the first terminal event per run contributes counts and usage; coverage of Only the first terminal event per run contributes counts and usage; coverage of
currently observed members is separate from token-usage completeness. Empty currently observed members is separate from token and cost completeness. Empty
groups report complete, known zero totals, while members without terminals do groups report complete, known zero totals, while members without terminals do
not invent totals. Overlapping groups are not generally additive. See the not invent totals. Overlapping groups are not generally additive. See the
[offline example](../../examples/go-library/observe/main.go) for complete setup. [offline example](../../examples/go-library/observe/main.go) for complete setup.

View File

@@ -223,7 +223,7 @@ Backend definitions are supplied through caller-selected configuration
sources. The conventional backend IDs are names only; PromptKit does not sources. The conventional backend IDs are names only; PromptKit does not
install definitions for them. install definitions for them.
A backend definition may have `endpoint`, `api_key_env`, A backend definition may have `provider`, `zero_cost`, `endpoint`, `api_key_env`,
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The `concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
final endpoint is required and must be an absolute HTTP(S) URL with a host, no final endpoint is required and must be an absolute HTTP(S) URL with a host, no
user information, query, or fragment. An omitted or null API-key environment user information, query, or fragment. An omitted or null API-key environment
@@ -237,6 +237,17 @@ unrecognized backend field is application data and is not automatically redacted
by Promptkit. Applications own sensitivity declarations and disclosure policy by Promptkit. Applications own sensitivity declarations and disclosure policy
for their extension fields. for their extension fields.
`provider` selects response and diagnostic conventions independently of the
backend ID, endpoint, or model. Its final value is trimmed and lowercased;
omitted, null, or blank selects `openai-compatible` at runtime. Supported
nonblank values are `openai-compatible`, `openrouter`, and `vllm`. `zero_cost`
is an optional Boolean: true supplies a zero-cost fallback when a dispatched
attempt has no usable reported cost; omitted, null, and false disable it.
Neither setting is sent to the provider. They follow normal layering and
inheritance, and endpoint overrides retain the selected backend's policy.
Resource inspection shows declared normalized values without inventing runtime
defaults; configuration inspection retains source history.
`smoke_tests` is optional backend metadata for explicit diagnostic operations. `smoke_tests` is optional backend metadata for explicit diagnostic operations.
It may contain `default_profile`, `reachability`, and `authentication`: It may contain `default_profile`, `reachability`, and `authentication`:
@@ -253,7 +264,7 @@ backends:
endpoint: https://openrouter.ai/api/v1/key endpoint: https://openrouter.ai/api/v1/key
``` ```
An omitted or null `smoke_tests` block disables these metadata defaults. An omitted or null `smoke_tests` block removes its declared metadata.
`default_profile` is preserved as an exact, nonblank profile ID; its existence `default_profile` is preserved as an exact, nonblank profile ID; its existence
is checked only if an inference diagnostic selects that default. Omitted/null is checked only if an inference diagnostic selects that default. Omitted/null
`default_profile` means there is no inference default. Omitted/null `default_profile` means there is no inference default. Omitted/null
@@ -262,12 +273,22 @@ reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
host; unlike the backend base endpoint, its path and query are allowed. Null host; unlike the backend base endpoint, its path and query are allowed. Null
means fallback, while an explicit blank value is invalid. means fallback, while an explicit blank value is invalid.
Omitted/null `authentication` means no authentication adapter is declared. A Omitted/null `authentication` leaves the provider's diagnostic default in
non-null object requires `kind`, which is trimmed and case-normalized; the effect; `openrouter` defaults to `openrouter_key`, `vllm` defaults to
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint `vllm_models`, and the generic provider has no default. A non-null object's
optional `kind` is trimmed and case-normalized; accepted
values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint
uses the adapter's backend-relative destination (`openrouter_key` appends uses the adapter's backend-relative destination (`openrouter_key` appends
`/key` to the effective backend base); an explicit endpoint follows the same `/key` to the effective backend base); an explicit endpoint follows the same
full-URL rules as reachability. `none` declares authentication inapplicable. full-URL rules as reachability. `none` declares authentication inapplicable.
Without a selected adapter, authentication diagnostics report an unsupported
selection. An endpoint-only authentication
block overrides the destination of a provider default; `kind: none` suppresses
that default.
Previously, a supplied authentication object required `kind`. Endpoint-only
objects now use a provider default where available; clearing the object
restores that default rather than suppressing it. Use `kind: none` to suppress
authentication testing.
These values are configuration only: engine construction does not contact a These values are configuration only: engine construction does not contact a
backend. backend.
@@ -358,7 +379,7 @@ does not disable the caller context or HTTP transport timeout.
### Bundled OpenRouter Configuration ### Bundled OpenRouter Configuration
The default bundle installs root backend `openrouter`, endpoint The default bundle installs root backend `openrouter`, provider `openrouter`, endpoint
`https://openrouter.ai/api/v1`, optional credential environment name `https://openrouter.ai/api/v1`, optional credential environment name
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the `OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback `openrouter_key` authentication adapter. Existing diagnostic endpoint fallback

View File

@@ -58,7 +58,7 @@ report facts, and injected-client behavior.
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
one unauthenticated `GET` to the selected diagnostic URL; an observed response one unauthenticated `GET` to the selected diagnostic URL; an observed response
counts as reachable even when its status is non-2xx. Authentication behavior counts as reachable even when its status is non-2xx. Authentication behavior
is selected from backend metadata, not inferred from the host. The is selected from backend metadata and provider defaults, not inferred from the host. The
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective `openrouter_key` adapter makes one `GET` to `/key` relative to the effective
backend base, unless configuration supplies a full diagnostic endpoint (see backend base, unless configuration supplies a full diagnostic endpoint (see
the [backend format reference](../formats.md#backend-definitions)). It may send the [backend format reference](../formats.md#backend-definitions)). It may send
@@ -73,6 +73,15 @@ not validation of the full account document. Other statuses and unrecognized
recognition allowance reports `authentication_response_limit_exceeded`, leaving recognition allowance reports `authentication_response_limit_exceeded`, leaving
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go). credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
The `vllm_models` adapter issues one `GET` to `/models` relative to the
configured base unless an explicit diagnostic endpoint is supplied. It sends
an available credential but does not require one. A 200 response is recognized
when it is a JSON object with `object: "list"` and an array-valued `data`;
empty arrays are accepted. A 401 means rejection; other responses do not
establish acceptance. Recognition uses the same bounded response handling as
the key probe. Acceptance does not establish that authentication is enforced or
that inference is authorized.
Diagnostic HTTP evidence describes the last request observed at Promptkit's Diagnostic HTTP evidence describes the last request observed at Promptkit's
RoundTripper boundary and any response associated with it. It includes the RoundTripper boundary and any response associated with it. It includes the
selected destination without URL user information, response status, and selected destination without URL user information, response status, and
@@ -154,7 +163,22 @@ data, a second JSON value, absent choices, missing content, `null` content,
non-string content, and size overflow are malformed responses and return no non-string content, and size overflow are malformed responses and return no
partial result. partial result.
Each of the five token fields retains whether the provider reported a For OpenRouter and vLLM, the client also reads reasoning tokens from
`usage.completion_tokens_details.reasoning_tokens`. OpenRouter cache writes
come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
`usage.prompt_tokens_details.created_cache_tokens`. The generic provider keeps
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
fractional, or unrepresentable new details remain unknown without invalidating
otherwise usable content. Existing common count decoding remains strict.
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
amounts are ignored without invalidating content. The generic and vLLM
providers do not interpret that field.
vLLM prompt-token details depend on server configuration, including
`--enable-prompt-tokens-details`; reasoning details depend on a configured
reasoning parser. Servers without these details remain usable. vLLM cache
writes describe local prefix-cache creation, not a billable write charge.
Each common token field retains whether the provider reported a
non-null integer, including zero. Omitted and `null` fields are unknown for non-null integer, including zero. Omitted and `null` fields are unknown for
observations. A strictly decoded successful envelope can yield observed usage observations. A strictly decoded successful envelope can yield observed usage
even when a later missing choice or content makes the attempt malformed. even when a later missing choice or content makes the attempt malformed.

View File

@@ -79,7 +79,7 @@ other backend IDs.
The coordinator does not retry injected clients; it normalizes a nil successful The coordinator does not retry injected clients; it normalizes a nil successful
response as malformed generation. The built-in HTTP client remains responsible response as malformed generation. The built-in HTTP client remains responsible
for provider transport behavior. The runner, rather than the for provider transport behavior. The runner, rather than the
capacity layer, sums all five usage fields from the initial response and every capacity layer, sums usage fields from the initial response and every
completed repair response into the successful run result. completed repair response into the successful run result.
An invocation-local observer receives attempt events after active permit An invocation-local observer receives attempt events after active permit

View File

@@ -35,6 +35,12 @@ unknown usage. Transport, timeout, malformed, and noneligible status failures
stop the sequence without retry exhaustion. Context is checked before each stop the sequence without retry exhaustion. Context is checked before each
wait, acquisition, and dispatch. wait, acquisition, and dispatch.
At every dispatched completion the coordinator resolves any reported cost
against the frozen backend zero-cost policy and adds it once to the run's
[cost accumulator](observation.md), independently of observer selection.
Injected responses may retain reported cost alongside an error. Undispatched
acquisitions and retry waits contribute no cost.
The public client adapter translates explicitly supplied public transport and The public client adapter translates explicitly supplied public transport and
malformed-response identities into internal categories while retaining the malformed-response identities into internal categories while retaining the
original error chain. This keeps injected attempt classifications consistent original error chain. This keeps injected attempt classifications consistent

View File

@@ -142,6 +142,14 @@ known. Strictly decoded usage can still be observed if later choice/content
checks fail; invalid framing or a non-success error body contributes none. checks fail; invalid framing or a non-success error body contributes none.
Existing successful-result accounting remains value-based. Existing successful-result accounting remains value-based.
The captured provider selects additional response details and authentication
diagnostic defaults. Optional new reasoning, cache-write, and OpenRouter cost
fields are decoded independently, without relaxing common-field or framing
validation. Cost remains an optional raw amount at this boundary; shared
accounting applies configured-zero fallback after dispatch. The integration
contract specifies the [wire mappings](../integrations/openai-compatible-chat.md#response-handling)
and [probe recognition](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
For a non-success response, `ProviderHTTPError` retains the HTTP status and For a non-success response, `ProviderHTTPError` retains the HTTP status and
only normalized detail from the bounded recognized envelope. It retains only normalized detail from the bounded recognized envelope. It retains
`ErrUnexpectedStatus` through unwrapping. The client owns response closure; `ErrUnexpectedStatus` through unwrapping. The client owns response closure;

View File

@@ -18,11 +18,14 @@ Timeout attribution is finalized and the child scope released before terminal
delivery, so observer latency cannot change the already selected outcome. delivery, so observer latency cannot change the already selected outcome.
Prepared-content notification is a separate opt-in callback, not an event. Prepared-content notification is a separate opt-in callback, not an event.
The facade masks inherited recorder and prepared-claim state when entering The facade masks inherited recorder, cost, and prepared-claim state when entering
preparation or execution. Delegated calls from an injected collaborator retain preparation or execution. Delegated calls from an injected collaborator retain
the caller's cancellation, deadline, and unrelated context values, but do not the caller's cancellation, deadline, and unrelated context values, but do not
reuse the enclosing invocation's accounting. Only the new request's observer reuse the enclosing invocation's accounting. Ordinary runs compose the request
(or the claimed handle's retained observer) enables its observation lifecycle. observer before any observers selected by the invocation context; contextual
selection survives this boundary. Prepared executions compose the retained
request observer with selection from the execution context after a successful
claim; the preparation context does not select execution observers.
A recorder keeps only counters, resolved identity fields, and the selected A recorder keeps only counters, resolved identity fields, and the selected
callback. It has no global history, queue, goroutine, or mutex held across a callback. It has no global history, queue, goroutine, or mutex held across a
@@ -53,12 +56,24 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its one dispatch. A terminal event is an aggregate and must not be added to its
attempt events. attempt events.
An execution group is a zero-ready synchronized collector of explicitly selected Ordinary runs and claimed prepared executions install invocation-local cost
accounting even without an observer.
The generation coordinator resolves cost once per dispatched attempt and adds
it to that state independently of event delivery. The result and recorder read
the same accumulator; groups merge first terminal costs independently of
terminal coverage.
Reasoning counts use the same per-field rules and are a subset of completion
tokens. The public cache-hit helper derives a ratio from complete, consistent
summed input and cache-read counts; it does not aggregate ratios or use writes.
An execution group is a zero-ready synchronized collector of selected
events. It enrolls by run ID and aggregates each run's first terminal counters and events. It enrolls by run ID and aggregates each run's first terminal counters and
presence-aware usage exactly once. Snapshots copy members and totals under one presence-aware usage exactly once. Snapshots copy members and totals under one
lock; terminal coverage and token completeness are distinct. Observer fan-out lock; terminal coverage and token completeness are distinct. Observer fan-out
copies the callback slice and invokes callbacks synchronously without holding a copies the callback slice and invokes callbacks synchronously without holding a
group lock. No group history, registry, or automatic nested membership exists. group lock. No group history or registry exists; contextual selection can enroll
nested ordinary runs that use the scoped context.
Smoke diagnostics mask inherited execution-recorder state while invoking Smoke diagnostics mask inherited execution-recorder state while invoking
generation or HTTP collaborators. Their private attempt evidence is assembled generation or HTTP collaborators. Their private attempt evidence is assembled

View File

@@ -21,7 +21,7 @@ contributor workflow and validation.
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) | | `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) | | `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) | | `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) | | `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded retries, per-attempt cost, ordinary observations, and separate diagnostic evidence. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
@@ -30,8 +30,8 @@ contributor workflow and validation.
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | | `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible requests, provider-specific response interpretation and authentication probes, bounded decoding, deadlines, and diagnostic evidence. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) | | `internal/observation` | Owns invocation-local run IDs, presence-aware token and cost accounting, contextual observer selection, synchronous event delivery, and terminal-based group aggregation; smoke diagnostics retain separate evidence. | [Internal execution observation](observation.md) |
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | | `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The internal engine assembles these internal components without exposing their The internal engine assembles these internal components without exposing their

View File

@@ -216,10 +216,16 @@ ordinary `Run` reuses its run ID and carries it to the generation coordinator
through context. Prompt and profile identities are enriched only after their through context. Prompt and profile identities are enriched only after their
respective resolutions succeed, before admission. The facade starts observation respective resolutions succeed, before admission. The facade starts observation
before request conversion and emits its terminal event after public error before request conversion and emits its terminal event after public error
mapping and after the runner's deferred admission release. Preparation and mapping and after the runner's deferred admission release. Preparation emits no
prepared execution follows the same terminal mapping after a successful claim; events; prepared execution follows the same terminal mapping after a successful claim;
invalid or discarded handles do not acquire an observer. invalid or discarded handles do not acquire an observer.
Ordinary runs establish fresh private cost state before observation; a claimed
prepared execution establishes its own state after claim. Contextual observers
remain available at public boundaries, while enclosing recorder and cost state
are masked. Prepared handles retain the request observer and frozen provider
policy, not preparation-context selection or an accumulator.
The facade establishes an optional operation scope for ordinary runs before The facade establishes an optional operation scope for ordinary runs before
observation and preparation. A claimed prepared execution establishes its own observation and preparation. A claimed prepared execution establishes its own
scope from the retained duration; preparation and idle time do not consume it. scope from the retained duration; preparation and idle time do not consume it.
@@ -273,11 +279,13 @@ run identifier, and UTC timing. The same effective session reaches initial
generation and any repair attempt through the rendered prompt. The same generation and any repair attempt through the rendered prompt. The same
effective target and presence metadata, including backend identity and direct effective target and presence metadata, including backend identity and direct
credential during execution, reaches generation and every repair attempt. credential during execution, reaches generation and every repair attempt.
Result usage is the field-wise sum of all five usage values from the initial Result usage is the field-wise sum of usage values from the initial
response and every completed repair response. Final raw output, artifact, and response and every completed repair response. Final raw output, artifact, and
validation state still come from the last candidate. A repair error returns no validation state still come from the last candidate. A repair error returns no
partial run result or partial result usage, although the invocation-local partial run result or partial result usage, although the invocation-local
observer retains completed-attempt usage and reports it in the terminal event. observer retains completed-attempt usage and reports it in the terminal event.
Result cost instead snapshots the shared accumulator across all dispatched
attempts, including retries and repairs, whether or not observation is enabled.
## Failure Categories ## Failure Categories

88
docs/releases/v0.25.0.md Normal file
View File

@@ -0,0 +1,88 @@
# Promptkit v0.25.0
This minor release adds invocation-scoped execution observation, provider-specific
reasoning and cache usage, and optional USD cost reporting. Backend definitions
can now select API conventions independently of their names and endpoints.
Consumers upgrading from v0.24.x should review the configuration and accounting
changes below.
## Added And Changed
- `WithExecutionObserver` selects observation through a caller-owned context
across nested, concurrent, coordinated, and multi-engine calls. Request
observers run before contextual observers. Prepared handles retain their
explicit request observer but select contextual observers when executed,
not when prepared. Each execution retains independent accounting; groups
count each run's first terminal once. See the
[observation guide](../consumers/pkg-promptkit.md#observe-an-ordinary-run)
and [offline example](../../examples/go-library/observe/main.go).
- Backend `provider` selects `openai-compatible`, `openrouter`, or `vllm`
conventions. Omitted, null, or blank selects the generic OpenAI-compatible
behavior. The bundled OpenRouter backend explicitly selects `openrouter`.
`InspectBackend` exposes declared provider and `zero_cost` settings without
manufacturing runtime defaults. See the
[backend reference](../formats.md#backend-definitions).
- Usage values, presence indicators, observations, and group summaries include
`ReasoningTokens`. OpenRouter and vLLM decoding includes their respective
reasoning, cache-read, and cache-write fields. Reasoning is a subset of
completion tokens; cache activity does not add to reported token totals.
`ObservedTokenUsage.CacheHitRatio` derives a ratio from complete, consistent
input and cache-read totals. See the
[response contract](../integrations/openai-compatible-chat.md#response-handling).
- `RunResult.CostUSD`, execution events, and group summaries report a USD
subtotal with availability, completeness, and source metadata. OpenRouter
supplies optional `usage.cost`; injected clients can supply
`GenerateResponse.CostUSD`. Explicit backend `zero_cost: true` supplies a
zero fallback when no usable amount is reported. A reported amount, including
zero, takes precedence. See the [cost GoDoc](../../cost.go).
- Authentication diagnostics can use provider defaults: `openrouter_key` for
OpenRouter and the new `vllm_models` adapter for vLLM. The latter probes
base-relative `/models` and reports recognized response facts without
asserting authentication enforcement or inference authorization. See the
[diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
## Compatibility And Upgrade
- For application-defined OpenRouter backends, set `provider: openrouter` to
enable OpenRouter-specific usage and cost decoding and its authentication
default. Backend IDs and endpoint hostnames do not select conventions.
Generic backends retain their existing request and token conventions and
do not decode monetary cost. Unsupported nonblank provider names are
configuration errors; supplied names are trimmed and lowercased.
- OpenRouter cache-write reporting now reads
`usage.prompt_tokens_details.cache_write_tokens`, not the generic top-level
`usage.cache_write_tokens` field. vLLM uses
`usage.prompt_tokens_details.created_cache_tokens`. Missing or unusable new
details remain unknown rather than invalidating otherwise usable content.
Availability depends on the provider and server settings; consult the
[response contract](../integrations/openai-compatible-chat.md#response-handling).
- A supplied authentication object no longer requires `kind`: an omitted or
null kind uses the provider default. Clearing the authentication block also
restores that default. Set `kind: none` to declare authentication testing
inapplicable. The generic provider has no authentication default; explicit
adapters remain supported independently of provider selection. Probes still
run only when explicitly requested. See the
[backend reference](../formats.md#backend-definitions).
- Cost includes all dispatched attempts, including transient retries and
structural repairs, and is available on successful results without an
observer. On operational failure, observations can retain cost even though
no partial result is returned. Existing `RunResult.Usage` continues to sum
successful generation responses; presence-aware observations cover all
dispatched attempts. Do not add terminal totals to attempt totals.
- Unknown cost is distinct from reported or configured zero. No backend is
automatically declared zero-cost, and no token-rate estimation is performed.
Group cost completeness describes included terminals, independently of
terminal coverage. Consumers own presentation and interpretation; complete
reporting does not verify billing.
- Result JSON adds `cost_usd`, and token JSON adds `reasoning_tokens`.
Update strict decoders if necessary. Older result JSON without cost decodes
to unknown cost. Consumers using positional literals for expanded public
structs may need to update them. See the [public types](../../types.go).
The Go consumer minimum remains 1.27.0; dependencies are unchanged from v0.24.0.
```sh
go get go.maximumdirect.net/promptkit@v0.25.0
```
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/docs/oss/attribution.md)

View File

@@ -535,7 +535,8 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr
// returns no handle on error. A nil Engine returns an error matching // returns no handle on error. A nil Engine returns an error matching
// ErrInvalidConfig. The effective transient retry budget is frozen privately // ErrInvalidConfig. The effective transient retry budget is frozen privately
// for RunPrepared and is absent from Details. A supplied Observer is retained // for RunPrepared and is absent from Details. A supplied Observer is retained
// privately for RunPrepared; preparation itself emits no events. // privately for RunPrepared; preparation itself emits no events and does not
// retain contextual observers.
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without // RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
// starting a timer or invoking the notification during preparation. // starting a timer or invoking the notification during preparation.
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) { func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
@@ -602,16 +603,24 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
// result on error. // result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
ctx = observation.WithoutExecutionState(ctx) ctx = observation.WithoutExecutionState(ctx)
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := usecase.NewOperationScope(ctx, req.OperationTimeout) scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
defer scope.Close() defer scope.Close()
ctx = scope.Context ctx = scope.Context
if req.Observer == nil { var explicit domain.ExecutionObserver
if req.Observer != nil {
explicit = func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) }
}
observer := observation.Compose(explicit, observation.ObserverFromContext(ctx))
if observer == nil {
result, err := e.run(ctx, req) result, err := e.run(ctx, req)
err = mapPublicError(scope.Finalize(err)) err = mapPublicError(scope.Finalize(err))
scope.Close() scope.Close()
return result, err return result, err
} }
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) }) recorder, idErr := observation.New(observer)
recorder.SetCostAccumulator(cost)
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil { if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
recorder.SetParentRunID(req.ContinueFrom.internal.RunID()) recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
} }
@@ -682,10 +691,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
// content-validation rejection, including repair exhaustion, is returned in // content-validation rejection, including repair exhaustion, is returned in
// RunResult, not as an operational // RunResult, not as an operational
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An // error. RunPrepared uses the retry choice frozen during PrepareExecution. An
// operational error returns no partial RunResult. A retained Observer begins // operational error returns no partial RunResult. The retained Observer and
// only after a successful atomic claim, with identities from the frozen // execution-context observers begin only after a successful atomic claim, with
// snapshot, and receives a terminal event on normal return after public error // identities from the frozen snapshot, and receive a terminal event on normal
// mapping. Invalid, foreign, discarded, or reused handles emit no events. // return after public error mapping. Invalid, foreign, discarded, or reused
// handles emit no events.
// Callbacks run outside the handle and capacity locks; panics propagate and // Callbacks run outside the handle and capacity locks; panics propagate and
// do not promise a terminal event. // do not promise a terminal event.
// The retained RunRequest.OperationTimeout starts after claim; the retained // The retained RunRequest.OperationTimeout starts after claim; the retained

View File

@@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error)
func modelResponse(content string, promptTokens, completionTokens int) *http.Response { func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
body, _ := json.Marshal(map[string]any{ body, _ := json.Marshal(map[string]any{
"choices": []any{map[string]any{"message": map[string]string{"content": content}}}, "choices": []any{map[string]any{"message": map[string]string{"content": content}}},
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens}, "usage": map[string]any{
"prompt_tokens": promptTokens, "completion_tokens": completionTokens,
"prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1},
"completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01,
},
}) })
return httpResponse(http.StatusOK, string(body)) return httpResponse(http.StatusOK, string(body))
} }
@@ -80,6 +84,10 @@ type summary struct {
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"` ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"` GroupMembers int `json:"group_members"`
GroupAttempts uint64 `json:"group_attempts"` GroupAttempts uint64 `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"` PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"` PreparedParentMatches bool `json:"prepared_parent_matches"`
} }
@@ -97,6 +105,8 @@ func run() (summary, error) {
filesystem := configkit.NewFilesystem("example schemas", assets) filesystem := configkit.NewFilesystem("example schemas", assets)
backend := promptkit.LocalBackend("https://example.invalid/v1", 1) backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
backend.ID = backendID backend.ID = backendID
provider, zeroCost := "openrouter", true
backend.Provider, backend.ZeroCost = &provider, &zeroCost
inputs = append(inputs, inputs = append(inputs,
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}}, configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{ promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
@@ -117,8 +127,9 @@ func run() (summary, error) {
var proposalLog eventLog var proposalLog eventLog
var executions promptkit.ExecutionGroup var executions promptkit.ExecutionGroup
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{ ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe)
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe), proposal, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record,
}) })
if err != nil { if err != nil {
return summary{}, err return summary{}, err
@@ -133,8 +144,8 @@ func run() (summary, error) {
var assessmentLog eventLog var assessmentLog eventLog
var preparedDetails promptkit.PreparedRun var preparedDetails promptkit.PreparedRun
decision := "rejected" decision := "rejected"
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{ coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe, Start: proposal.Exchange, AdditionalTurns: 1,
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
if turn.Result == nil { if turn.Result == nil {
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
@@ -142,7 +153,7 @@ func run() (summary, error) {
if !turn.Result.Validation.IsValid { if !turn.Result.Validation.IsValid {
return nil, fmt.Errorf("follow-up failed structural validation") return nil, fmt.Errorf("follow-up failed structural validation")
} }
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)}) assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -181,6 +192,8 @@ func run() (summary, error) {
return summary{}, err return summary{}, err
} }
group := executions.Snapshot() group := executions.Snapshot()
_, ratioAvailable := group.Usage.CacheHitRatio()
assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio()
return summary{ return summary{
ProposalAttempts: proposalTerminal.Attempts, ProposalAttempts: proposalTerminal.Attempts,
ProposalTransientRetries: proposalTerminal.TransientRetries, ProposalTransientRetries: proposalTerminal.TransientRetries,
@@ -192,6 +205,10 @@ func run() (summary, error) {
ExhaustedOutcome: exhausted.Outcome, ExhaustedOutcome: exhausted.Outcome,
GroupMembers: len(group.Members), GroupMembers: len(group.Members),
GroupAttempts: group.Attempts, GroupAttempts: group.Attempts,
GroupCost: group.CostUSD,
GroupReasoning: group.Usage.ReasoningTokens,
GroupCacheRatioAvailable: ratioAvailable,
AssessmentCacheRatio: assessmentRatio,
PreparedMessages: len(preparedDetails.Messages), PreparedMessages: len(preparedDetails.Messages),
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID, PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
}, nil }, nil

View File

@@ -3,7 +3,10 @@ package main
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"math"
"testing" "testing"
"go.maximumdirect.net/promptkit"
) )
func TestObserveExample(t *testing.T) { func TestObserveExample(t *testing.T) {
@@ -22,6 +25,10 @@ func TestObserveExample(t *testing.T) {
ExhaustedOutcome string `json:"exhausted_outcome"` ExhaustedOutcome string `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"` GroupMembers int `json:"group_members"`
GroupAttempts int `json:"group_attempts"` GroupAttempts int `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"` PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"` PreparedParentMatches bool `json:"prepared_parent_matches"`
} }
@@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) {
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches { got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
t.Fatalf("unexpected observation summary: %+v", got) t.Fatalf("unexpected observation summary: %+v", got)
} }
if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 {
t.Fatalf("unexpected usage summary: %+v", got)
}
} }

View File

@@ -5,7 +5,7 @@ import (
"go.maximumdirect.net/promptkit/internal/observation" "go.maximumdirect.net/promptkit/internal/observation"
) )
// ExecutionGroup collects facts from explicitly selected execution events. // ExecutionGroup collects facts from selected execution events.
// Its zero value is ready for concurrent use. Do not copy it after first use. // Its zero value is ready for concurrent use. Do not copy it after first use.
type ExecutionGroup struct{ group observation.Group } type ExecutionGroup struct{ group observation.Group }
@@ -21,7 +21,8 @@ type ExecutionGroupMember struct {
// member has a terminal, independently of usage completeness. An empty group // member has a terminal, independently of usage completeness. An empty group
// has complete coverage and known complete zero usage; a group with members // has complete coverage and known complete zero usage; a group with members
// but no terminals has unknown, incomplete usage. Overlapping groups are not // but no terminals has unknown, incomplete usage. Overlapping groups are not
// generally additive. // generally additive. CostUSD covers included terminals independently of
// terminal coverage, using the same known, complete, and source merge rules.
type ExecutionGroupSummary struct { type ExecutionGroupSummary struct {
Members []ExecutionGroupMember `json:"members"` Members []ExecutionGroupMember `json:"members"`
TerminalCoverageComplete bool `json:"terminal_coverage_complete"` TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
@@ -29,6 +30,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64 `json:"transient_retries"` TransientRetries uint64 `json:"transient_retries"`
RepairCalls uint64 `json:"repair_calls"` RepairCalls uint64 `json:"repair_calls"`
Usage ObservedTokenUsage `json:"usage"` Usage ObservedTokenUsage `json:"usage"`
CostUSD ObservedCost `json:"cost_usd"`
} }
// Observe enrolls an event's run, counting its first terminal event only. // Observe enrolls an event's run, counting its first terminal event only.
@@ -36,8 +38,8 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) {
count := func(c ObservedTokenCount) domain.ObservedTokenCount { count := func(c ObservedTokenCount) domain.ObservedTokenCount {
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete} return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
} }
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, CostUSD: toDomainCost(event.CostUSD),
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}}) Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
} }
// Snapshot returns an internally consistent, caller-owned summary. Coverage // Snapshot returns an internally consistent, caller-owned summary. Coverage
@@ -51,6 +53,6 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
for i, member := range value.Members { for i, member := range value.Members {
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved} members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
} }
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, CostUSD: fromDomainCost(value.CostUSD),
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}} Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
} }

View File

@@ -1,5 +1,6 @@
backends: backends:
- id: openrouter - id: openrouter
provider: openrouter
endpoint: https://openrouter.ai/api/v1 endpoint: https://openrouter.ai/api/v1
api_key_env: OPENROUTER_API_KEY api_key_env: OPENROUTER_API_KEY
smoke_tests: smoke_tests:

View File

@@ -17,7 +17,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
backend, err := state.GetBackend("openrouter") backend, err := state.GetBackend("openrouter")
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" { if err != nil || backend.Provider != "openrouter" || backend.ZeroCost || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
t.Fatalf("backend = %+v, %v", backend, err) t.Fatalf("backend = %+v, %v", backend, err)
} }
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 { if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
@@ -40,6 +40,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
t.Fatalf("profile %s = %+v, %v", id, profile, err) t.Fatalf("profile %s = %+v, %v", id, profile, err)
} }
want := defaults.ExecutionTargetDefault() want := defaults.ExecutionTargetDefault()
want.Provider = "openrouter"
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY" want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
if tier, ok := strings.CutPrefix(id, "auto-"); ok { if tier, ok := strings.CutPrefix(id, "auto-"); ok {
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}} want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}

View File

@@ -59,6 +59,35 @@ func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
} }
} }
func TestProviderPolicyCaptureAndInspection(t *testing.T) {
state, err := load(mapping("provider policy", map[string]any{
"collections": map[string]any{"team": map[string]any{
"backends": []any{map[string]any{"id": "renamed", "endpoint": "https://example.invalid/v1", "provider": "OPENROUTER", "zero_cost": true}, map[string]any{"id": "child", "extends": map[string]any{"id": "renamed"}, "zero_cost": false}},
"profiles": []any{map[string]any{"id": "selected", "backend": "team/child", "endpoint": "https://other.invalid/v1", "model": "m"}},
}},
"profiles": []any{map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1", "model": "m"}},
}), nil)
if err != nil {
t.Fatal(err)
}
backend, err := state.GetBackend("team/child")
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost {
t.Fatalf("backend: %+v, %v", backend, err)
}
profile, err := state.GetProfile(context.Background(), "team/selected")
if err != nil || profile.ResolvedTarget.Provider != "openrouter" || profile.ResolvedTarget.ZeroCost || profile.ResolvedTarget.Endpoint != "https://other.invalid/v1" {
t.Fatalf("profile: %+v, %v", profile, err)
}
profile, err = state.GetProfile(context.Background(), "generic")
if err != nil || profile.ResolvedTarget.Provider != "openai-compatible" || profile.ResolvedTarget.ZeroCost {
t.Fatalf("generic: %+v, %v", profile, err)
}
resource, err := state.InspectBackend(context.Background(), "team/child")
if err != nil || resource.Provider == nil || *resource.Provider != "openrouter" || resource.ZeroCost == nil || *resource.ZeroCost {
t.Fatalf("inspection: %+v, %v", resource, err)
}
}
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) { func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
definition := map[string]any{ definition := map[string]any{
"id": "example", "inputs": []any{map[string]any{"name": " document "}}, "id": "example", "inputs": []any{map[string]any{"name": " document "}},

View File

@@ -291,6 +291,9 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
if backend.Endpoint == nil || *backend.Endpoint == "" { if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required") validation.Error(base.Key("endpoint"), "endpoint is required")
} }
if backend.Provider != nil && !llm.SupportedProvider(*backend.Provider) {
validation.Error(base.Key("provider"), "unsupported provider")
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) { if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid") validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
} }
@@ -319,10 +322,8 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
} }
if authentication := smokeTests.Authentication; authentication != nil { if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind") kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil { if authentication.Kind != nil && *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" && *authentication.Kind != "vllm_models" {
validation.Error(kindPath, "authentication kind is required") validation.Error(kindPath, "unsupported authentication kind")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
} }
} }
} }

View File

@@ -532,6 +532,54 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
} }
} }
func TestBackendProviderFinalConfiguration(t *testing.T) {
underlays := []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": "bad", "zero_cost": true}},
})}}}
main := mapping("override", map[string]any{"backends": []any{
map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": " OPENROUTER ", "zero_cost": false, "smoke_tests": map[string]any{"authentication": map[string]any{}}},
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}},
map[string]any{"id": "cleared", "extends": map[string]any{"id": "base"}, "provider": nil, "zero_cost": nil},
map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1"},
}})
state, err := load(main, underlays)
if err != nil {
var detail *configkit.LoadError
if errors.As(err, &detail) {
t.Fatalf("%v: %+v", err, detail.Diagnostics())
}
t.Fatal(err)
}
for _, name := range []string{"base", "child"} {
b := state.backendRecords[name]
if b.Provider == nil || *b.Provider != "openrouter" || b.ZeroCost == nil || *b.ZeroCost {
t.Fatalf("%s: %+v", name, b)
}
}
for _, name := range []string{"cleared", "generic"} {
b := state.backendRecords[name]
if b.Provider != nil || b.ZeroCost != nil {
t.Fatalf("%s: %+v", name, b)
}
}
field, found := state.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("provider"))
if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("provider normalization: %+v", field)
}
for _, provider := range []string{"openai-compatible", "openrouter", "vllm", " ", ""} {
_, err := load(mapping("provider", map[string]any{"backends": []any{map[string]any{"id": "x", "endpoint": "https://example.invalid/v1", "provider": provider, "smoke_tests": map[string]any{"authentication": map[string]any{"kind": " VLLM_MODELS "}}}}}), nil)
if err != nil {
t.Errorf("provider %q: %v", provider, err)
}
}
for _, fields := range []map[string]any{{"provider": "unsupported"}, {"zero_cost": "false"}} {
fields["id"], fields["endpoint"] = "x", "https://example.invalid/v1"
if _, err := load(mapping("invalid", map[string]any{"backends": []any{fields}}), nil); err == nil {
t.Fatalf("accepted invalid fields: %v", fields)
}
}
}
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) { func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
for _, test := range []struct { for _, test := range []struct {
name string name string
@@ -638,11 +686,6 @@ func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
data: map[string]any{"default_profile": " \t "}, data: map[string]any{"default_profile": " \t "},
path: "/promptkit/backends/0/smoke_tests/default_profile", path: "/promptkit/backends/0/smoke_tests/default_profile",
}, },
{
name: "authentication kind required",
data: map[string]any{"authentication": map[string]any{}},
path: "/promptkit/backends/0/smoke_tests/authentication/kind",
},
{ {
name: "unknown authentication kind", name: "unknown authentication kind",
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}}, data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},

View File

@@ -21,6 +21,8 @@ func copyProviderMap(value map[string]any) map[string]any {
} }
func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings { func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings {
in.Provider = copyPointer(in.Provider)
in.ZeroCost = copyPointer(in.ZeroCost)
in.Endpoint = copyPointer(in.Endpoint) in.Endpoint = copyPointer(in.Endpoint)
in.APIKeyEnv = copyPointer(in.APIKeyEnv) in.APIKeyEnv = copyPointer(in.APIKeyEnv)
in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit) in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit)

View File

@@ -15,6 +15,9 @@ func resolveBackends(entries []backendConfig, base configkit.Path, resolution *c
for i := range entries { for i := range entries {
entry := &entries[i] entry := &entries[i]
path := base.Index(i) path := base.Index(i)
if entry.Provider != nil {
recordStringNormalization(&entry.Provider, path.Key("provider"), resolution, strings.ToLower(strings.TrimSpace(*entry.Provider)))
}
if entry.Endpoint != nil { if entry.Endpoint != nil {
if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil { if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil {
recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized) recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized)

View File

@@ -26,6 +26,8 @@ type ProfileResource struct {
// Its fields borrow immutable configuration captured by the adapter. // Its fields borrow immutable configuration captured by the adapter.
type BackendResource struct { type BackendResource struct {
ID, Endpoint string ID, Endpoint string
Provider *string
ZeroCost *bool
APIKeyEnv *string APIKeyEnv *string
ConcurrencyLimit, QueueCapacity *int ConcurrencyLimit, QueueCapacity *int
ExtraParams map[string]any ExtraParams map[string]any
@@ -131,7 +133,7 @@ func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendRes
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id) return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
} }
return &BackendResource{ return &BackendResource{
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, Provider: config.Provider, ZeroCost: config.ZeroCost,
ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity, ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity,
ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests, ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests,
}, nil }, nil

View File

@@ -8,6 +8,7 @@ import (
"go.maximumdirect.net/promptkit/internal/defaults" "go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain" "go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue" "go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm"
profilepkg "go.maximumdirect.net/promptkit/internal/profile" profilepkg "go.maximumdirect.net/promptkit/internal/profile"
"go.maximumdirect.net/promptkit/internal/settings" "go.maximumdirect.net/promptkit/internal/settings"
) )
@@ -36,7 +37,7 @@ func (state *loaded) buildExecutionProfiles(config *settings.Settings) error {
} }
func captureBackend(id string, config backendConfig) domain.Backend { func captureBackend(id string, config backendConfig) domain.Backend {
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams} definition := domain.Backend{ID: id, Provider: llm.EffectiveProvider(valueOrZero(config.Provider)), ZeroCost: valueOrZero(config.ZeroCost), ExtraParams: config.ExtraParams}
if config.SmokeTests != nil { if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile} smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil { if config.SmokeTests.Reachability != nil {
@@ -67,6 +68,7 @@ func captureBackend(id string, config backendConfig) domain.Backend {
func captureExecutionProfile(id string, config profileConfig, backends map[string]domain.Backend) (domain.ExecutionProfile, error) { func captureExecutionProfile(id string, config profileConfig, backends map[string]domain.Backend) (domain.ExecutionProfile, error) {
target := defaults.ExecutionTargetDefault() target := defaults.ExecutionTargetDefault()
target.Provider = llm.EffectiveProvider("")
if config.Backend != nil && *config.Backend != "" { if config.Backend != nil && *config.Backend != "" {
// These defaults are capture-owned and immutable. Public and runtime // These defaults are capture-owned and immutable. Public and runtime
// lookups still detach mutable values; profile overlays copy before merging. // lookups still detach mutable values; profile overlays copy before merging.
@@ -75,6 +77,8 @@ func captureExecutionProfile(id string, config profileConfig, backends map[strin
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w: %q", id, backend.ErrBackendNotFound, *config.Backend) return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w: %q", id, backend.ErrBackendNotFound, *config.Backend)
} }
target.BackendID = selected.ID target.BackendID = selected.ID
target.Provider = selected.Provider
target.ZeroCost = selected.ZeroCost
target.Endpoint = selected.Endpoint target.Endpoint = selected.Endpoint
target.APIKeyEnv = selected.APIKeyEnv target.APIKeyEnv = selected.APIKeyEnv
target.ExtraParams = selected.ExtraParams target.ExtraParams = selected.ExtraParams

17
internal/domain/cost.go Normal file
View File

@@ -0,0 +1,17 @@
package domain
type CostSource string
const (
CostSourceNone CostSource = ""
CostSourceProvider CostSource = "provider"
CostSourceConfiguredZero CostSource = "configured_zero"
CostSourceMixed CostSource = "mixed"
)
type ObservedCost struct {
Value float64
Known bool
Complete bool
Source CostSource
}

View File

@@ -99,6 +99,7 @@ type RunResult struct {
EffectiveModelParams ExecutionTarget EffectiveModelParams ExecutionTarget
InputHashes map[string]string InputHashes map[string]string
Usage TokenUsage Usage TokenUsage
CostUSD ObservedCost
StartTime time.Time StartTime time.Time
EndTime time.Time EndTime time.Time
Duration time.Duration Duration time.Duration
@@ -188,6 +189,8 @@ type PromptMessageTemplate struct {
// diagnostic metadata. // diagnostic metadata.
type Backend struct { type Backend struct {
ID string ID string
Provider string
ZeroCost bool
Endpoint string Endpoint string
APIKeyEnv string APIKeyEnv string
ExtraParams map[string]any ExtraParams map[string]any
@@ -236,6 +239,8 @@ type ExecutionTargetPresence struct {
// ExecutionTarget represents effective model runtime settings for a run. // ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct { type ExecutionTarget struct {
Provider string `yaml:"-" json:"-"`
ZeroCost bool `yaml:"-" json:"-"`
BackendID string `yaml:"backend" json:"backend_id,omitempty"` BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"` Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"` Model string `yaml:"model" json:"model"`
@@ -324,12 +329,14 @@ type GenerateResponse struct {
Content string Content string
Usage TokenUsage Usage TokenUsage
UsagePresence *TokenUsagePresence UsagePresence *TokenUsagePresence
CostUSD *float64
} }
// TokenUsagePresence records which token fields a provider reported. // TokenUsagePresence records which token fields a provider reported.
type TokenUsagePresence struct { type TokenUsagePresence struct {
PromptTokens bool PromptTokens bool
CompletionTokens bool CompletionTokens bool
ReasoningTokens bool
TotalTokens bool TotalTokens bool
CachedTokens bool CachedTokens bool
CacheWriteTokens bool CacheWriteTokens bool
@@ -339,6 +346,7 @@ type TokenUsagePresence struct {
type TokenUsage struct { type TokenUsage struct {
PromptTokens int PromptTokens int
CompletionTokens int CompletionTokens int
ReasoningTokens int
TotalTokens int TotalTokens int
CachedTokens int CachedTokens int
CacheWriteTokens int CacheWriteTokens int

View File

@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct { type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount CacheWriteTokens ObservedTokenCount
@@ -75,6 +76,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64 TransientRetries uint64
RepairCalls uint64 RepairCalls uint64
Usage ObservedTokenUsage Usage ObservedTokenUsage
CostUSD ObservedCost
} }
// ExecutionEvent contains metadata only; it carries no mutable containers or // ExecutionEvent contains metadata only; it carries no mutable containers or
@@ -100,6 +102,7 @@ type ExecutionEvent struct {
StatusCode int StatusCode int
RetriesExhausted bool RetriesExhausted bool
Usage ObservedTokenUsage Usage ObservedTokenUsage
CostUSD ObservedCost
Attempts uint64 Attempts uint64
TransientRetries uint64 TransientRetries uint64
RepairCalls uint64 RepairCalls uint64

View File

@@ -130,7 +130,7 @@ func (c *Coordinator) generate(
if result.Dispatched { if result.Dispatched {
attempts++ attempts++
} }
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal) recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal)
if result.Outcome.Err == nil { if result.Outcome.Err == nil {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
if captureEvidence { if captureEvidence {
@@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
generation = 1 generation = 1
} }
result := c.executeInjected(ctx, req) result := c.executeInjected(ctx, req)
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1) recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1)
if captureEvidence { if captureEvidence {
evidence.Usage = usage.Observed() evidence.Usage = usage.Observed()
} }
@@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe
requested := req.StructuredOutput != nil requested := req.StructuredOutput != nil
result.Outcome.StructuredOutputRequested = &requested result.Outcome.StructuredOutputRequested = &requested
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req) result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
if result.Outcome.Response != nil {
result.Outcome.CostUSD = result.Outcome.Response.CostUSD
}
result.RequestDuration = elapsed(start, c.now()) result.RequestDuration = elapsed(start, c.now())
if result.Outcome.Err == nil && result.Outcome.Response == nil { if result.Outcome.Err == nil && result.Outcome.Response == nil {
result.Outcome.Err = llm.ErrMalformedResponse result.Outcome.Err = llm.ErrMalformedResponse
@@ -240,8 +243,15 @@ type attemptResult struct {
// recordAttempt runs after the attempt has released its permit, including when // recordAttempt runs after the attempt has released its permit, including when
// the recorder calls application code. Injected clients expose no HTTP facts. // the recorder calls application code. Injected clients expose no HTTP facts.
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) { func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) {
outcome := result.Outcome outcome := result.Outcome
var resolved domain.ObservedCost
if result.Dispatched {
resolved = observation.ResolveCost(outcome.CostUSD, zeroCost)
if cost != nil {
cost.Add(resolved)
}
}
failure := ClassifyFailure(outcome.Err) failure := ClassifyFailure(outcome.Err)
if capture { if capture {
attempt := AttemptEvidence{ attempt := AttemptEvidence{
@@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal, recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration, Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode, Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}) Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved})
} }
} }

View File

@@ -58,6 +58,89 @@ func TestObservedRetryAndExhaustion(t *testing.T) {
} }
} }
func TestDispatchedAttemptCostsAreCollectedWithoutObserver(t *testing.T) {
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return response(503, `{"error":{}}`), nil
}
return response(200, `{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`), nil
})
var cost observation.CostAccumulator
ctx := observation.WithCostAccumulator(context.Background(), &cost)
req := testRequest(1)
req.Target.Provider = "openrouter"
if _, err := coordinator.Generate(ctx, req); err != nil {
t.Fatal(err)
}
got := cost.Observed()
if got.Value != .02 || !got.Known || got.Complete || got.Source != domain.CostSourceProvider {
t.Fatalf("cost=%+v", got)
}
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { return nil, errors.New("transport failed") })
var zero observation.CostAccumulator
req.Target.ZeroCost = true
if _, err := configured.Generate(observation.WithCostAccumulator(context.Background(), &zero), req); err == nil {
t.Fatal("expected error")
}
if got := zero.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
t.Fatalf("fallback=%+v", got)
}
}
func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) {
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
charge := .25
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{CostUSD: &charge}, err: errors.New("failed")})
var cost observation.CostAccumulator
var events []domain.ExecutionEvent
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
if err != nil {
t.Fatal(err)
}
recorder.SetCostAccumulator(&cost)
ctx := observation.WithRecorder(observation.WithCostAccumulator(context.Background(), &cost), recorder)
if _, err := coordinator.Generate(ctx, testRequest(0)); err == nil {
t.Fatal("expected error")
}
recorder.Finish(observation.FinishData{})
want := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
if len(events) != 2 || events[0].CostUSD != want || events[1].CostUSD != want || cost.Observed() != want {
t.Fatalf("events=%+v cost=%+v", events, cost.Observed())
}
}
func TestCanceledDispatchCostFallbackAndUndispatchedEmptyCost(t *testing.T) {
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
ctx, cancel := context.WithCancel(context.Background())
coordinator := NewInjected(configured.manager, cancelingCostClient{cancel: cancel})
var cost observation.CostAccumulator
req := testRequest(0)
req.Target.ZeroCost = true
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &cost), req); !errors.Is(err, context.Canceled) {
t.Fatalf("dispatch error=%v", err)
}
if got := cost.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
t.Fatalf("dispatched=%+v", got)
}
var untouched observation.CostAccumulator
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &untouched), req); !errors.Is(err, context.Canceled) {
t.Fatalf("pre-dispatch error=%v", err)
}
if got := untouched.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("undispatched=%+v", got)
}
}
type cancelingCostClient struct{ cancel context.CancelFunc }
func (c cancelingCostClient) Generate(ctx context.Context, _ domain.GenerateRequest) (*domain.GenerateResponse, error) {
c.cancel()
return nil, ctx.Err()
}
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) { func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil }) coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())

View File

@@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
// PreparedGeneration holds one encoded request and resolved authentication for // PreparedGeneration holds one encoded request and resolved authentication for
// repeated attempts. Each execution creates a fresh HTTP request and context. // repeated attempts. Each execution creates a fresh HTTP request and context.
type PreparedGeneration struct { type PreparedGeneration struct {
provider string
endpoint string endpoint string
payload []byte payload []byte
apiKey string apiKey string
@@ -104,6 +105,7 @@ type AttemptOutcome struct {
StatusCode int StatusCode int
Usage domain.TokenUsage Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence UsagePresence *domain.TokenUsagePresence
CostUSD *float64
RetryNotBefore time.Time RetryNotBefore time.Time
Evidence HTTPExecutionEvidence Evidence HTTPExecutionEvidence
StructuredOutputRequested *bool StructuredOutputRequested *bool
@@ -156,7 +158,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault} httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
} }
return &PreparedGeneration{ return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey, endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider),
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient, timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
structuredOutputRequested: wirePayload["response_format"] != nil, structuredOutputRequested: wirePayload["response_format"] != nil,
}, nil }, nil
@@ -266,16 +268,40 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
} }
return outcome return outcome
} }
var writes, reasoning *int
switch p.provider {
case "openrouter", "vllm":
var details map[string]json.RawMessage
if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil {
reasoning = optionalInteger(details["reasoning_tokens"])
}
if p.provider == "openrouter" {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens)
} else {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens)
}
default:
if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil {
outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
return outcome
}
}
outcome.ResponseDecoded = true outcome.ResponseDecoded = true
outcome.Usage = domain.TokenUsage{ outcome.Usage = domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens), PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens), TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens), CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning),
} }
outcome.UsagePresence = &domain.TokenUsagePresence{ outcome.UsagePresence = &domain.TokenUsagePresence{
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil, PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil, TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil, CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
}
if p.provider == "openrouter" {
outcome.CostUSD = optionalFloat(wireResp.Usage.Cost)
} }
if len(wireResp.Choices) == 0 { if len(wireResp.Choices) == 0 {
@@ -297,7 +323,7 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
} }
return outcome return outcome
} }
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence} outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: outcome.CostUSD}
outcome.GenerationCompleted = true outcome.GenerationCompleted = true
if captureEvidence { if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil { if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
@@ -541,11 +567,37 @@ type openAIChatResponse struct {
TotalTokens *int `json:"total_tokens"` TotalTokens *int `json:"total_tokens"`
PromptTokensDetails struct { PromptTokensDetails struct {
CachedTokens *int `json:"cached_tokens"` CachedTokens *int `json:"cached_tokens"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"`
} `json:"prompt_tokens_details"` } `json:"prompt_tokens_details"`
CacheWriteTokens *int `json:"cache_write_tokens"` CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
Cost json.RawMessage `json:"cost"`
} `json:"usage"` } `json:"usage"`
} }
func optionalInteger(raw json.RawMessage) *int {
if len(raw) == 0 {
return nil
}
var value *int
if json.Unmarshal(raw, &value) != nil {
return nil
}
return value
}
func optionalFloat(raw json.RawMessage) *float64 {
if len(raw) == 0 {
return nil
}
var value *float64
if json.Unmarshal(raw, &value) != nil || (value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0))) {
return nil
}
return value
}
type openAIResponseFormat struct { type openAIResponseFormat struct {
Type string `json:"type"` Type string `json:"type"`
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"` JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`

View File

@@ -882,6 +882,85 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
} }
} }
func TestProviderSpecificUsageDetails(t *testing.T) {
for _, tc := range []struct {
name, provider, usage string
writes, reasoning int
writesKnown, reasoningKnown bool
}{
{name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true},
{name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true},
{name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true},
{name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`},
{name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`},
{name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`},
{name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`},
} {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`)
request := provider.generateRequest()
request.Target.Provider = tc.provider
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown {
t.Fatalf("response=%+v error=%v", response, err)
}
})
}
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`)
request := provider.generateRequest()
request.Target.Provider = "openrouter"
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.Execute(context.Background())
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens {
t.Fatalf("outcome=%+v", outcome)
}
}
func TestProviderReportedCostIsOptional(t *testing.T) {
for _, tc := range []struct {
name, provider, cost string
known bool
value float64
}{
{"charge", "openrouter", `0.002`, true, .002},
{"zero", "openrouter", `0`, true, 0},
{"null", "openrouter", `null`, false, 0},
{"string", "openrouter", `"0.2"`, false, 0},
{"object", "openrouter", `{}`, false, 0},
{"overflow", "openrouter", `1e9999`, false, 0},
{"generic ignores cost", "openai-compatible", `0.02`, false, 0},
{"vllm ignores cost", "vllm", `0.02`, false, 0},
} {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"cost":`+tc.cost+`},"choices":[{"message":{"content":"ok"}}]}`)
request := provider.generateRequest()
request.Target.Provider = tc.provider
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
if err != nil || response == nil || (response.CostUSD != nil) != tc.known || (response.CostUSD != nil && *response.CostUSD != tc.value) {
t.Fatalf("response=%+v err=%v", response, err)
}
})
}
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"cost":0.02},"choices":[]}`)
request := provider.generateRequest()
request.Target.Provider = "openrouter"
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.Execute(context.Background())
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.CostUSD == nil || *outcome.CostUSD != .02 {
t.Fatalf("outcome=%+v", outcome)
}
}
func checkOptionalResponseFieldOmission(t *testing.T) { func checkOptionalResponseFieldOmission(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})

View File

@@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
@@ -26,8 +27,11 @@ func OpenRouterKeyEndpoint(base string) (string, error) {
return url.JoinPath(base, "key") return url.JoinPath(base, "key")
} }
// ProbeOpenRouterKey executes and interprets an OpenRouter key request. // ProbeAuthentication executes the selected authentication adapter.
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) { func (c *OpenAICompatibleClient) ProbeAuthentication(ctx context.Context, adapter, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
if !SupportedAuthenticationAdapter(adapter) {
return AuthenticationEvidence{}, fmt.Errorf("unsupported authentication adapter %q", adapter)
}
parsed, err := url.Parse(endpoint) parsed, err := url.Parse(endpoint)
if err != nil { if err != nil {
return AuthenticationEvidence{}, err return AuthenticationEvidence{}, err
@@ -69,7 +73,14 @@ func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoin
if err != nil { if err != nil {
return result, err return result, err
} }
if recognizedOpenRouterKeyResponse(contents) { recognized := false
switch adapter {
case "openrouter_key":
recognized = recognizedOpenRouterKeyResponse(contents)
case "vllm_models":
recognized = recognizedVLLMModelsResponse(contents)
}
if recognized {
accepted := true accepted := true
result.Accepted = &accepted result.Accepted = &accepted
} else { } else {

View File

@@ -38,7 +38,7 @@ func TestProbeOpenRouterKey(t *testing.T) {
authorization = req.Header.Get("Authorization") authorization = req.Header.Get("Authorization")
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
})}}) })}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key}) got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -67,7 +67,7 @@ func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
})}}) })}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{}) got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
if !errors.Is(err, want) || !got.HTTP.ResponseReceived { if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
t.Fatalf("evidence = %+v, error = %v", got, err) t.Fatalf("evidence = %+v, error = %v", got, err)
} }
@@ -105,7 +105,7 @@ func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
})}}) })}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{}) got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) { if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err) t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
} }
@@ -127,7 +127,7 @@ func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
})}}) })}})
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{}) got, err := client.ProbeAuthentication(ctx, "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived { if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
t.Fatalf("evidence = %+v, error = %v", got, err) t.Fatalf("evidence = %+v, error = %v", got, err)
} }

59
internal/llm/provider.go Normal file
View File

@@ -0,0 +1,59 @@
package llm
import (
"net/url"
"go.maximumdirect.net/promptkit/internal/domain"
)
// EffectiveProvider resolves an omitted provider to the generic conventions.
func EffectiveProvider(provider string) string {
if provider == "" {
return "openai-compatible"
}
return provider
}
// SupportedProvider reports whether a normalized provider selection is supported.
// An empty selection uses the generic OpenAI-compatible conventions.
func SupportedProvider(provider string) bool {
switch provider {
case "", "openai-compatible", "openrouter", "vllm":
return true
default:
return false
}
}
// AuthenticationAdapter selects declared diagnostic metadata or provider defaults.
func AuthenticationAdapter(provider string, smoke *domain.BackendSmokeTestConfig) string {
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Kind != nil {
return *smoke.Authentication.Kind
}
switch provider {
case "openrouter":
return "openrouter_key"
case "vllm":
return "vllm_models"
}
return ""
}
// AuthenticationEndpoint resolves a diagnostic destination without changing configuration.
func AuthenticationEndpoint(adapter, base string, smoke *domain.BackendSmokeTestConfig) (string, error) {
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Endpoint != nil {
return *smoke.Authentication.Endpoint, nil
}
if adapter == "openrouter_key" {
return OpenRouterKeyEndpoint(base)
}
if adapter == "vllm_models" {
return url.JoinPath(base, "models")
}
return base, nil
}
// SupportedAuthenticationAdapter identifies implemented authentication probes.
func SupportedAuthenticationAdapter(adapter string) bool {
return adapter == "openrouter_key" || adapter == "vllm_models"
}

View File

@@ -0,0 +1,34 @@
package llm
import (
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestAuthenticationSelection(t *testing.T) {
none, explicit, models, endpoint := "none", "openrouter_key", "vllm_models", "https://diagnostics.example/probe"
for _, tc := range []struct {
name, provider, adapter, destination string
metadata *domain.BackendSmokeTestConfig
}{
{name: "provider default", provider: "openrouter", adapter: "openrouter_key", destination: "https://example.test/v1/key"},
{name: "cleared block", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
{name: "endpoint only", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "openrouter_key", destination: endpoint},
{name: "explicit adapter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &explicit}}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
{name: "disabled", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
{name: "generic unsupported", destination: "https://example.test/v1"},
{name: "vllm default", provider: "vllm", adapter: "vllm_models", destination: "https://example.test/v1/models"},
{name: "vllm override", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "vllm_models", destination: endpoint},
{name: "explicit vllm on generic", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &models}}, adapter: "vllm_models", destination: "https://example.test/v1/models"},
{name: "disabled vllm", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
} {
t.Run(tc.name, func(t *testing.T) {
kind := AuthenticationAdapter(tc.provider, tc.metadata)
destination, err := AuthenticationEndpoint(kind, "https://example.test/v1", tc.metadata)
if err != nil || kind != tc.adapter || destination != tc.destination {
t.Fatalf("selection=%q destination=%q error=%v", kind, destination, err)
}
})
}
}

View File

@@ -0,0 +1,16 @@
package llm
import "encoding/json"
func recognizedVLLMModelsResponse(contents []byte) bool {
var envelope map[string]json.RawMessage
if json.Unmarshal(contents, &envelope) != nil || envelope == nil {
return false
}
var object string
if json.Unmarshal(envelope["object"], &object) != nil || object != "list" {
return false
}
var data []json.RawMessage
return json.Unmarshal(envelope["data"], &data) == nil && data != nil
}

View File

@@ -0,0 +1,51 @@
package llm
import (
"context"
"io"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestVLLMModelsAuthentication(t *testing.T) {
for _, tc := range []struct {
name, body, key string
status int
accepted *bool
unrecognized, overflow bool
}{
{name: "empty list", status: 200, body: `{"object":"list","data":[]}`, accepted: boolPtr(true)},
{name: "nonempty list", status: 200, body: `{"object":"list","data":[{"id":"model"}]}`, key: "key", accepted: boolPtr(true)},
{name: "wrong object", status: 200, body: `{"object":"model","data":[]}`, unrecognized: true},
{name: "null data", status: 200, body: `{"object":"list","data":null}`, unrecognized: true},
{name: "object data", status: 200, body: `{"object":"list","data":{}}`, unrecognized: true},
{name: "invalid framing", status: 200, body: `{"object":"list","data":[]} {}`, unrecognized: true},
{name: "overflow", status: 200, body: strings.Repeat(" ", maxAuthenticationResponseBytes+1), overflow: true},
{name: "unauthorized", status: 401, accepted: boolPtr(false)},
{name: "other status", status: 503},
} {
t.Run(tc.name, func(t *testing.T) {
var credential string
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
credential = req.Header.Get("Authorization")
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
})}})
got, err := client.ProbeAuthentication(context.Background(), "vllm_models", "https://example.invalid/v1/models", 0, domain.ExecutionTarget{APIKey: tc.key})
if err != nil || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow || (got.Accepted == nil) != (tc.accepted == nil) || (got.Accepted != nil && *got.Accepted != *tc.accepted) {
t.Fatalf("result=%+v err=%v", got, err)
}
want := ""
if tc.key != "" {
want = "Bearer " + tc.key
}
if credential != want {
t.Fatalf("authorization=%q", credential)
}
})
}
}
func boolPtr(value bool) *bool { return &value }

View File

@@ -0,0 +1,67 @@
package observation
import (
"context"
"math"
"go.maximumdirect.net/promptkit/internal/domain"
)
type costContextKey struct{}
// WithCostAccumulator carries one invocation's cost state.
func WithCostAccumulator(ctx context.Context, accumulator *CostAccumulator) context.Context {
return context.WithValue(ctx, costContextKey{}, accumulator)
}
// CostFromContext returns the current invocation's cost state, if installed.
func CostFromContext(ctx context.Context) *CostAccumulator {
value, _ := ctx.Value(costContextKey{}).(*CostAccumulator)
return value
}
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {
return domain.ObservedCost{Value: *reported, Known: true, Complete: true, Source: domain.CostSourceProvider}
}
if zeroCost {
return domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}
}
return domain.ObservedCost{}
}
// CostAccumulator merges dispatched attempts or already aggregated costs.
type CostAccumulator struct {
value domain.ObservedCost
started bool
}
// Add incorporates one contribution.
func (c *CostAccumulator) Add(next domain.ObservedCost) {
if !c.started {
c.started = true
c.value = next
return
}
if next.Known {
c.value.Value += next.Value
if next.Source != domain.CostSourceNone {
if c.value.Source == domain.CostSourceNone {
c.value.Source = next.Source
} else if c.value.Source != next.Source {
c.value.Source = domain.CostSourceMixed
}
}
}
c.value.Known = c.value.Known || next.Known
c.value.Complete = c.value.Complete && next.Complete
}
// Observed returns known complete zero when no contribution has been added.
func (c CostAccumulator) Observed() domain.ObservedCost {
if !c.started {
return domain.ObservedCost{Known: true, Complete: true}
}
return c.value
}

View File

@@ -0,0 +1,63 @@
package observation
import (
"math"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestResolveCostPrecedence(t *testing.T) {
zero, negative, invalid := 0.0, -1.5, math.Inf(1)
for _, tc := range []struct {
name string
reported *float64
fallback bool
want domain.ObservedCost
}{
{"unknown", nil, false, domain.ObservedCost{}},
{"configured zero", nil, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
{"reported zero", &zero, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceProvider}},
{"negative", &negative, false, domain.ObservedCost{Value: -1.5, Known: true, Complete: true, Source: domain.CostSourceProvider}},
{"invalid fallback", &invalid, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
{"invalid unknown", &invalid, false, domain.ObservedCost{}},
} {
t.Run(tc.name, func(t *testing.T) {
if got := ResolveCost(tc.reported, tc.fallback); got != tc.want {
t.Fatalf("got %+v want %+v", got, tc.want)
}
})
}
}
func TestCostAccumulatorMergesSourcesAndCoverage(t *testing.T) {
var acc CostAccumulator
if got := acc.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty: %+v", got)
}
acc.Add(domain.ObservedCost{})
if got := acc.Observed(); got != (domain.ObservedCost{}) {
t.Fatalf("unknown: %+v", got)
}
a, b := .1, .2
acc.Add(ResolveCost(&a, false))
acc.Add(ResolveCost(&b, false))
acc.Add(ResolveCost(nil, true))
got := acc.Observed()
if math.Abs(got.Value-.3) > 1e-12 || !got.Known || got.Complete || got.Source != domain.CostSourceMixed {
t.Fatalf("merged: %+v", got)
}
var zeros CostAccumulator
zero := 0.0
zeros.Add(ResolveCost(&zero, false))
zeros.Add(ResolveCost(nil, true))
if got := zeros.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceMixed}) {
t.Fatalf("zeros: %+v", got)
}
var aggregate CostAccumulator
aggregate.Add(domain.ObservedCost{Known: true, Complete: true})
aggregate.Add(got)
if aggregate.Observed() != got {
t.Fatalf("aggregate: %+v", aggregate.Observed())
}
}

View File

@@ -14,6 +14,7 @@ type Group struct {
terminals int terminals int
attempts, retries, repairs uint64 attempts, retries, repairs uint64
usage domain.ObservedTokenUsage usage domain.ObservedTokenUsage
cost CostAccumulator
} }
// Observe enrolls a run and incorporates its first terminal event only. // Observe enrolls a run and incorporates its first terminal event only.
@@ -36,9 +37,10 @@ func (g *Group) Observe(event domain.ExecutionEvent) {
g.repairs += event.RepairCalls g.repairs += event.RepairCalls
if g.terminals == 0 { if g.terminals == 0 {
complete := domain.ObservedTokenCount{Complete: true} complete := domain.ObservedTokenCount{Complete: true}
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete} g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
} }
mergeGroupUsage(&g.usage, event.Usage) mergeGroupUsage(&g.usage, event.Usage)
g.cost.Add(event.CostUSD)
g.terminals++ g.terminals++
} }
@@ -47,12 +49,15 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary {
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage} result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage}
if g.terminals > 0 || len(g.members) == 0 {
result.CostUSD = g.cost.Observed()
}
for id, terminal := range g.members { for id, terminal := range g.members {
result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal}) result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal})
} }
if len(g.members) == 0 { if len(g.members) == 0 {
zero := domain.ObservedTokenCount{Known: true, Complete: true} zero := domain.ObservedTokenCount{Known: true, Complete: true}
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero} result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, ReasoningTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
} }
return result return result
} }
@@ -67,6 +72,7 @@ func mergeGroupUsage(total *domain.ObservedTokenUsage, next domain.ObservedToken
} }
merge(&total.PromptTokens, next.PromptTokens) merge(&total.PromptTokens, next.PromptTokens)
merge(&total.CompletionTokens, next.CompletionTokens) merge(&total.CompletionTokens, next.CompletionTokens)
merge(&total.ReasoningTokens, next.ReasoningTokens)
merge(&total.TotalTokens, next.TotalTokens) merge(&total.TotalTokens, next.TotalTokens)
merge(&total.CachedTokens, next.CachedTokens) merge(&total.CachedTokens, next.CachedTokens)
merge(&total.CacheWriteTokens, next.CacheWriteTokens) merge(&total.CacheWriteTokens, next.CacheWriteTokens)

View File

@@ -41,6 +41,25 @@ func TestGroupTerminalAccounting(t *testing.T) {
} }
} }
func TestGroupCostCoverageIsIndependentOfTerminalCoverage(t *testing.T) {
var group Group
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty=%+v", got)
}
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionStarted, RunID: "pending"})
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{}) {
t.Fatalf("pending=%+v", got)
}
charge := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
terminal := domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "done", CostUSD: charge}
group.Observe(terminal)
group.Observe(terminal)
got := group.Snapshot()
if got.TerminalCoverageComplete || got.CostUSD != charge {
t.Fatalf("snapshot=%+v", got)
}
}
func TestGroupConcurrentSnapshots(t *testing.T) { func TestGroupConcurrentSnapshots(t *testing.T) {
var group Group var group Group
var wg sync.WaitGroup var wg sync.WaitGroup

View File

@@ -30,6 +30,7 @@ type Recorder struct {
generationDuration time.Duration generationDuration time.Duration
retryDelay time.Duration retryDelay time.Duration
usage UsageAccumulator usage UsageAccumulator
cost *CostAccumulator
} }
// Selection contains resolved identities only. Unresolved fields stay empty. // Selection contains resolved identities only. Unresolved fields stay empty.
@@ -53,6 +54,7 @@ type AttemptData struct {
StatusCode int StatusCode int
Usage domain.TokenUsage Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence UsagePresence *domain.TokenUsagePresence
CostUSD domain.ObservedCost
} }
// DelayData describes one completed or interrupted retry wait. // DelayData describes one completed or interrupted retry wait.
@@ -99,6 +101,9 @@ func (r *Recorder) RunID() string { return r.runID }
// SetParentRunID records the immediate source identity before start delivery. // SetParentRunID records the immediate source identity before start delivery.
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id } func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
// SetCostAccumulator selects the invocation-owned cost state for terminal reporting.
func (r *Recorder) SetCostAccumulator(cost *CostAccumulator) { r.cost = cost }
// Start delivers the opening event with only identities already resolved. // Start delivers the opening event with only identities already resolved.
func (r *Recorder) Start() { func (r *Recorder) Start() {
event := r.event(domain.ExecutionStarted) event := r.event(domain.ExecutionStarted)
@@ -138,6 +143,7 @@ func (r *Recorder) Attempt(data AttemptData) {
event.StatusCode = data.StatusCode event.StatusCode = data.StatusCode
if data.Dispatched { if data.Dispatched {
event.Usage = observedUsage(data.Usage, data.UsagePresence) event.Usage = observedUsage(data.Usage, data.UsagePresence)
event.CostUSD = data.CostUSD
} }
r.deliver(event) r.deliver(event)
} }
@@ -176,6 +182,9 @@ func (r *Recorder) Finish(data FinishData) {
event.Duration = 0 event.Duration = 0
} }
event.Usage = r.usage.Observed() event.Usage = r.usage.Observed()
if r.cost != nil {
event.CostUSD = r.cost.Observed()
}
r.deliver(event) r.deliver(event)
} }
@@ -195,6 +204,22 @@ func (r *Recorder) deliver(event domain.ExecutionEvent) {
type contextKey struct{} type contextKey struct{}
type observerKey struct{}
// WithObserver adds an observer to the inherited contextual selection.
func WithObserver(ctx context.Context, observer domain.ExecutionObserver) context.Context {
if observer == nil {
return ctx
}
return context.WithValue(ctx, observerKey{}, Compose(ObserverFromContext(ctx), observer))
}
// ObserverFromContext returns the contextual observer selection, if any.
func ObserverFromContext(ctx context.Context) domain.ExecutionObserver {
observer, _ := ctx.Value(observerKey{}).(domain.ExecutionObserver)
return observer
}
// WithoutExecutionState starts a public operation without inheriting another // WithoutExecutionState starts a public operation without inheriting another
// invocation's recorder or prepared-claim sink. Other context values and all // invocation's recorder or prepared-claim sink. Other context values and all
// cancellation/deadline behavior remain with the caller's context. // cancellation/deadline behavior remain with the caller's context.
@@ -206,7 +231,7 @@ type executionBoundary struct{ context.Context }
func (c executionBoundary) Value(key any) any { func (c executionBoundary) Value(key any) any {
switch key.(type) { switch key.(type) {
case contextKey, claimSinkKey: case contextKey, claimSinkKey, costContextKey:
return nil return nil
default: default:
return c.Context.Value(key) return c.Context.Value(key)
@@ -244,7 +269,7 @@ type usageField struct {
known, complete bool known, complete bool
} }
type usageTotals struct { type usageTotals struct {
prompt, completion, total, cached, cacheWrite usageField prompt, completion, reasoning, total, cached, cacheWrite usageField
} }
// UsageAccumulator combines presence-aware usage across dispatched attempts. // UsageAccumulator combines presence-aware usage across dispatched attempts.
@@ -270,7 +295,7 @@ func (u UsageAccumulator) Observed() domain.ObservedTokenUsage {
} }
func completeUsageTotals() usageTotals { func completeUsageTotals() usageTotals {
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, reasoning: usageField{complete: true},
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}} total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
} }
@@ -278,6 +303,7 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
known := usagePresence(usage, presence) known := usagePresence(usage, presence)
u.prompt.add(usage.PromptTokens, known.PromptTokens) u.prompt.add(usage.PromptTokens, known.PromptTokens)
u.completion.add(usage.CompletionTokens, known.CompletionTokens) u.completion.add(usage.CompletionTokens, known.CompletionTokens)
u.reasoning.add(usage.ReasoningTokens, known.ReasoningTokens)
u.total.add(usage.TotalTokens, known.TotalTokens) u.total.add(usage.TotalTokens, known.TotalTokens)
u.cached.add(usage.CachedTokens, known.CachedTokens) u.cached.add(usage.CachedTokens, known.CachedTokens)
u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens) u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens)
@@ -286,6 +312,7 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage { func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage {
return domain.ObservedTokenUsage{ return domain.ObservedTokenUsage{
PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts), PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts),
ReasoningTokens: u.reasoning.snapshot(attempts),
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts), TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
CacheWriteTokens: u.cacheWrite.snapshot(attempts), CacheWriteTokens: u.cacheWrite.snapshot(attempts),
} }
@@ -312,6 +339,7 @@ func observedUsage(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
return domain.ObservedTokenUsage{ return domain.ObservedTokenUsage{
PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens), PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens),
CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens), CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens),
ReasoningTokens: observedCount(usage.ReasoningTokens, known.ReasoningTokens),
TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens), TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens),
CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens), CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens),
CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens), CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens),
@@ -324,6 +352,7 @@ func usagePresence(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
} }
return domain.TokenUsagePresence{ return domain.TokenUsagePresence{
PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0, PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0,
ReasoningTokens: usage.ReasoningTokens != 0,
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0, TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
CacheWriteTokens: usage.CacheWriteTokens != 0, CacheWriteTokens: usage.CacheWriteTokens != 0,
} }

View File

@@ -19,10 +19,15 @@ func TestExecutionBoundaryPreservesCallerContext(t *testing.T) {
recorder := newRecorder("outer", time.Now, nil) recorder := newRecorder("outer", time.Now, nil)
sink := recorder sink := recorder
parent = WithClaimSink(WithRecorder(parent, recorder), &sink) parent = WithClaimSink(WithRecorder(parent, recorder), &sink)
var cost CostAccumulator
parent = WithCostAccumulator(parent, &cost)
child := WithoutExecutionState(parent) child := WithoutExecutionState(parent)
if FromContext(child) != nil || FromContext(parent) != recorder { if FromContext(child) != nil || FromContext(parent) != recorder {
t.Fatal("execution boundary did not isolate recorder state") t.Fatal("execution boundary did not isolate recorder state")
} }
if CostFromContext(child) != nil || CostFromContext(parent) != &cost {
t.Fatal("execution boundary did not isolate cost state")
}
RegisterClaim(child, nil) RegisterClaim(child, nil)
if sink != recorder { if sink != recorder {
t.Fatal("execution boundary retained the parent claim sink") t.Fatal("execution boundary retained the parent claim sink")

View File

@@ -25,3 +25,18 @@ func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) {
t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens) t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens)
} }
} }
func TestReasoningUsagePresence(t *testing.T) {
var usage UsageAccumulator
usage.Add(domain.TokenUsage{}, &domain.TokenUsagePresence{ReasoningTokens: true})
usage.Add(domain.TokenUsage{ReasoningTokens: 4}, nil)
got := usage.Observed().ReasoningTokens
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: true}) {
t.Fatalf("usage=%+v", got)
}
usage.Add(domain.TokenUsage{}, nil)
got = usage.Observed().ReasoningTokens
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}) {
t.Fatalf("partial usage=%+v", got)
}
}

View File

@@ -37,6 +37,10 @@ type CollectionSettings struct {
type BackendSettings struct { type BackendSettings struct {
// ID identifies the backend in its resource list. // ID identifies the backend in its resource list.
ID string `yaml:"id"` ID string `yaml:"id"`
// Provider selects response and diagnostic conventions when supplied.
Provider *string `yaml:"provider"`
// ZeroCost enables a configured-zero fallback when no cost is reported.
ZeroCost *bool `yaml:"zero_cost"`
// Endpoint is the backend's absolute base URL. // Endpoint is the backend's absolute base URL.
Endpoint *string `yaml:"endpoint"` Endpoint *string `yaml:"endpoint"`
// APIKeyEnv names an environment variable containing a credential. // APIKeyEnv names an environment variable containing a credential.
@@ -163,7 +167,6 @@ type BackendSmokeTestConfig struct {
// diagnostics. A nil endpoint uses the effective backend endpoint. // diagnostics. A nil endpoint uses the effective backend endpoint.
Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"` Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"`
// Authentication optionally declares a named authentication diagnostic. // Authentication optionally declares a named authentication diagnostic.
// A supplied object requires Kind.
Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"` Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"`
} }
@@ -178,7 +181,8 @@ type ReachabilityProbeConfig struct {
// AuthenticationProbeConfig configures an optional named authentication diagnostic. // AuthenticationProbeConfig configures an optional named authentication diagnostic.
type AuthenticationProbeConfig struct { type AuthenticationProbeConfig struct {
// Kind selects the authentication adapter. Supported values are "none" and // Kind selects the authentication adapter. Supported values are "none" and
// "openrouter_key"; surrounding whitespace and case are normalized. // "openrouter_key" and "vllm_models"; surrounding whitespace and case are
// normalized. Nil leaves adapter selection to the provider default.
Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"` Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"`
// Endpoint is an absolute HTTP(S) URL, including any desired path or query. // Endpoint is an absolute HTTP(S) URL, including any desired path or query.
// Nil selects the adapter's backend-relative destination; an explicit blank // Nil selects the adapter's backend-relative destination; an explicit blank

View File

@@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
return nil, err return nil, err
} }
defer payload.clear() defer payload.clear()
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := NewOperationScope(ctx, payload.operationTimeout) scope := NewOperationScope(ctx, payload.operationTimeout)
defer scope.Close() defer scope.Close()
defer func() { err = scope.Finalize(err) }() defer func() { err = scope.Finalize(err) }()
@@ -117,9 +119,11 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
var recorder *observation.Recorder var recorder *observation.Recorder
var runID string var runID string
if payload.observer != nil { observer := observation.Compose(payload.observer, observation.ObserverFromContext(ctx))
if observer != nil {
var idErr error var idErr error
recorder, idErr = observation.New(payload.observer) recorder, idErr = observation.New(observer)
recorder.SetCostAccumulator(cost)
recorder.SetParentRunID(payload.parent.RunID()) recorder.SetParentRunID(payload.parent.RunID())
observation.RegisterClaim(ctx, recorder) observation.RegisterClaim(ctx, recorder)
ctx = observation.WithRecorder(ctx, recorder) ctx = observation.WithRecorder(ctx, recorder)

View File

@@ -95,6 +95,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
} }
admitter := &fakeRunAdmitter{} admitter := &fakeRunAdmitter{}
profile := defaultExecutionProfile() profile := defaultExecutionProfile()
profile.ResolvedTarget.Provider = "vllm"
profile.ResolvedTarget.ZeroCost = true
profile.ResolvedTarget.ExtraParams = map[string]any{ profile.ResolvedTarget.ExtraParams = map[string]any{
"metadata": map[string]any{"source": "original"}, "metadata": map[string]any{"source": "original"},
} }
@@ -144,9 +146,11 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
} }
renderer.rendered.Messages[0].Content = "source mutation" renderer.rendered.Messages[0].Content = "source mutation"
profile.ResolvedTarget.Provider = "openrouter"
profile.ResolvedTarget.ZeroCost = false
second := prepared.Details() second := prepared.Details()
if second != first || second.Messages[0].Content != "original message" || if second != first || second.Messages[0].Content != "original message" || second.EffectiveModelParams.Provider != "vllm" || !second.EffectiveModelParams.ZeroCost ||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" || second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" { second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
t.Fatalf("borrowed details did not preserve the snapshot: %#v", second) t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
@@ -313,7 +317,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
responses: []*domain.GenerateResponse{{ responses: []*domain.GenerateResponse{{
Content: `{"repaired":true}`, Content: `{"repaired":true}`,
Usage: domain.TokenUsage{ Usage: domain.TokenUsage{
PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5, PromptTokens: 2, CompletionTokens: 3, ReasoningTokens: 2, TotalTokens: 5,
CachedTokens: 7, CacheWriteTokens: 11, CachedTokens: 7, CacheWriteTokens: 11,
}, },
}}, }},
@@ -324,7 +328,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
client := &sequenceLLM{responses: []*domain.GenerateResponse{{ client := &sequenceLLM{responses: []*domain.GenerateResponse{{
Content: `{"broken":true}`, Content: `{"broken":true}`,
Usage: domain.TokenUsage{ Usage: domain.TokenUsage{
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19, PromptTokens: 13, CompletionTokens: 17, ReasoningTokens: 4, TotalTokens: 19,
CachedTokens: 23, CacheWriteTokens: 29, CachedTokens: 23, CacheWriteTokens: 29,
}, },
}}} }}}
@@ -366,7 +370,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
t.Fatalf("unexpected repaired validation result: %+v", result.Validation) t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
} }
wantUsage := domain.TokenUsage{ wantUsage := domain.TokenUsage{
PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24, PromptTokens: 15, CompletionTokens: 20, ReasoningTokens: 6, TotalTokens: 24,
CachedTokens: 30, CacheWriteTokens: 40, CachedTokens: 30, CacheWriteTokens: 40,
} }
if result.Usage != wantUsage { if result.Usage != wantUsage {

View File

@@ -11,6 +11,7 @@ import (
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) { func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{ profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
Provider: "openrouter", ZeroCost: true,
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
Temperature: 0, ExtraParams: map[string]any{"configured": true}, Temperature: 0, ExtraParams: map[string]any{"configured": true},
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}} }, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
@@ -35,7 +36,7 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
t.Fatalf("request did not complete partial target: %v", err) t.Fatalf("request did not complete partial target: %v", err)
} }
if prepared.EffectiveModelParams.Endpoint != "https://runtime.example/v1" || if prepared.EffectiveModelParams.Endpoint != "https://runtime.example/v1" ||
prepared.EffectiveModelParams.Model != "runtime-model" || !prepared.TargetPresence.Temperature || prepared.EffectiveModelParams.Model != "runtime-model" || prepared.EffectiveModelParams.Provider != "openrouter" || !prepared.EffectiveModelParams.ZeroCost || !prepared.TargetPresence.Temperature ||
prepared.EffectiveModelParams.ExtraParams["request"] != true { prepared.EffectiveModelParams.ExtraParams["request"] != true {
t.Fatalf("request overrides were not applied: %+v", prepared) t.Fatalf("request overrides were not applied: %+v", prepared)
} }

View File

@@ -137,6 +137,9 @@ func NewRunnerWithRepairer(
} }
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) { func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
if observation.CostFromContext(ctx) == nil {
ctx = observation.WithCostAccumulator(ctx, &observation.CostAccumulator{})
}
recorder := observation.FromContext(ctx) recorder := observation.FromContext(ctx)
runID := "" runID := ""
if recorder != nil { if recorder != nil {
@@ -274,7 +277,7 @@ func (r *Runner) executePreparedRun(
exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content) exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content)
} }
return &domain.RunResult{ result := &domain.RunResult{
Exchange: exchange, Exchange: exchange,
ParentRunID: prepared.ParentRunID, ParentRunID: prepared.ParentRunID,
RunID: runID, RunID: runID,
@@ -295,7 +298,11 @@ func (r *Runner) executePreparedRun(
StartTime: start, StartTime: start,
EndTime: end, EndTime: end,
Duration: end.Sub(start), Duration: end.Sub(start),
}, nil }
if cost := observation.CostFromContext(ctx); cost != nil {
result.CostUSD = cost.Observed()
}
return result, nil
} }
func wrapGenerationError(err error) error { func wrapGenerationError(err error) error {
@@ -309,6 +316,7 @@ func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
return domain.TokenUsage{ return domain.TokenUsage{
PromptTokens: total.PromptTokens + next.PromptTokens, PromptTokens: total.PromptTokens + next.PromptTokens,
CompletionTokens: total.CompletionTokens + next.CompletionTokens, CompletionTokens: total.CompletionTokens + next.CompletionTokens,
ReasoningTokens: total.ReasoningTokens + next.ReasoningTokens,
TotalTokens: total.TotalTokens + next.TotalTokens, TotalTokens: total.TotalTokens + next.TotalTokens,
CachedTokens: total.CachedTokens + next.CachedTokens, CachedTokens: total.CachedTokens + next.CachedTokens,
CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens, CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens,

View File

@@ -11,7 +11,7 @@ import (
) )
type smokeAuthenticationClient interface { type smokeAuthenticationClient interface {
ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error) ProbeAuthentication(context.Context, string, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
} }
type authenticationProbe struct{ client smokeAuthenticationClient } type authenticationProbe struct{ client smokeAuthenticationClient }
@@ -30,7 +30,8 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) { if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
return outcome, err return outcome, err
} }
result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings) adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
result, requestErr := p.client.ProbeAuthentication(ctx, adapter, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
outcome.evidence = result.HTTP outcome.evidence = result.HTTP
outcome.credentialsAccepted = result.Accepted outcome.credentialsAccepted = result.Accepted
if result.Overflow { if result.Overflow {
@@ -55,22 +56,18 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
} }
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) { func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil { adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
if adapter == "" {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
} }
authentication := backend.SmokeTests.Authentication if adapter == "none" {
if authentication.Kind == nil {
return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest)
}
switch *authentication.Kind {
case "none":
return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil
case "openrouter_key": }
if !llm.SupportedAuthenticationAdapter(adapter) {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
}
if p.client == nil { if p.client == nil {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
} }
return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil
default:
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
}
} }

View File

@@ -95,6 +95,11 @@ func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
backend: backendWithAuthenticationKind("openrouter_key"), backend: backendWithAuthenticationKind("openrouter_key"),
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable, reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
}, },
{
name: "vllm HTTP adapter unavailable",
backend: domain.Backend{Provider: "vllm"},
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
probe := authenticationProbe{} probe := authenticationProbe{}

View File

@@ -110,6 +110,8 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
} else { } else {
settings := defaults.ExecutionTargetDefault() settings := defaults.ExecutionTargetDefault()
settings.BackendID = selection.backend.ID settings.BackendID = selection.backend.ID
settings.Provider = selection.backend.Provider
settings.ZeroCost = selection.backend.ZeroCost
settings.Endpoint = selection.backend.Endpoint settings.Endpoint = selection.backend.Endpoint
settings.APIKeyEnv = selection.backend.APIKeyEnv settings.APIKeyEnv = selection.backend.APIKeyEnv
if len(selection.backend.ExtraParams) > 0 { if len(selection.backend.ExtraParams) > 0 {
@@ -132,7 +134,7 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
if promptInspection != nil { if promptInspection != nil {
selection.target.PromptID = promptInspection.PromptID selection.target.PromptID = promptInspection.PromptID
} }
selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend.SmokeTests) selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err)
} }
@@ -165,22 +167,15 @@ func isInferenceSmokeTest(kind domain.SmokeTestKind) bool {
return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput
} }
func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, smokeTests *domain.BackendSmokeTestConfig) (string, error) { func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, backend domain.Backend) (string, error) {
smokeTests := backend.SmokeTests
switch kind { switch kind {
case domain.SmokeTestReachability: case domain.SmokeTestReachability:
if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil { if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil {
return *smokeTests.Reachability.Endpoint, nil return *smokeTests.Reachability.Endpoint, nil
} }
case domain.SmokeTestAuthentication: case domain.SmokeTestAuthentication:
if smokeTests != nil && smokeTests.Authentication != nil { return llm.AuthenticationEndpoint(llm.AuthenticationAdapter(backend.Provider, smokeTests), baseEndpoint, smokeTests)
authentication := smokeTests.Authentication
if authentication.Endpoint != nil {
return *authentication.Endpoint, nil
}
if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
return llm.OpenRouterKeyEndpoint(baseEndpoint)
}
}
} }
return baseEndpoint, nil return baseEndpoint, nil
} }

View File

@@ -0,0 +1,127 @@
package promptkit_test
import (
"context"
"encoding/json"
"io"
"math"
"net/http"
"strings"
"sync"
"sync/atomic"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) {
pendingStarted, release := make(chan struct{}), make(chan struct{})
releasePending := sync.OnceFunc(func() { close(release) })
defer releasePending()
var retryCalls atomic.Int32
transport := costRoundTrip(func(req *http.Request) (*http.Response, error) {
var body struct {
Model string `json:"model"`
}
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
return nil, err
}
status, payload := 200, ""
switch body.Model {
case "router":
if retryCalls.Add(1) == 1 {
status, payload = 503, `{"error":{}}`
} else {
payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`
}
case "failure":
payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}`
case "vllm":
payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}`
case "pending":
close(pendingStarted)
<-release
payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}`
default:
t.Errorf("unexpected model %q", body.Model)
}
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil
})
engine := func(provider, model string) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false))
if err != nil {
t.Fatal(err)
}
return result
}
router, vllm := engine("openrouter", "router"), engine("vllm", "vllm")
var group promptkit.ExecutionGroup
if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty=%+v", empty)
}
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
results := make(chan error, 4)
var completed sync.WaitGroup
var pendingDone sync.WaitGroup
start := func(e *promptkit.Engine, model string, fail bool) {
completed.Add(1)
go func() {
defer completed.Done()
req := promptkit.RunRequest{PromptID: "prompt"}
if model != "" {
req.Execution = &promptkit.ExecutionTargetOverride{Model: model}
}
_, err := e.Run(ctx, req)
if fail && err != nil {
err = nil
} else if fail {
err = io.ErrUnexpectedEOF
}
results <- err
}()
}
pendingDone.Add(1)
go func() {
defer pendingDone.Done()
_, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}})
results <- err
}()
<-pendingStarted
if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete {
t.Fatalf("pending=%+v", pending)
}
start(router, "", false)
start(router, "failure", true)
start(vllm, "", false)
completed.Wait()
for range 3 {
if err := <-results; err != nil {
t.Fatal(err)
}
}
snapshot := group.Snapshot()
if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 {
t.Fatalf("snapshot=%+v", snapshot)
}
if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete {
t.Fatalf("usage=%+v", snapshot.Usage)
}
if _, ok := snapshot.Usage.CacheHitRatio(); ok {
t.Fatal("ratio available despite an unreported retry")
}
releasePending()
if err := <-results; err != nil {
t.Fatal(err)
}
pendingDone.Wait()
if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 {
t.Fatalf("final=%+v", final)
}
}

View File

@@ -19,9 +19,9 @@ func stringPointerForTest(value string) *string {
func TestGenerateResponseUsagePresenceJSON(t *testing.T) { func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
value := promptkit.GenerateResponse{ value := promptkit.GenerateResponse{
Content: "ok", Content: "ok",
Usage: promptkit.TokenUsage{TotalTokens: 5}, Usage: promptkit.TokenUsage{TotalTokens: 5, ReasoningTokens: 2},
UsagePresence: &promptkit.TokenUsagePresence{ UsagePresence: &promptkit.TokenUsagePresence{
PromptTokens: true, CompletionTokens: true, TotalTokens: true, PromptTokens: true, CompletionTokens: true, TotalTokens: true, ReasoningTokens: true,
CachedTokens: true, CacheWriteTokens: true, CachedTokens: true, CacheWriteTokens: true,
}, },
} }
@@ -30,6 +30,9 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
object := decodeJSONObject(t, payload) object := decodeJSONObject(t, payload)
if string(decodeJSONObject(t, object["usage"])["reasoning_tokens"]) != "2" {
t.Fatalf("reasoning field missing: %s", payload)
}
if _, ok := object["usage_presence"]; !ok { if _, ok := object["usage_presence"]; !ok {
t.Fatalf("missing usage_presence: %s", payload) t.Fatalf("missing usage_presence: %s", payload)
} }
@@ -46,6 +49,34 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
} }
} }
func TestCostJSONPreservesUnknownAndReportedZero(t *testing.T) {
var old promptkit.RunResult
if err := json.Unmarshal([]byte(`{"prompt_id":"old"}`), &old); err != nil || old.CostUSD != (promptkit.ObservedCost{}) {
t.Fatalf("old result=%+v err=%v", old, err)
}
value := promptkit.RunResult{CostUSD: promptkit.ObservedCost{Value: .125, Known: true, Complete: false, Source: promptkit.CostSourceProvider}}
data, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
var decoded promptkit.RunResult
if err := json.Unmarshal(data, &decoded); err != nil || decoded.CostUSD != value.CostUSD {
t.Fatalf("round trip=%+v err=%v", decoded, err)
}
zero := 0.0
data, err = json.Marshal(promptkit.GenerateResponse{CostUSD: &zero})
if err != nil || string(decodeJSONObject(t, data)["cost_usd"]) != "0" {
t.Fatalf("reported zero=%s err=%v", data, err)
}
data, err = json.Marshal(promptkit.GenerateResponse{})
if err != nil {
t.Fatal(err)
}
if _, present := decodeJSONObject(t, data)["cost_usd"]; present {
t.Fatalf("unknown cost=%s", data)
}
}
func TestRemovedPromptVersionIsAbsentFromJSON(t *testing.T) { func TestRemovedPromptVersionIsAbsentFromJSON(t *testing.T) {
values := map[string]any{ values := map[string]any{
"prepared": promptkit.PreparedRun{PromptID: "prompt"}, "prepared": promptkit.PreparedRun{PromptID: "prompt"},
@@ -173,6 +204,7 @@ func TestRunResultJSONContractRoundTripsAllFields(t *testing.T) {
"effective_model_params", "effective_model_params",
"input_hashes", "input_hashes",
"usage", "usage",
"cost_usd",
"start_time", "start_time",
"end_time", "end_time",
"duration_ms", "duration_ms",

View File

@@ -50,10 +50,12 @@ func TestPublicLLMClientAdapterGivesClientOwnedNestedValues(t *testing.T) {
func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) { func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
presence := &TokenUsagePresence{PromptTokens: true} presence := &TokenUsagePresence{PromptTokens: true}
cost := .25
client := &usagePresenceClient{response: &GenerateResponse{ client := &usagePresenceClient{response: &GenerateResponse{
Content: "ok", Content: "ok",
Usage: TokenUsage{PromptTokens: 0, TotalTokens: 5}, Usage: TokenUsage{PromptTokens: 0, TotalTokens: 5},
UsagePresence: presence, UsagePresence: presence,
CostUSD: &cost,
}} }}
adapter := publicLLMClientAdapter{client: client} adapter := publicLLMClientAdapter{client: client}
got, err := adapter.Generate(context.Background(), domain.GenerateRequest{}) got, err := adapter.Generate(context.Background(), domain.GenerateRequest{})
@@ -61,17 +63,23 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
presence.PromptTokens = false presence.PromptTokens = false
cost = .5
if got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.UsagePresence.TotalTokens || got.Usage.TotalTokens != 5 { if got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.UsagePresence.TotalTokens || got.Usage.TotalTokens != 5 {
t.Fatalf("copied response = %+v", got) t.Fatalf("copied response = %+v", got)
} }
if got.CostUSD == nil || *got.CostUSD != .25 {
t.Fatalf("copied cost=%v", got.CostUSD)
}
} }
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) { func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {
presence := &TokenUsagePresence{PromptTokens: true} presence := &TokenUsagePresence{PromptTokens: true}
cost := .125
source := &GenerateResponse{ source := &GenerateResponse{
Content: "returned with error", Content: "returned with error",
Usage: TokenUsage{PromptTokens: 6}, Usage: TokenUsage{PromptTokens: 6},
UsagePresence: presence, UsagePresence: presence,
CostUSD: &cost,
} }
marker := errors.New("injected failure") marker := errors.New("injected failure")
adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}} adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}}
@@ -82,7 +90,8 @@ func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T
} }
source.Content = "mutated after return" source.Content = "mutated after return"
presence.PromptTokens = false presence.PromptTokens = false
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens { cost = .5
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.CostUSD == nil || *got.CostUSD != .125 {
t.Fatalf("adapter response was not owned: %+v", got) t.Fatalf("adapter response was not owned: %+v", got)
} }
} }

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"reflect"
"testing" "testing"
"time" "time"
@@ -11,6 +12,66 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestContextualObserversSelectIndependentRuns(t *testing.T) {
var order []string
var events []promptkit.ExecutionEvent
observer := func(name string) promptkit.ExecutionObserver {
return func(event promptkit.ExecutionEvent) {
order = append(order, name)
if name == "inherited" {
events = append(events, event)
}
}
}
base := context.Background()
if promptkit.WithExecutionObserver(base, nil) != base {
t.Fatal("nil observer changed context")
}
ctx := promptkit.WithExecutionObserver(base, observer("inherited"))
ctx = promptkit.WithExecutionObserver(ctx, observer("new"))
ctx = promptkit.WithExecutionObserver(ctx, nil)
child := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: "child"}, nil
}))
var childResult *promptkit.RunResult
outer := observationEngine(t, observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
var err error
childResult, err = child.Run(callCtx, promptkit.RunRequest{PromptID: "prompt", Observer: observer("child explicit")})
return &promptkit.GenerateResponse{Content: "outer"}, err
}))
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: observer("outer explicit")})
if err != nil {
t.Fatal(err)
}
want := []string{"outer explicit", "inherited", "new", "child explicit", "inherited", "new", "child explicit", "inherited", "new", "child explicit", "inherited", "new", "outer explicit", "inherited", "new", "outer explicit", "inherited", "new"}
if !reflect.DeepEqual(order, want) {
t.Fatalf("observer order = %v", order)
}
if childResult.RunID == "" || result.RunID == "" || childResult.RunID == result.RunID {
t.Fatalf("run IDs: outer=%q child=%q", result.RunID, childResult.RunID)
}
counts := map[string]int{}
for _, event := range events {
counts[event.RunID]++
}
if counts[result.RunID] != 3 || counts[childResult.RunID] != 3 || len(counts) != 2 {
t.Fatalf("events = %+v", events)
}
order = nil
if _, err := child.Run(base, promptkit.RunRequest{PromptID: "prompt"}); err != nil {
t.Fatal(err)
}
if len(order) != 0 {
t.Fatalf("unrelated context selected observer: %v", order)
}
if _, err := child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"inherited", "new", "inherited", "new", "inherited", "new"}) {
t.Fatalf("context-only observer order = %v", order)
}
}
type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
func (f observationClientFunc) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { func (f observationClientFunc) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {

View File

@@ -16,6 +16,19 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client)) return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
} }
func TestInjectedReasoningPresenceReachesObservation(t *testing.T) {
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok", UsagePresence: &promptkit.TokenUsagePresence{ReasoningTokens: true}}})
var events []promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
if err != nil || result == nil || len(events) != 3 {
t.Fatalf("result=%+v events=%+v error=%v", result, events, err)
}
want := promptkit.ObservedTokenCount{Known: true, Complete: true}
if events[1].Usage.ReasoningTokens != want || events[2].Usage.ReasoningTokens != want {
t.Fatalf("events=%+v", events)
}
}
func TestContinuationObservationAncestry(t *testing.T) { func TestContinuationObservationAncestry(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}} client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}}
engine := observationEngine(t, client) engine := observationEngine(t, client)

View File

@@ -17,6 +17,20 @@ import (
// event, and Promptkit keeps no asynchronous queue or durable history. // event, and Promptkit keeps no asynchronous queue or durable history.
type ExecutionObserver func(ExecutionEvent) type ExecutionObserver func(ExecutionEvent)
// WithExecutionObserver adds an observer to executions using the returned
// context. Request observers run first, then inherited contextual observers
// in registration order. A nil observer leaves the context unchanged.
// Prepared executions use their execution context, not their preparation
// context. Callers own context propagation and synchronization of observers.
func WithExecutionObserver(ctx context.Context, observer ExecutionObserver) context.Context {
if observer == nil {
return ctx
}
return observation.WithObserver(ctx, func(event domain.ExecutionEvent) {
observer(fromDomainExecutionEvent(event))
})
}
// ComposeObservers returns a synchronous ordered fan-out of non-nil observers. // ComposeObservers returns a synchronous ordered fan-out of non-nil observers.
// It copies the input slice; a panic propagates and prevents later callbacks. // It copies the input slice; a panic propagates and prevents later callbacks.
func ComposeObservers(observers ...ExecutionObserver) ExecutionObserver { func ComposeObservers(observers ...ExecutionObserver) ExecutionObserver {
@@ -83,11 +97,24 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct { type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount CacheWriteTokens ObservedTokenCount
} }
// CacheHitRatio returns the fraction of input tokens served from cache when
// both counts are known, complete, and consistent, with positive input and
// reads between zero and input inclusive. Otherwise it returns (0, false).
// Cache writes are independent; aggregate ratios use summed counts.
func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) {
input, reads := u.PromptTokens, u.CachedTokens
if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value {
return 0, false
}
return float64(reads.Value) / float64(input.Value), true
}
// ExecutionEvent contains metadata only, not provider text, request content, // ExecutionEvent contains metadata only, not provider text, request content,
// credentials, URLs, headers, or error objects. It has no stable JSON contract. // credentials, URLs, headers, or error objects. It has no stable JSON contract.
// Identifiers themselves may be sensitive. Nonterminal aggregate counters are // Identifiers themselves may be sensitive. Nonterminal aggregate counters are
@@ -127,6 +154,7 @@ type ExecutionEvent struct {
StatusCode int StatusCode int
RetriesExhausted bool RetriesExhausted bool
Usage ObservedTokenUsage Usage ObservedTokenUsage
CostUSD ObservedCost
Attempts uint64 Attempts uint64
TransientRetries uint64 TransientRetries uint64
RepairCalls uint64 RepairCalls uint64
@@ -144,8 +172,9 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration, GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure), Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted, StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
CostUSD: fromDomainCost(value.CostUSD),
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}, TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
} }
} }

View File

@@ -15,6 +15,8 @@ const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
// A successful PrepareExecution may retain an Observer function in this // A successful PrepareExecution may retain an Observer function in this
// handle's private payload. The callback and other private execution settings // handle's private payload. The callback and other private execution settings
// are cleared on claim completion or Discard. Details never exposes them. // are cleared on claim completion or Discard. Details never exposes them.
// Contextual observers are selected from the RunPrepared context, not retained
// from preparation.
// It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for // It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for
// execution of the frozen snapshot. Only RunPrepared can produce its completed // execution of the frozen snapshot. Only RunPrepared can produce its completed
// [Exchange]; preparation alone does not. // [Exchange]; preparation alone does not.

View File

@@ -3,6 +3,7 @@ package promptkit_test
import ( import (
"context" "context"
"errors" "errors"
"reflect"
"sync" "sync"
"testing" "testing"
@@ -10,6 +11,50 @@ import (
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestPreparedObservationUsesExecutionContext(t *testing.T) {
engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
foreign := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
var order []string
observe := func(name string) promptkit.ExecutionObserver {
return func(promptkit.ExecutionEvent) { order = append(order, name) }
}
prepareCtx := promptkit.WithExecutionObserver(context.Background(), observe("preparation"))
executeCtx := promptkit.WithExecutionObserver(context.Background(), observe("execution"))
handle, err := engine.PrepareExecution(prepareCtx, promptkit.RunRequest{PromptID: "prepared", Observer: observe("explicit")})
if err != nil {
t.Fatal(err)
}
if len(order) != 0 {
t.Fatalf("preparation emitted events: %v", order)
}
if _, err := foreign.RunPrepared(executeCtx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
t.Fatalf("foreign claim: %v, %v", err, order)
}
if _, err := engine.RunPrepared(executeCtx, handle); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"explicit", "execution", "explicit", "execution", "explicit", "execution"}) {
t.Fatalf("observer order: %v", order)
}
order = nil
if _, err := engine.RunPrepared(executeCtx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
t.Fatalf("reused claim: %v, %v", err, order)
}
if _, err := engine.RunPrepared(executeCtx, nil); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
t.Fatalf("nil claim: %v, %v", err, order)
}
handle, err = engine.PrepareExecution(prepareCtx, promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatal(err)
}
if _, err := engine.RunPrepared(executeCtx, handle); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"execution", "execution", "execution"}) {
t.Fatalf("context-only order: %v", order)
}
}
func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) { func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) {
engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello") engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
var events []promptkit.ExecutionEvent var events []promptkit.ExecutionEvent

View File

@@ -29,6 +29,10 @@ type ProfileInspection struct {
// explicit zero values and empty collections in JSON and YAML. // explicit zero values and empty collections in JSON and YAML.
type BackendInspection struct { type BackendInspection struct {
BackendID string `json:"id" yaml:"id"` BackendID string `json:"id" yaml:"id"`
// Provider is the normalized declaration, not a manufactured runtime default.
Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"`
// ZeroCost preserves an explicitly declared false as well as true.
ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"`
Endpoint string `json:"endpoint" yaml:"endpoint"` Endpoint string `json:"endpoint" yaml:"endpoint"`
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"` APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
ConcurrencyLimit *int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit,omitempty"` ConcurrencyLimit *int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit,omitempty"`

View File

@@ -13,7 +13,7 @@ import (
func TestResourceInspectionGenericEncoding(t *testing.T) { func TestResourceInspectionGenericEncoding(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{ engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}}, "backends": []any{map[string]any{"id": "local", "provider": " VLLM ", "zero_cost": false, "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}}, "profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}},
"prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, "prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}}}) })}}})
@@ -77,7 +77,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
} }
} }
b := decode(backend) b := decode(backend)
if b["id"] != "local" || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" { if b["id"] != "local" || b["provider"] != "vllm" || b["zero_cost"] != false || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
t.Errorf("backend = %#v", b) t.Errorf("backend = %#v", b)
} }
params := b["extra_params"].(map[string]any) params := b["extra_params"].(map[string]any)
@@ -92,7 +92,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
t.Errorf("reachability = %#v", reach) t.Errorf("reachability = %#v", reach)
} }
absent(b, "extends") absent(b, "extends")
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit") absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit", "provider", "zero_cost")
p := decode(profile) p := decode(profile)
if p["id"] != "profile" || p["backend"] != "local" || p["model"] != "model" || p["endpoint"] != "" || p["api_key_env"] != "" || p["service_tier"] != "" || p["reasoning_effort"] != "" || p["api_key_required"] != false || fmt.Sprint(p["temperature"]) != "0" || fmt.Sprint(p["top_p"]) != "0" || fmt.Sprint(p["max_tokens"]) != "0" || fmt.Sprint(p["timeout_seconds"]) != "0" || len(p["extra_params"].(map[string]any)) != 0 { if p["id"] != "profile" || p["backend"] != "local" || p["model"] != "model" || p["endpoint"] != "" || p["api_key_env"] != "" || p["service_tier"] != "" || p["reasoning_effort"] != "" || p["api_key_required"] != false || fmt.Sprint(p["temperature"]) != "0" || fmt.Sprint(p["top_p"]) != "0" || fmt.Sprint(p["max_tokens"]) != "0" || fmt.Sprint(p["timeout_seconds"]) != "0" || len(p["extra_params"].(map[string]any)) != 0 {
t.Errorf("profile = %#v", p) t.Errorf("profile = %#v", p)

View File

@@ -65,3 +65,21 @@ func TestSharedSmokeSettingsWorkWithBackendSource(t *testing.T) {
t.Fatalf("backend source = %+v, %v", result, err) t.Fatalf("backend source = %+v, %v", result, err)
} }
} }
func TestBackendSourcePreservesProviderFieldPresence(t *testing.T) {
blank, disabled := "", false
result, err := configkit.Load[promptkit.Settings](promptkit.BackendSource("backends", []promptkit.BackendConfig{{ID: "omitted"}, {ID: "supplied", Provider: &blank, ZeroCost: &disabled}}))
if err != nil {
t.Fatal(err)
}
if result.Config.Backends[0].Provider != nil || result.Config.Backends[0].ZeroCost != nil {
t.Fatalf("omitted values: %+v", result.Config.Backends[0])
}
got := result.Config.Backends[1]
if got.Provider == nil || *got.Provider != "" || got.ZeroCost == nil || *got.ZeroCost {
t.Fatalf("supplied values: %+v", got)
}
if local := promptkit.LocalBackend("https://example.invalid/v1", 0); local.Provider != nil || local.ZeroCost != nil {
t.Fatalf("local defaults: %+v", local)
}
}

View File

@@ -368,6 +368,7 @@ func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedToken
return ObservedTokenUsage{ return ObservedTokenUsage{
PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens), PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens),
CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens), CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens),
ReasoningTokens: fromDomainObservedTokenCount(usage.ReasoningTokens),
TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens), TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens),
CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens), CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens),
CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens), CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens),

View File

@@ -5,13 +5,40 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"net/http"
"net/http/httptest"
"strings" "strings"
"testing" "testing"
"time" "time"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
func TestSmokeAuthenticationUsesProviderDefault(t *testing.T) {
var path string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path = r.URL.Path
fmt.Fprint(w, `{"data":{"label":"member","is_free_tier":false,"usage":0}}`)
}))
defer server.Close()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("diagnostic", map[string]any{
"backends": []any{map[string]any{"id": "renamed", "provider": "openrouter", "endpoint": server.URL + "/v1"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectBackend(context.Background(), "renamed")
if err != nil || inspection.SmokeTests != nil {
t.Fatalf("declared metadata: %+v, %v", inspection, err)
}
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "renamed"})
if err != nil || !report.Supported || !report.Completed || path != "/v1/key" || report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted {
t.Fatalf("report=%+v path=%q err=%v", report, path, err)
}
}
type gatedSmokeClient struct { type gatedSmokeClient struct {
started chan promptkit.GenerateRequest started chan promptkit.GenerateRequest
release <-chan struct{} release <-chan struct{}

View File

@@ -298,6 +298,8 @@ type RunResult struct {
InputHashes map[string]string `json:"input_hashes,omitempty"` InputHashes map[string]string `json:"input_hashes,omitempty"`
// Usage is the token accounting reported by the LLM client. // Usage is the token accounting reported by the LLM client.
Usage TokenUsage `json:"usage"` Usage TokenUsage `json:"usage"`
// CostUSD is the observed charge subtotal across dispatched attempts.
CostUSD ObservedCost `json:"cost_usd"`
// StartTime is the UTC time immediately before ordinary Run preparation or // StartTime is the UTC time immediately before ordinary Run preparation or
// after RunPrepared claims its handle. // after RunPrepared claims its handle.
StartTime time.Time `json:"start_time,omitempty"` StartTime time.Time `json:"start_time,omitempty"`
@@ -568,9 +570,11 @@ type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"` PromptTokens int `json:"prompt_tokens"`
// CompletionTokens is the reported generated-token count. // CompletionTokens is the reported generated-token count.
CompletionTokens int `json:"completion_tokens"` CompletionTokens int `json:"completion_tokens"`
// ReasoningTokens is the reported reasoning subset of completion tokens.
ReasoningTokens int `json:"reasoning_tokens"`
// TotalTokens is the reported total-token count. // TotalTokens is the reported total-token count.
TotalTokens int `json:"total_tokens"` TotalTokens int `json:"total_tokens"`
// CachedTokens is the reported cached-input-token count. // CachedTokens is the reported input-token cache-read count.
CachedTokens int `json:"cached_tokens"` CachedTokens int `json:"cached_tokens"`
// CacheWriteTokens is the reported cache-write-token count. // CacheWriteTokens is the reported cache-write-token count.
CacheWriteTokens int `json:"cache_write_tokens"` CacheWriteTokens int `json:"cache_write_tokens"`
@@ -656,7 +660,9 @@ type StructuredOutputJSONSpec struct {
// treated as known and zero counts as unknown for observation only; the legacy // treated as known and zero counts as unknown for observation only; the legacy
// interface cannot distinguish omitted zero from measured zero. An explicit // interface cannot distinguish omitted zero from measured zero. An explicit
// UsagePresence can report known zero counts. Result usage sums the supplied // UsagePresence can report known zero counts. Result usage sums the supplied
// Usage values independently of presence. // Usage values independently of presence. A non-nil CostUSD is copied and
// retained for attempt accounting even when returned alongside an error;
// non-finite values are treated as unknown before configured-zero fallback.
type LLMClient interface { type LLMClient interface {
Generate(context.Context, GenerateRequest) (*GenerateResponse, error) Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
} }
@@ -690,6 +696,8 @@ type GenerateResponse struct {
// UsagePresence states which usage fields were reported, including zeros. // UsagePresence states which usage fields were reported, including zeros.
// Nil uses the legacy observation fallback described by LLMClient. // Nil uses the legacy observation fallback described by LLMClient.
UsagePresence *TokenUsagePresence `json:"usage_presence,omitempty"` UsagePresence *TokenUsagePresence `json:"usage_presence,omitempty"`
// CostUSD is an optional reported charge; a pointer to zero reports zero.
CostUSD *float64 `json:"cost_usd,omitempty"`
} }
// TokenUsagePresence identifies provider-reported token counts for observation. // TokenUsagePresence identifies provider-reported token counts for observation.
@@ -697,6 +705,7 @@ type GenerateResponse struct {
type TokenUsagePresence struct { type TokenUsagePresence struct {
PromptTokens bool `json:"prompt_tokens"` PromptTokens bool `json:"prompt_tokens"`
CompletionTokens bool `json:"completion_tokens"` CompletionTokens bool `json:"completion_tokens"`
ReasoningTokens bool `json:"reasoning_tokens"`
TotalTokens bool `json:"total_tokens"` TotalTokens bool `json:"total_tokens"`
CachedTokens bool `json:"cached_tokens"` CachedTokens bool `json:"cached_tokens"`
CacheWriteTokens bool `json:"cache_write_tokens"` CacheWriteTokens bool `json:"cache_write_tokens"`