28 Commits

Author SHA1 Message Date
207677e994 Prepare the v0.25.0 release documentation 2026-11-22 19:08:29 +00:00
20c5ea47af Document invocation observation and cost reporting 2026-11-22 18:38:11 +00:00
d4509b163e Cover concurrent multi-provider invocation accounting 2026-11-22 18:32:29 +00:00
ba4de9d11e Isolate cost accounting for prepared executions 2026-11-22 18:30:02 +00:00
d77e3e2f83 Report costs for ordinary runs 2026-11-22 18:28:39 +00:00
81a2a7591d Collect costs for dispatched model attempts 2026-11-22 18:27:01 +00:00
cd0e1f5d39 Capture optional provider reported costs 2026-11-22 18:25:32 +00:00
6ed0e037c4 Define observed cost values and aggregation 2026-11-22 18:24:33 +00:00
3d82cad2c9 Aggregate reasoning usage and derive cache hit ratios 2026-11-22 18:22:07 +00:00
db1cad6d2d Decode provider reasoning and cache usage details 2026-11-22 18:20:11 +00:00
c3257b63f0 Add vLLM models authentication diagnostics 2026-11-22 18:17:30 +00:00
ca8cf1b827 Resolve authentication diagnostics by provider 2026-11-22 18:16:13 +00:00
adc700f96e Capture provider policy in backend execution targets 2026-11-22 18:13:17 +00:00
6056f93e55 Add backend provider and cost configuration 2026-11-22 18:11:10 +00:00
f2b0ed8d6c Select prepared execution observers at run time 2026-11-22 18:08:28 +00:00
ea7cc170ad Select execution observers from context 2026-11-22 18:07:17 +00:00
f762c0358e Plan invocation observation and usage implementation 2026-11-22 18:01:31 +00:00
a47df4ec20 Consolidate the observation roadmap around the agreed target state 2026-11-22 17:52:58 +00:00
fed9ad082d Clarify inspection and diagnostic accounting roadmap 2026-11-22 17:16:14 +00:00
4ee64b6eda Add explicit zero-cost declarations and cost-source reporting to the roadmap 2026-11-22 17:11:57 +00:00
440d9a2c88 Add vLLM provider conventions to the observation roadmap 2026-11-22 15:57:37 +00:00
23ad9e2a26 Add reasoning and cache usage reporting to the roadmap 2026-11-22 15:54:02 +00:00
80649c579d Define the default OpenAI-compatible provider in the roadmap 2026-11-22 15:49:17 +00:00
0254eb1b5b Clarify provider selection and diagnostic defaults in the roadmap 2026-11-22 15:41:20 +00:00
a10d1a2245 Select provider conventions through backend configuration 2026-11-22 15:27:12 +00:00
0564c8086c Clarify observation accounting and implementation planning 2026-11-22 10:02:22 +00:00
03209947f6 Add optional OpenRouter cost reporting to the observation roadmap 2026-11-22 09:48:38 +00:00
1070be8d89 Define invocation-scoped observation roadmap 2026-11-22 09:39:27 +00:00
73 changed files with 1908 additions and 140 deletions

View File

@@ -45,6 +45,12 @@ boundary and constraints that framework work must preserve.
## Release Guidance
The [v0.25.0 release notes](docs/releases/v0.25.0.md) cover invocation-scoped
observation, provider conventions, reasoning and cache usage, and optional USD
cost reporting. Upgrading consumers should review provider selection,
authentication diagnostic defaults, and the corrected OpenRouter cache-write
mapping.
The [v0.24.0 release notes](docs/releases/v0.24.0.md) cover application-owned
configuration loading, prompt-owned schemas, default output repair, and the
prepared message-content hash. Upgrading consumers should review the breaking

View File

@@ -26,6 +26,10 @@ type BackendConfig struct {
ID string
// Extends names a sibling backend to inherit through ConfigKit.
Extends *string
// Provider selects the backend's supported API conventions.
Provider *string
// ZeroCost enables a zero-cost fallback when no usable charge is reported.
ZeroCost *bool
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
Endpoint *string
// APIKeyEnv optionally names an environment variable, not a credential.
@@ -55,6 +59,12 @@ func BackendSource(label string, backends []BackendConfig) configkit.Source {
if config.Extends != nil {
fields["extends"] = map[string]any{"id": *config.Extends}
}
if config.Provider != nil {
fields["provider"] = *config.Provider
}
if config.ZeroCost != nil {
fields["zero_cost"] = *config.ZeroCost
}
if config.Endpoint != nil {
fields["endpoint"] = *config.Endpoint
}

View File

@@ -0,0 +1,70 @@
package promptkit_test
import (
"context"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestDuplicateContextAndRequestGroupEnrollment(t *testing.T) {
charge := .5
engine := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe}); err != nil {
t.Fatal(err)
}
got := group.Snapshot()
if len(got.Members) != 1 || got.Attempts != 1 || got.CostUSD.Value != .5 {
t.Fatalf("summary=%+v", got)
}
}
func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) {
known := func(v int) promptkit.ObservedTokenCount {
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
}
for _, tc := range []struct {
name string
input, reads promptkit.ObservedTokenCount
ratio float64
ok bool
}{
{name: "fraction", input: known(8), reads: known(3), ratio: .375, ok: true},
{name: "zero reads", input: known(8), reads: known(0), ok: true},
{name: "zero input", input: known(0), reads: known(0)},
{name: "negative reads", input: known(8), reads: known(-1)},
{name: "reads exceed input", input: known(8), reads: known(9)},
{name: "unknown", input: known(8)},
{name: "incomplete", input: known(8), reads: promptkit.ObservedTokenCount{Value: 3, Known: true}},
} {
t.Run(tc.name, func(t *testing.T) {
usage := promptkit.ObservedTokenUsage{PromptTokens: tc.input, CachedTokens: tc.reads}
got, ok := usage.CacheHitRatio()
if ok != tc.ok || math.Abs(got-tc.ratio) > 1e-12 {
t.Fatalf("ratio=%v available=%t", got, ok)
}
})
}
}
func TestGroupCacheRatioUsesSummedTokensAndDeduplicatesTerminals(t *testing.T) {
known := func(v int) promptkit.ObservedTokenCount {
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
}
var group promptkit.ExecutionGroup
first := promptkit.ExecutionEvent{RunID: "a", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(10), CachedTokens: known(10), ReasoningTokens: known(1)}}
second := promptkit.ExecutionEvent{RunID: "b", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(90), CachedTokens: known(0), ReasoningTokens: known(2)}}
group.Observe(first)
group.Observe(first)
group.Observe(second)
snapshot := group.Snapshot()
ratio, ok := snapshot.Usage.CacheHitRatio()
if !ok || math.Abs(ratio-.1) > 1e-12 || snapshot.Usage.ReasoningTokens != known(3) {
t.Fatalf("summary=%+v ratio=%v available=%t", snapshot, ratio, ok)
}
}

View File

@@ -134,6 +134,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
InputHashes: copyStringMap(result.InputHashes),
Usage: fromDomainTokenUsage(result.Usage),
CostUSD: fromDomainCost(result.CostUSD),
StartTime: result.StartTime,
EndTime: result.EndTime,
Duration: result.Duration,
@@ -158,6 +159,7 @@ func toDomainGenerateResponse(resp *GenerateResponse) *domain.GenerateResponse {
Content: resp.Content,
Usage: toDomainTokenUsage(resp.Usage),
UsagePresence: toDomainTokenUsagePresence(resp.UsagePresence),
CostUSD: copyFloat64Ptr(resp.CostUSD),
}
}
@@ -168,6 +170,7 @@ func toDomainTokenUsagePresence(presence *TokenUsagePresence) *domain.TokenUsage
return &domain.TokenUsagePresence{
PromptTokens: presence.PromptTokens,
CompletionTokens: presence.CompletionTokens,
ReasoningTokens: presence.ReasoningTokens,
TotalTokens: presence.TotalTokens,
CachedTokens: presence.CachedTokens,
CacheWriteTokens: presence.CacheWriteTokens,
@@ -265,7 +268,7 @@ func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
}
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, Provider: copyStringPtr(v.Provider), ZeroCost: copyBoolPtr(v.ZeroCost), APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
if v.ExtraParams != nil {
data := copyAnyMap(v.ExtraParams)
out.ExtraParams = &data
@@ -356,6 +359,7 @@ func fromDomainTokenUsage(usage domain.TokenUsage) TokenUsage {
return TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,
@@ -366,6 +370,7 @@ func toDomainTokenUsage(usage TokenUsage) domain.TokenUsage {
return domain.TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
ReasoningTokens: usage.ReasoningTokens,
TotalTokens: usage.TotalTokens,
CachedTokens: usage.CachedTokens,
CacheWriteTokens: usage.CacheWriteTokens,

View File

@@ -3,6 +3,7 @@ package promptkit_test
import (
"context"
"errors"
"reflect"
"testing"
"testing/synctest"
"time"
@@ -10,6 +11,31 @@ import (
"go.maximumdirect.net/promptkit"
)
func TestCoordinateObserverOrderWithContext(t *testing.T) {
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
if err != nil {
t.Fatal(err)
}
var order []string
observer := func(name string) promptkit.ExecutionObserver {
return func(promptkit.ExecutionEvent) { order = append(order, name) }
}
ctx := promptkit.WithExecutionObserver(context.Background(), observer("context"))
_, err = engine.Coordinate(ctx, promptkit.CoordinateRequest{Start: initial.Exchange, AdditionalTurns: 1, Observer: observer("coordinator"), Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
if turn.RemainingTurns == 0 {
return nil, nil
}
return &promptkit.RunRequest{PromptID: "prompt", Observer: observer("request")}, nil
}})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"request", "coordinator", "context", "request", "coordinator", "context", "request", "coordinator", "context"}) {
t.Fatalf("observer order = %v", order)
}
}
func TestCoordinateUsesIndependentOperationBudgets(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
engine := newBackendCapacityEngine(t, observationClientFunc(func(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {

42
cost.go Normal file
View File

@@ -0,0 +1,42 @@
package promptkit
import "go.maximumdirect.net/promptkit/internal/domain"
// CostSource identifies how an observed USD cost was supplied.
type CostSource string
const (
// CostSourceNone denotes no known sourced contribution.
CostSourceNone CostSource = ""
// CostSourceProvider denotes a reported provider or injected-client amount.
CostSourceProvider CostSource = "provider"
// CostSourceConfiguredZero denotes an explicitly configured zero fallback.
CostSourceConfiguredZero CostSource = "configured_zero"
// CostSourceMixed denotes both provider and configured-zero contributions.
CostSourceMixed CostSource = "mixed"
)
// ObservedCost reports an available USD subtotal and its coverage and source.
// Known means at least one contribution is available. Complete means all
// included dispatched attempts have an amount under the selected policy; it
// does not verify billing. A completed run with no dispatched attempts, or an
// empty group, has known complete zero with no source. A group with members but
// no terminals has unknown cost. Amounts are unrounded binary floating point.
type ObservedCost struct {
// Value is the known USD subtotal.
Value float64 `json:"value"`
// Known reports whether a subtotal is available.
Known bool `json:"known"`
// Complete reports whether all included dispatched attempts are covered.
Complete bool `json:"complete"`
// Source summarizes the known contributions.
Source CostSource `json:"source"`
}
func fromDomainCost(value domain.ObservedCost) ObservedCost {
return ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: CostSource(value.Source)}
}
func toDomainCost(value ObservedCost) domain.ObservedCost {
return domain.ObservedCost{Value: value.Value, Known: value.Known, Complete: value.Complete, Source: domain.CostSource(value.Source)}
}

View File

@@ -0,0 +1,131 @@
package promptkit_test
import (
"context"
"errors"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestPreparedCostAndRejectedClaims(t *testing.T) {
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "bad"}, nil
}
return nil, errors.New("repair failed")
})))
ctx := context.Background()
var events []promptkit.ExecutionEvent
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
if err != nil || len(events) != 0 {
t.Fatalf("prepare error=%v events=%+v", err, events)
}
result, err := engine.RunPrepared(ctx, handle)
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
}
events = nil
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
t.Fatalf("reused err=%v events=%+v", err, events)
}
}
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
childCalls := 0
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
childCalls++
charge := .2
if childCalls == 2 {
charge = .3
}
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
var childRun, childPrepared *promptkit.RunResult
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
var err error
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
return nil, err
}
childPrepared, err = child.RunPrepared(ctx, handle)
if err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
}
summary := group.Snapshot()
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 2.0
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
})))
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
summary := group.Snapshot()
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
}
}
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := .5
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var childResult *promptkit.RunResult
var childErr error
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
}})
if err != nil {
t.Fatal(err)
}
var group promptkit.ExecutionGroup
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
}
summary := group.Snapshot()
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}

127
cost_run_contract_test.go Normal file
View File

@@ -0,0 +1,127 @@
package promptkit_test
import (
"context"
"errors"
"io"
"math"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type costRoundTrip func(*http.Request) (*http.Response, error)
func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
options = append(options, promptkit.WithBundledOpenRouter(false))
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...)
if err != nil {
t.Fatal(err)
}
return engine
}
func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
for _, observed := range []bool{false, true} {
t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) {
calls := 0
transport := costRoundTrip(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil
})
engine := costEngine(t, false, &http.Client{Transport: transport})
var events []promptkit.ExecutionEvent
req := promptkit.RunRequest{PromptID: "prompt"}
if observed {
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
}
result, err := engine.Run(context.Background(), req)
want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider}
if err != nil || result == nil || result.CostUSD != want || calls != 2 {
t.Fatalf("result=%+v calls=%d err=%v", result, calls, err)
}
if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) {
t.Fatalf("events=%+v", events)
}
})
}
}
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
charge := .125
invalid := math.Inf(1)
for _, tc := range []struct {
name string
response *promptkit.GenerateResponse
err error
want promptkit.ObservedCost
}{
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
{"nonfinite report falls back", &promptkit.GenerateResponse{Content: "ok", CostUSD: &invalid}, nil, promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
} {
t.Run(tc.name, func(t *testing.T) {
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return tc.response, tc.err
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
})
}
}
func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) {
charge := .1
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "invalid"}, nil
}
return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed}
if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want {
t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err)
}
}
func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) {
engine := costEngine(t, true, nil)
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
}

View File

@@ -728,6 +728,40 @@ counts, but its zero counts remain unknown to observations; supply explicit
presence when measured zeros matter. Existing `RunResult.Usage` sums returned
values independently of observation presence.
For an application-owned invocation spanning several calls or engines, attach
an observer once to a context and propagate it to those calls:
```go
var group promptkit.ExecutionGroup
ctx = promptkit.WithExecutionObserver(ctx, group.Observe)
result, err := engine.Run(ctx, request)
// After joining all application-owned work, inspect group.Snapshot(), even on error.
```
Explicit request observers run first, followed by contextual observers in
registration order. Coordinated continuations pick up the context through
`Run`; prepared handles retain the request observer but use the execution
context's observers, not the preparation context's. Each nested call has its
own run ID and accounting. Duplicate registration can deliver duplicate events;
groups count each run's first terminal once.
Reasoning tokens are a subset of completion tokens; cache reads and writes are
input details, not extra tokens. `ObservedTokenUsage.CacheHitRatio()` derives
the cache-read fraction only when input and read totals are known, complete,
positive-denominator, and consistent. It divides aggregate counts rather than
averaging per-run ratios and is independent of cache writes.
`RunResult.CostUSD` is available even without an observer; attempt and terminal
events and group snapshots also report cost. `Known` denotes an available
subtotal, `Complete` covers dispatched attempts, and `Source` distinguishes
provider reports, an explicitly configured zero fallback, and mixed sources.
Reported cost, including zero, takes precedence over fallback. Unknown failed
attempts make a subtotal incomplete; configured zero may cover them. Terminal
events can report cost when a failed operation returns no result. Neither
complete reporting nor provider attribution verifies an invoice; consumers
choose presentation precision. See [backend definitions](../formats.md#backend-definitions)
and the [integration contract](../integrations/openai-compatible-chat.md#response-handling).
Callbacks run synchronously, in order for one operation, and can overlap for
different operations. Synchronize shared callback state and avoid blocking on
work that needs the same run's admission slot. Callback panics propagate;
@@ -879,15 +913,15 @@ outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
})
```
`assess` is application code and explicitly selects its observer on any nested
run. A completed turn releases its admission lease before the callback, so that
`assess` is application code and can select its observer explicitly or inherit
one from the context. A completed turn releases its admission lease before the callback, so that
assessment may use the same backend. `stopped` and `exhausted` describe control
flow, not content acceptance. Cancellation before callback delivery can return one
completed response in `Undelivered`; errors retain their ordinary identities.
### Collect Selected Execution Facts
Groups are independent of continuation and enrollment is explicit:
Groups are independent of continuation and can be enrolled explicitly:
```go
var group, validatorGroup promptkit.ExecutionGroup
@@ -904,7 +938,7 @@ _, _ = summary, validatorSummary
```
Only the first terminal event per run contributes counts and usage; coverage of
currently observed members is separate from token-usage completeness. Empty
currently observed members is separate from token and cost completeness. Empty
groups report complete, known zero totals, while members without terminals do
not invent totals. Overlapping groups are not generally additive. See the
[offline example](../../examples/go-library/observe/main.go) for complete setup.

View File

@@ -223,7 +223,7 @@ Backend definitions are supplied through caller-selected configuration
sources. The conventional backend IDs are names only; PromptKit does not
install definitions for them.
A backend definition may have `endpoint`, `api_key_env`,
A backend definition may have `provider`, `zero_cost`, `endpoint`, `api_key_env`,
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
user information, query, or fragment. An omitted or null API-key environment
@@ -237,6 +237,17 @@ unrecognized backend field is application data and is not automatically redacted
by Promptkit. Applications own sensitivity declarations and disclosure policy
for their extension fields.
`provider` selects response and diagnostic conventions independently of the
backend ID, endpoint, or model. Its final value is trimmed and lowercased;
omitted, null, or blank selects `openai-compatible` at runtime. Supported
nonblank values are `openai-compatible`, `openrouter`, and `vllm`. `zero_cost`
is an optional Boolean: true supplies a zero-cost fallback when a dispatched
attempt has no usable reported cost; omitted, null, and false disable it.
Neither setting is sent to the provider. They follow normal layering and
inheritance, and endpoint overrides retain the selected backend's policy.
Resource inspection shows declared normalized values without inventing runtime
defaults; configuration inspection retains source history.
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
It may contain `default_profile`, `reachability`, and `authentication`:
@@ -253,7 +264,7 @@ backends:
endpoint: https://openrouter.ai/api/v1/key
```
An omitted or null `smoke_tests` block disables these metadata defaults.
An omitted or null `smoke_tests` block removes its declared metadata.
`default_profile` is preserved as an exact, nonblank profile ID; its existence
is checked only if an inference diagnostic selects that default. Omitted/null
`default_profile` means there is no inference default. Omitted/null
@@ -262,12 +273,22 @@ reachability endpoint is trimmed and must be an absolute HTTP(S) URL with a
host; unlike the backend base endpoint, its path and query are allowed. Null
means fallback, while an explicit blank value is invalid.
Omitted/null `authentication` means no authentication adapter is declared. A
non-null object requires `kind`, which is trimmed and case-normalized; the
recognized values are `none` and `openrouter_key`. Its omitted/null endpoint
Omitted/null `authentication` leaves the provider's diagnostic default in
effect; `openrouter` defaults to `openrouter_key`, `vllm` defaults to
`vllm_models`, and the generic provider has no default. A non-null object's
optional `kind` is trimmed and case-normalized; accepted
values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint
uses the adapter's backend-relative destination (`openrouter_key` appends
`/key` to the effective backend base); an explicit endpoint follows the same
full-URL rules as reachability. `none` declares authentication inapplicable.
Without a selected adapter, authentication diagnostics report an unsupported
selection. An endpoint-only authentication
block overrides the destination of a provider default; `kind: none` suppresses
that default.
Previously, a supplied authentication object required `kind`. Endpoint-only
objects now use a provider default where available; clearing the object
restores that default rather than suppressing it. Use `kind: none` to suppress
authentication testing.
These values are configuration only: engine construction does not contact a
backend.
@@ -358,7 +379,7 @@ does not disable the caller context or HTTP transport timeout.
### Bundled OpenRouter Configuration
The default bundle installs root backend `openrouter`, endpoint
The default bundle installs root backend `openrouter`, provider `openrouter`, endpoint
`https://openrouter.ai/api/v1`, optional credential environment name
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback

View File

@@ -58,7 +58,7 @@ report facts, and injected-client behavior.
HTTP-only diagnostics use the configured built-in HTTP client. Reachability is
one unauthenticated `GET` to the selected diagnostic URL; an observed response
counts as reachable even when its status is non-2xx. Authentication behavior
is selected from backend metadata, not inferred from the host. The
is selected from backend metadata and provider defaults, not inferred from the host. The
`openrouter_key` adapter makes one `GET` to `/key` relative to the effective
backend base, unless configuration supplies a full diagnostic endpoint (see
the [backend format reference](../formats.md#backend-definitions)). It may send
@@ -73,6 +73,15 @@ not validation of the full account document. Other statuses and unrecognized
recognition allowance reports `authentication_response_limit_exceeded`, leaving
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
The `vllm_models` adapter issues one `GET` to `/models` relative to the
configured base unless an explicit diagnostic endpoint is supplied. It sends
an available credential but does not require one. A 200 response is recognized
when it is a JSON object with `object: "list"` and an array-valued `data`;
empty arrays are accepted. A 401 means rejection; other responses do not
establish acceptance. Recognition uses the same bounded response handling as
the key probe. Acceptance does not establish that authentication is enforced or
that inference is authorized.
Diagnostic HTTP evidence describes the last request observed at Promptkit's
RoundTripper boundary and any response associated with it. It includes the
selected destination without URL user information, response status, and
@@ -154,7 +163,22 @@ data, a second JSON value, absent choices, missing content, `null` content,
non-string content, and size overflow are malformed responses and return no
partial result.
Each of the five token fields retains whether the provider reported a
For OpenRouter and vLLM, the client also reads reasoning tokens from
`usage.completion_tokens_details.reasoning_tokens`. OpenRouter cache writes
come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
`usage.prompt_tokens_details.created_cache_tokens`. The generic provider keeps
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
fractional, or unrepresentable new details remain unknown without invalidating
otherwise usable content. Existing common count decoding remains strict.
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
amounts are ignored without invalidating content. The generic and vLLM
providers do not interpret that field.
vLLM prompt-token details depend on server configuration, including
`--enable-prompt-tokens-details`; reasoning details depend on a configured
reasoning parser. Servers without these details remain usable. vLLM cache
writes describe local prefix-cache creation, not a billable write charge.
Each common token field retains whether the provider reported a
non-null integer, including zero. Omitted and `null` fields are unknown for
observations. A strictly decoded successful envelope can yield observed usage
even when a later missing choice or content makes the attempt malformed.

View File

@@ -79,7 +79,7 @@ other backend IDs.
The coordinator does not retry injected clients; it normalizes a nil successful
response as malformed generation. The built-in HTTP client remains responsible
for provider transport behavior. The runner, rather than the
capacity layer, sums all five usage fields from the initial response and every
capacity layer, sums usage fields from the initial response and every
completed repair response into the successful run result.
An invocation-local observer receives attempt events after active permit

View File

@@ -35,6 +35,12 @@ unknown usage. Transport, timeout, malformed, and noneligible status failures
stop the sequence without retry exhaustion. Context is checked before each
wait, acquisition, and dispatch.
At every dispatched completion the coordinator resolves any reported cost
against the frozen backend zero-cost policy and adds it once to the run's
[cost accumulator](observation.md), independently of observer selection.
Injected responses may retain reported cost alongside an error. Undispatched
acquisitions and retry waits contribute no cost.
The public client adapter translates explicitly supplied public transport and
malformed-response identities into internal categories while retaining the
original error chain. This keeps injected attempt classifications consistent

View File

@@ -142,6 +142,14 @@ known. Strictly decoded usage can still be observed if later choice/content
checks fail; invalid framing or a non-success error body contributes none.
Existing successful-result accounting remains value-based.
The captured provider selects additional response details and authentication
diagnostic defaults. Optional new reasoning, cache-write, and OpenRouter cost
fields are decoded independently, without relaxing common-field or framing
validation. Cost remains an optional raw amount at this boundary; shared
accounting applies configured-zero fallback after dispatch. The integration
contract specifies the [wire mappings](../integrations/openai-compatible-chat.md#response-handling)
and [probe recognition](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
For a non-success response, `ProviderHTTPError` retains the HTTP status and
only normalized detail from the bounded recognized envelope. It retains
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;

View File

@@ -18,11 +18,14 @@ Timeout attribution is finalized and the child scope released before terminal
delivery, so observer latency cannot change the already selected outcome.
Prepared-content notification is a separate opt-in callback, not an event.
The facade masks inherited recorder and prepared-claim state when entering
The facade masks inherited recorder, cost, and prepared-claim state when entering
preparation or execution. Delegated calls from an injected collaborator retain
the caller's cancellation, deadline, and unrelated context values, but do not
reuse the enclosing invocation's accounting. Only the new request's observer
(or the claimed handle's retained observer) enables its observation lifecycle.
reuse the enclosing invocation's accounting. Ordinary runs compose the request
observer before any observers selected by the invocation context; contextual
selection survives this boundary. Prepared executions compose the retained
request observer with selection from the execution context after a successful
claim; the preparation context does not select execution observers.
A recorder keeps only counters, resolved identity fields, and the selected
callback. It has no global history, queue, goroutine, or mutex held across a
@@ -53,12 +56,24 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its
attempt events.
An execution group is a zero-ready synchronized collector of explicitly selected
Ordinary runs and claimed prepared executions install invocation-local cost
accounting even without an observer.
The generation coordinator resolves cost once per dispatched attempt and adds
it to that state independently of event delivery. The result and recorder read
the same accumulator; groups merge first terminal costs independently of
terminal coverage.
Reasoning counts use the same per-field rules and are a subset of completion
tokens. The public cache-hit helper derives a ratio from complete, consistent
summed input and cache-read counts; it does not aggregate ratios or use writes.
An execution group is a zero-ready synchronized collector of selected
events. It enrolls by run ID and aggregates each run's first terminal counters and
presence-aware usage exactly once. Snapshots copy members and totals under one
lock; terminal coverage and token completeness are distinct. Observer fan-out
copies the callback slice and invokes callbacks synchronously without holding a
group lock. No group history, registry, or automatic nested membership exists.
group lock. No group history or registry exists; contextual selection can enroll
nested ordinary runs that use the scoped context.
Smoke diagnostics mask inherited execution-recorder state while invoking
generation or HTTP collaborators. Their private attempt evidence is assembled

View File

@@ -21,7 +21,7 @@ contributor workflow and validation.
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded retries, per-attempt cost, ordinary observations, and separate diagnostic evidence. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
@@ -30,8 +30,8 @@ contributor workflow and validation.
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible requests, provider-specific response interpretation and authentication probes, bounded decoding, deadlines, and diagnostic evidence. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware token and cost accounting, contextual observer selection, synchronous event delivery, and terminal-based group aggregation; smoke diagnostics retain separate evidence. | [Internal execution observation](observation.md) |
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The internal engine assembles these internal components without exposing their

View File

@@ -216,10 +216,16 @@ ordinary `Run` reuses its run ID and carries it to the generation coordinator
through context. Prompt and profile identities are enriched only after their
respective resolutions succeed, before admission. The facade starts observation
before request conversion and emits its terminal event after public error
mapping and after the runner's deferred admission release. Preparation and
prepared execution follows the same terminal mapping after a successful claim;
mapping and after the runner's deferred admission release. Preparation emits no
events; prepared execution follows the same terminal mapping after a successful claim;
invalid or discarded handles do not acquire an observer.
Ordinary runs establish fresh private cost state before observation; a claimed
prepared execution establishes its own state after claim. Contextual observers
remain available at public boundaries, while enclosing recorder and cost state
are masked. Prepared handles retain the request observer and frozen provider
policy, not preparation-context selection or an accumulator.
The facade establishes an optional operation scope for ordinary runs before
observation and preparation. A claimed prepared execution establishes its own
scope from the retained duration; preparation and idle time do not consume it.
@@ -273,11 +279,13 @@ run identifier, and UTC timing. The same effective session reaches initial
generation and any repair attempt through the rendered prompt. The same
effective target and presence metadata, including backend identity and direct
credential during execution, reaches generation and every repair attempt.
Result usage is the field-wise sum of all five usage values from the initial
Result usage is the field-wise sum of usage values from the initial
response and every completed repair response. Final raw output, artifact, and
validation state still come from the last candidate. A repair error returns no
partial run result or partial result usage, although the invocation-local
observer retains completed-attempt usage and reports it in the terminal event.
Result cost instead snapshots the shared accumulator across all dispatched
attempts, including retries and repairs, whether or not observation is enabled.
## Failure Categories

88
docs/releases/v0.25.0.md Normal file
View File

@@ -0,0 +1,88 @@
# Promptkit v0.25.0
This minor release adds invocation-scoped execution observation, provider-specific
reasoning and cache usage, and optional USD cost reporting. Backend definitions
can now select API conventions independently of their names and endpoints.
Consumers upgrading from v0.24.x should review the configuration and accounting
changes below.
## Added And Changed
- `WithExecutionObserver` selects observation through a caller-owned context
across nested, concurrent, coordinated, and multi-engine calls. Request
observers run before contextual observers. Prepared handles retain their
explicit request observer but select contextual observers when executed,
not when prepared. Each execution retains independent accounting; groups
count each run's first terminal once. See the
[observation guide](../consumers/pkg-promptkit.md#observe-an-ordinary-run)
and [offline example](../../examples/go-library/observe/main.go).
- Backend `provider` selects `openai-compatible`, `openrouter`, or `vllm`
conventions. Omitted, null, or blank selects the generic OpenAI-compatible
behavior. The bundled OpenRouter backend explicitly selects `openrouter`.
`InspectBackend` exposes declared provider and `zero_cost` settings without
manufacturing runtime defaults. See the
[backend reference](../formats.md#backend-definitions).
- Usage values, presence indicators, observations, and group summaries include
`ReasoningTokens`. OpenRouter and vLLM decoding includes their respective
reasoning, cache-read, and cache-write fields. Reasoning is a subset of
completion tokens; cache activity does not add to reported token totals.
`ObservedTokenUsage.CacheHitRatio` derives a ratio from complete, consistent
input and cache-read totals. See the
[response contract](../integrations/openai-compatible-chat.md#response-handling).
- `RunResult.CostUSD`, execution events, and group summaries report a USD
subtotal with availability, completeness, and source metadata. OpenRouter
supplies optional `usage.cost`; injected clients can supply
`GenerateResponse.CostUSD`. Explicit backend `zero_cost: true` supplies a
zero fallback when no usable amount is reported. A reported amount, including
zero, takes precedence. See the [cost GoDoc](../../cost.go).
- Authentication diagnostics can use provider defaults: `openrouter_key` for
OpenRouter and the new `vllm_models` adapter for vLLM. The latter probes
base-relative `/models` and reports recognized response facts without
asserting authentication enforcement or inference authorization. See the
[diagnostic integration contract](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
## Compatibility And Upgrade
- For application-defined OpenRouter backends, set `provider: openrouter` to
enable OpenRouter-specific usage and cost decoding and its authentication
default. Backend IDs and endpoint hostnames do not select conventions.
Generic backends retain their existing request and token conventions and
do not decode monetary cost. Unsupported nonblank provider names are
configuration errors; supplied names are trimmed and lowercased.
- OpenRouter cache-write reporting now reads
`usage.prompt_tokens_details.cache_write_tokens`, not the generic top-level
`usage.cache_write_tokens` field. vLLM uses
`usage.prompt_tokens_details.created_cache_tokens`. Missing or unusable new
details remain unknown rather than invalidating otherwise usable content.
Availability depends on the provider and server settings; consult the
[response contract](../integrations/openai-compatible-chat.md#response-handling).
- A supplied authentication object no longer requires `kind`: an omitted or
null kind uses the provider default. Clearing the authentication block also
restores that default. Set `kind: none` to declare authentication testing
inapplicable. The generic provider has no authentication default; explicit
adapters remain supported independently of provider selection. Probes still
run only when explicitly requested. See the
[backend reference](../formats.md#backend-definitions).
- Cost includes all dispatched attempts, including transient retries and
structural repairs, and is available on successful results without an
observer. On operational failure, observations can retain cost even though
no partial result is returned. Existing `RunResult.Usage` continues to sum
successful generation responses; presence-aware observations cover all
dispatched attempts. Do not add terminal totals to attempt totals.
- Unknown cost is distinct from reported or configured zero. No backend is
automatically declared zero-cost, and no token-rate estimation is performed.
Group cost completeness describes included terminals, independently of
terminal coverage. Consumers own presentation and interpretation; complete
reporting does not verify billing.
- Result JSON adds `cost_usd`, and token JSON adds `reasoning_tokens`.
Update strict decoders if necessary. Older result JSON without cost decodes
to unknown cost. Consumers using positional literals for expanded public
structs may need to update them. See the [public types](../../types.go).
The Go consumer minimum remains 1.27.0; dependencies are unchanged from v0.24.0.
```sh
go get go.maximumdirect.net/promptkit@v0.25.0
```
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/COPYRIGHT) | [LICENSE](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/LICENSE) | [ATTRIBUTION](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.25.0/docs/oss/attribution.md)

View File

@@ -535,7 +535,8 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr
// returns no handle on error. A nil Engine returns an error matching
// ErrInvalidConfig. The effective transient retry budget is frozen privately
// for RunPrepared and is absent from Details. A supplied Observer is retained
// privately for RunPrepared; preparation itself emits no events.
// privately for RunPrepared; preparation itself emits no events and does not
// retain contextual observers.
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
// starting a timer or invoking the notification during preparation.
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
@@ -602,16 +603,24 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
// result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
ctx = observation.WithoutExecutionState(ctx)
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
defer scope.Close()
ctx = scope.Context
if req.Observer == nil {
var explicit domain.ExecutionObserver
if req.Observer != nil {
explicit = func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) }
}
observer := observation.Compose(explicit, observation.ObserverFromContext(ctx))
if observer == nil {
result, err := e.run(ctx, req)
err = mapPublicError(scope.Finalize(err))
scope.Close()
return result, err
}
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
recorder, idErr := observation.New(observer)
recorder.SetCostAccumulator(cost)
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
}
@@ -682,10 +691,11 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
// content-validation rejection, including repair exhaustion, is returned in
// RunResult, not as an operational
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An
// operational error returns no partial RunResult. A retained Observer begins
// only after a successful atomic claim, with identities from the frozen
// snapshot, and receives a terminal event on normal return after public error
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
// operational error returns no partial RunResult. The retained Observer and
// execution-context observers begin only after a successful atomic claim, with
// identities from the frozen snapshot, and receive a terminal event on normal
// return after public error mapping. Invalid, foreign, discarded, or reused
// handles emit no events.
// Callbacks run outside the handle and capacity locks; panics propagate and
// do not promise a terminal event.
// The retained RunRequest.OperationTimeout starts after claim; the retained

View File

@@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error)
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
body, _ := json.Marshal(map[string]any{
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens},
"usage": map[string]any{
"prompt_tokens": promptTokens, "completion_tokens": completionTokens,
"prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1},
"completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01,
},
})
return httpResponse(http.StatusOK, string(body))
}
@@ -80,6 +84,10 @@ type summary struct {
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"`
GroupAttempts uint64 `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"`
}
@@ -97,6 +105,8 @@ func run() (summary, error) {
filesystem := configkit.NewFilesystem("example schemas", assets)
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
backend.ID = backendID
provider, zeroCost := "openrouter", true
backend.Provider, backend.ZeroCost = &provider, &zeroCost
inputs = append(inputs,
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
@@ -117,8 +127,9 @@ func run() (summary, error) {
var proposalLog eventLog
var executions promptkit.ExecutionGroup
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe)
proposal, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record,
})
if err != nil {
return summary{}, err
@@ -133,8 +144,8 @@ func run() (summary, error) {
var assessmentLog eventLog
var preparedDetails promptkit.PreparedRun
decision := "rejected"
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
Start: proposal.Exchange, AdditionalTurns: 1,
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
if turn.Result == nil {
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
@@ -142,7 +153,7 @@ func run() (summary, error) {
if !turn.Result.Validation.IsValid {
return nil, fmt.Errorf("follow-up failed structural validation")
}
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record})
if err != nil {
return nil, err
}
@@ -181,6 +192,8 @@ func run() (summary, error) {
return summary{}, err
}
group := executions.Snapshot()
_, ratioAvailable := group.Usage.CacheHitRatio()
assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio()
return summary{
ProposalAttempts: proposalTerminal.Attempts,
ProposalTransientRetries: proposalTerminal.TransientRetries,
@@ -192,6 +205,10 @@ func run() (summary, error) {
ExhaustedOutcome: exhausted.Outcome,
GroupMembers: len(group.Members),
GroupAttempts: group.Attempts,
GroupCost: group.CostUSD,
GroupReasoning: group.Usage.ReasoningTokens,
GroupCacheRatioAvailable: ratioAvailable,
AssessmentCacheRatio: assessmentRatio,
PreparedMessages: len(preparedDetails.Messages),
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
}, nil

View File

@@ -3,7 +3,10 @@ package main
import (
"bytes"
"encoding/json"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestObserveExample(t *testing.T) {
@@ -12,18 +15,22 @@ func TestObserveExample(t *testing.T) {
t.Fatal(err)
}
var got struct {
ProposalAttempts *int `json:"proposal_attempts"`
ProposalRetries *int `json:"proposal_transient_retries"`
UsageComplete *bool `json:"proposal_usage_complete"`
AssessmentAttempts *int `json:"assessment_attempts"`
AssessmentRetries *int `json:"assessment_transient_retries"`
Outcome string `json:"assessment_outcome"`
CoordinationOutcome string `json:"coordination_outcome"`
ExhaustedOutcome string `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"`
GroupAttempts int `json:"group_attempts"`
PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"`
ProposalAttempts *int `json:"proposal_attempts"`
ProposalRetries *int `json:"proposal_transient_retries"`
UsageComplete *bool `json:"proposal_usage_complete"`
AssessmentAttempts *int `json:"assessment_attempts"`
AssessmentRetries *int `json:"assessment_transient_retries"`
Outcome string `json:"assessment_outcome"`
CoordinationOutcome string `json:"coordination_outcome"`
ExhaustedOutcome string `json:"exhausted_outcome"`
GroupMembers int `json:"group_members"`
GroupAttempts int `json:"group_attempts"`
GroupCost promptkit.ObservedCost `json:"group_cost"`
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
PreparedMessages int `json:"prepared_messages"`
PreparedParentMatches bool `json:"prepared_parent_matches"`
}
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
t.Fatal(err)
@@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) {
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
t.Fatalf("unexpected observation summary: %+v", got)
}
if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 {
t.Fatalf("unexpected usage summary: %+v", got)
}
}

View File

@@ -5,7 +5,7 @@ import (
"go.maximumdirect.net/promptkit/internal/observation"
)
// ExecutionGroup collects facts from explicitly selected execution events.
// ExecutionGroup collects facts from selected execution events.
// Its zero value is ready for concurrent use. Do not copy it after first use.
type ExecutionGroup struct{ group observation.Group }
@@ -21,7 +21,8 @@ type ExecutionGroupMember struct {
// member has a terminal, independently of usage completeness. An empty group
// has complete coverage and known complete zero usage; a group with members
// but no terminals has unknown, incomplete usage. Overlapping groups are not
// generally additive.
// generally additive. CostUSD covers included terminals independently of
// terminal coverage, using the same known, complete, and source merge rules.
type ExecutionGroupSummary struct {
Members []ExecutionGroupMember `json:"members"`
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
@@ -29,6 +30,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64 `json:"transient_retries"`
RepairCalls uint64 `json:"repair_calls"`
Usage ObservedTokenUsage `json:"usage"`
CostUSD ObservedCost `json:"cost_usd"`
}
// Observe enrolls an event's run, counting its first terminal event only.
@@ -36,8 +38,8 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) {
count := func(c ObservedTokenCount) domain.ObservedTokenCount {
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
}
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls, CostUSD: toDomainCost(event.CostUSD),
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
}
// Snapshot returns an internally consistent, caller-owned summary. Coverage
@@ -51,6 +53,6 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
for i, member := range value.Members {
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
}
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls, CostUSD: fromDomainCost(value.CostUSD),
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
}

View File

@@ -1,5 +1,6 @@
backends:
- id: openrouter
provider: openrouter
endpoint: https://openrouter.ai/api/v1
api_key_env: OPENROUTER_API_KEY
smoke_tests:

View File

@@ -17,7 +17,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
t.Fatal(err)
}
backend, err := state.GetBackend("openrouter")
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
t.Fatalf("backend = %+v, %v", backend, err)
}
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
@@ -40,6 +40,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
t.Fatalf("profile %s = %+v, %v", id, profile, err)
}
want := defaults.ExecutionTargetDefault()
want.Provider = "openrouter"
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}

View File

@@ -59,6 +59,35 @@ func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
}
}
func TestProviderPolicyCaptureAndInspection(t *testing.T) {
state, err := load(mapping("provider policy", map[string]any{
"collections": map[string]any{"team": map[string]any{
"backends": []any{map[string]any{"id": "renamed", "endpoint": "https://example.invalid/v1", "provider": "OPENROUTER", "zero_cost": true}, map[string]any{"id": "child", "extends": map[string]any{"id": "renamed"}, "zero_cost": false}},
"profiles": []any{map[string]any{"id": "selected", "backend": "team/child", "endpoint": "https://other.invalid/v1", "model": "m"}},
}},
"profiles": []any{map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1", "model": "m"}},
}), nil)
if err != nil {
t.Fatal(err)
}
backend, err := state.GetBackend("team/child")
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost {
t.Fatalf("backend: %+v, %v", backend, err)
}
profile, err := state.GetProfile(context.Background(), "team/selected")
if err != nil || profile.ResolvedTarget.Provider != "openrouter" || profile.ResolvedTarget.ZeroCost || profile.ResolvedTarget.Endpoint != "https://other.invalid/v1" {
t.Fatalf("profile: %+v, %v", profile, err)
}
profile, err = state.GetProfile(context.Background(), "generic")
if err != nil || profile.ResolvedTarget.Provider != "openai-compatible" || profile.ResolvedTarget.ZeroCost {
t.Fatalf("generic: %+v, %v", profile, err)
}
resource, err := state.InspectBackend(context.Background(), "team/child")
if err != nil || resource.Provider == nil || *resource.Provider != "openrouter" || resource.ZeroCost == nil || *resource.ZeroCost {
t.Fatalf("inspection: %+v, %v", resource, err)
}
}
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
definition := map[string]any{
"id": "example", "inputs": []any{map[string]any{"name": " document "}},

View File

@@ -291,6 +291,9 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
if backend.Endpoint == nil || *backend.Endpoint == "" {
validation.Error(base.Key("endpoint"), "endpoint is required")
}
if backend.Provider != nil && !llm.SupportedProvider(*backend.Provider) {
validation.Error(base.Key("provider"), "unsupported provider")
}
if backend.APIKeyEnv != nil && *backend.APIKeyEnv != "" && !envName.MatchString(*backend.APIKeyEnv) {
validation.Error(base.Key("api_key_env"), "environment variable name is invalid")
}
@@ -319,10 +322,8 @@ func validateBackendRecord(backend backendConfig, base configkit.Path, validatio
}
if authentication := smokeTests.Authentication; authentication != nil {
kindPath := smokePath.Key("authentication").Key("kind")
if authentication.Kind == nil {
validation.Error(kindPath, "authentication kind is required")
} else if *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" {
validation.Error(kindPath, "authentication kind must be none or openrouter_key")
if authentication.Kind != nil && *authentication.Kind != "none" && *authentication.Kind != "openrouter_key" && *authentication.Kind != "vllm_models" {
validation.Error(kindPath, "unsupported authentication kind")
}
}
}

View File

@@ -532,6 +532,54 @@ func TestLoadInterpretsBackendNormalizationAndNullDefault(t *testing.T) {
}
}
func TestBackendProviderFinalConfiguration(t *testing.T) {
underlays := []configkit.Underlay{{Inputs: []configkit.Input{mapping("defaults", map[string]any{
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": "bad", "zero_cost": true}},
})}}}
main := mapping("override", map[string]any{"backends": []any{
map[string]any{"id": "base", "endpoint": "https://example.invalid/v1", "provider": " OPENROUTER ", "zero_cost": false, "smoke_tests": map[string]any{"authentication": map[string]any{}}},
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}},
map[string]any{"id": "cleared", "extends": map[string]any{"id": "base"}, "provider": nil, "zero_cost": nil},
map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1"},
}})
state, err := load(main, underlays)
if err != nil {
var detail *configkit.LoadError
if errors.As(err, &detail) {
t.Fatalf("%v: %+v", err, detail.Diagnostics())
}
t.Fatal(err)
}
for _, name := range []string{"base", "child"} {
b := state.backendRecords[name]
if b.Provider == nil || *b.Provider != "openrouter" || b.ZeroCost == nil || *b.ZeroCost {
t.Fatalf("%s: %+v", name, b)
}
}
for _, name := range []string{"cleared", "generic"} {
b := state.backendRecords[name]
if b.Provider != nil || b.ZeroCost != nil {
t.Fatalf("%s: %+v", name, b)
}
}
field, found := state.inspection.Metadata().Lookup(configkit.Keys("promptkit", "backends").Index(0).Key("provider"))
if !found || field.Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("provider normalization: %+v", field)
}
for _, provider := range []string{"openai-compatible", "openrouter", "vllm", " ", ""} {
_, err := load(mapping("provider", map[string]any{"backends": []any{map[string]any{"id": "x", "endpoint": "https://example.invalid/v1", "provider": provider, "smoke_tests": map[string]any{"authentication": map[string]any{"kind": " VLLM_MODELS "}}}}}), nil)
if err != nil {
t.Errorf("provider %q: %v", provider, err)
}
}
for _, fields := range []map[string]any{{"provider": "unsupported"}, {"zero_cost": "false"}} {
fields["id"], fields["endpoint"] = "x", "https://example.invalid/v1"
if _, err := load(mapping("invalid", map[string]any{"backends": []any{fields}}), nil); err == nil {
t.Fatalf("accepted invalid fields: %v", fields)
}
}
}
func TestLoadBackendSmokeTestsOmissionAndNull(t *testing.T) {
for _, test := range []struct {
name string
@@ -638,11 +686,6 @@ func TestLoadRejectsInvalidFinalBackendSmokeTests(t *testing.T) {
data: map[string]any{"default_profile": " \t "},
path: "/promptkit/backends/0/smoke_tests/default_profile",
},
{
name: "authentication kind required",
data: map[string]any{"authentication": map[string]any{}},
path: "/promptkit/backends/0/smoke_tests/authentication/kind",
},
{
name: "unknown authentication kind",
data: map[string]any{"authentication": map[string]any{"kind": "bearer"}},

View File

@@ -21,6 +21,8 @@ func copyProviderMap(value map[string]any) map[string]any {
}
func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings {
in.Provider = copyPointer(in.Provider)
in.ZeroCost = copyPointer(in.ZeroCost)
in.Endpoint = copyPointer(in.Endpoint)
in.APIKeyEnv = copyPointer(in.APIKeyEnv)
in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit)

View File

@@ -15,6 +15,9 @@ func resolveBackends(entries []backendConfig, base configkit.Path, resolution *c
for i := range entries {
entry := &entries[i]
path := base.Index(i)
if entry.Provider != nil {
recordStringNormalization(&entry.Provider, path.Key("provider"), resolution, strings.ToLower(strings.TrimSpace(*entry.Provider)))
}
if entry.Endpoint != nil {
if normalized, err := domain.NormalizeOpenAICompatibleBaseEndpoint(*entry.Endpoint); err == nil {
recordStringNormalization(&entry.Endpoint, path.Key("endpoint"), resolution, normalized)

View File

@@ -26,6 +26,8 @@ type ProfileResource struct {
// Its fields borrow immutable configuration captured by the adapter.
type BackendResource struct {
ID, Endpoint string
Provider *string
ZeroCost *bool
APIKeyEnv *string
ConcurrencyLimit, QueueCapacity *int
ExtraParams map[string]any
@@ -131,7 +133,7 @@ func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendRes
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
}
return &BackendResource{
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv,
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, Provider: config.Provider, ZeroCost: config.ZeroCost,
ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity,
ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests,
}, nil

View File

@@ -8,6 +8,7 @@ import (
"go.maximumdirect.net/promptkit/internal/defaults"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/jsonvalue"
"go.maximumdirect.net/promptkit/internal/llm"
profilepkg "go.maximumdirect.net/promptkit/internal/profile"
"go.maximumdirect.net/promptkit/internal/settings"
)
@@ -36,7 +37,7 @@ func (state *loaded) buildExecutionProfiles(config *settings.Settings) error {
}
func captureBackend(id string, config backendConfig) domain.Backend {
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
definition := domain.Backend{ID: id, Provider: llm.EffectiveProvider(valueOrZero(config.Provider)), ZeroCost: valueOrZero(config.ZeroCost), ExtraParams: config.ExtraParams}
if config.SmokeTests != nil {
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
if config.SmokeTests.Reachability != nil {
@@ -67,6 +68,7 @@ func captureBackend(id string, config backendConfig) domain.Backend {
func captureExecutionProfile(id string, config profileConfig, backends map[string]domain.Backend) (domain.ExecutionProfile, error) {
target := defaults.ExecutionTargetDefault()
target.Provider = llm.EffectiveProvider("")
if config.Backend != nil && *config.Backend != "" {
// These defaults are capture-owned and immutable. Public and runtime
// lookups still detach mutable values; profile overlays copy before merging.
@@ -75,6 +77,8 @@ func captureExecutionProfile(id string, config profileConfig, backends map[strin
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w: %q", id, backend.ErrBackendNotFound, *config.Backend)
}
target.BackendID = selected.ID
target.Provider = selected.Provider
target.ZeroCost = selected.ZeroCost
target.Endpoint = selected.Endpoint
target.APIKeyEnv = selected.APIKeyEnv
target.ExtraParams = selected.ExtraParams

17
internal/domain/cost.go Normal file
View File

@@ -0,0 +1,17 @@
package domain
type CostSource string
const (
CostSourceNone CostSource = ""
CostSourceProvider CostSource = "provider"
CostSourceConfiguredZero CostSource = "configured_zero"
CostSourceMixed CostSource = "mixed"
)
type ObservedCost struct {
Value float64
Known bool
Complete bool
Source CostSource
}

View File

@@ -99,6 +99,7 @@ type RunResult struct {
EffectiveModelParams ExecutionTarget
InputHashes map[string]string
Usage TokenUsage
CostUSD ObservedCost
StartTime time.Time
EndTime time.Time
Duration time.Duration
@@ -188,6 +189,8 @@ type PromptMessageTemplate struct {
// diagnostic metadata.
type Backend struct {
ID string
Provider string
ZeroCost bool
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
@@ -236,6 +239,8 @@ type ExecutionTargetPresence struct {
// ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct {
Provider string `yaml:"-" json:"-"`
ZeroCost bool `yaml:"-" json:"-"`
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"`
@@ -324,12 +329,14 @@ type GenerateResponse struct {
Content string
Usage TokenUsage
UsagePresence *TokenUsagePresence
CostUSD *float64
}
// TokenUsagePresence records which token fields a provider reported.
type TokenUsagePresence struct {
PromptTokens bool
CompletionTokens bool
ReasoningTokens bool
TotalTokens bool
CachedTokens bool
CacheWriteTokens bool
@@ -339,6 +346,7 @@ type TokenUsagePresence struct {
type TokenUsage struct {
PromptTokens int
CompletionTokens int
ReasoningTokens int
TotalTokens int
CachedTokens int
CacheWriteTokens int

View File

@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount
@@ -75,6 +76,7 @@ type ExecutionGroupSummary struct {
TransientRetries uint64
RepairCalls uint64
Usage ObservedTokenUsage
CostUSD ObservedCost
}
// ExecutionEvent contains metadata only; it carries no mutable containers or
@@ -100,6 +102,7 @@ type ExecutionEvent struct {
StatusCode int
RetriesExhausted bool
Usage ObservedTokenUsage
CostUSD ObservedCost
Attempts uint64
TransientRetries uint64
RepairCalls uint64

View File

@@ -130,7 +130,7 @@ func (c *Coordinator) generate(
if result.Dispatched {
attempts++
}
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal)
if result.Outcome.Err == nil {
if err := ctx.Err(); err != nil {
if captureEvidence {
@@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
generation = 1
}
result := c.executeInjected(ctx, req)
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1)
if captureEvidence {
evidence.Usage = usage.Observed()
}
@@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe
requested := req.StructuredOutput != nil
result.Outcome.StructuredOutputRequested = &requested
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
if result.Outcome.Response != nil {
result.Outcome.CostUSD = result.Outcome.Response.CostUSD
}
result.RequestDuration = elapsed(start, c.now())
if result.Outcome.Err == nil && result.Outcome.Response == nil {
result.Outcome.Err = llm.ErrMalformedResponse
@@ -240,8 +243,15 @@ type attemptResult struct {
// recordAttempt runs after the attempt has released its permit, including when
// the recorder calls application code. Injected clients expose no HTTP facts.
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) {
outcome := result.Outcome
var resolved domain.ObservedCost
if result.Dispatched {
resolved = observation.ResolveCost(outcome.CostUSD, zeroCost)
if cost != nil {
cost.Add(resolved)
}
}
failure := ClassifyFailure(outcome.Err)
if capture {
attempt := AttemptEvidence{
@@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved})
}
}

View File

@@ -58,6 +58,89 @@ func TestObservedRetryAndExhaustion(t *testing.T) {
}
}
func TestDispatchedAttemptCostsAreCollectedWithoutObserver(t *testing.T) {
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return response(503, `{"error":{}}`), nil
}
return response(200, `{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`), nil
})
var cost observation.CostAccumulator
ctx := observation.WithCostAccumulator(context.Background(), &cost)
req := testRequest(1)
req.Target.Provider = "openrouter"
if _, err := coordinator.Generate(ctx, req); err != nil {
t.Fatal(err)
}
got := cost.Observed()
if got.Value != .02 || !got.Known || got.Complete || got.Source != domain.CostSourceProvider {
t.Fatalf("cost=%+v", got)
}
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { return nil, errors.New("transport failed") })
var zero observation.CostAccumulator
req.Target.ZeroCost = true
if _, err := configured.Generate(observation.WithCostAccumulator(context.Background(), &zero), req); err == nil {
t.Fatal("expected error")
}
if got := zero.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
t.Fatalf("fallback=%+v", got)
}
}
func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) {
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
charge := .25
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{CostUSD: &charge}, err: errors.New("failed")})
var cost observation.CostAccumulator
var events []domain.ExecutionEvent
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
if err != nil {
t.Fatal(err)
}
recorder.SetCostAccumulator(&cost)
ctx := observation.WithRecorder(observation.WithCostAccumulator(context.Background(), &cost), recorder)
if _, err := coordinator.Generate(ctx, testRequest(0)); err == nil {
t.Fatal("expected error")
}
recorder.Finish(observation.FinishData{})
want := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
if len(events) != 2 || events[0].CostUSD != want || events[1].CostUSD != want || cost.Observed() != want {
t.Fatalf("events=%+v cost=%+v", events, cost.Observed())
}
}
func TestCanceledDispatchCostFallbackAndUndispatchedEmptyCost(t *testing.T) {
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
ctx, cancel := context.WithCancel(context.Background())
coordinator := NewInjected(configured.manager, cancelingCostClient{cancel: cancel})
var cost observation.CostAccumulator
req := testRequest(0)
req.Target.ZeroCost = true
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &cost), req); !errors.Is(err, context.Canceled) {
t.Fatalf("dispatch error=%v", err)
}
if got := cost.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
t.Fatalf("dispatched=%+v", got)
}
var untouched observation.CostAccumulator
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &untouched), req); !errors.Is(err, context.Canceled) {
t.Fatalf("pre-dispatch error=%v", err)
}
if got := untouched.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("undispatched=%+v", got)
}
}
type cancelingCostClient struct{ cancel context.CancelFunc }
func (c cancelingCostClient) Generate(ctx context.Context, _ domain.GenerateRequest) (*domain.GenerateResponse, error) {
c.cancel()
return nil, ctx.Err()
}
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
ctx, cancel := context.WithCancel(context.Background())

View File

@@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
// PreparedGeneration holds one encoded request and resolved authentication for
// repeated attempts. Each execution creates a fresh HTTP request and context.
type PreparedGeneration struct {
provider string
endpoint string
payload []byte
apiKey string
@@ -104,6 +105,7 @@ type AttemptOutcome struct {
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
CostUSD *float64
RetryNotBefore time.Time
Evidence HTTPExecutionEvidence
StructuredOutputRequested *bool
@@ -156,7 +158,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
}
return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey,
endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider),
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
structuredOutputRequested: wirePayload["response_format"] != nil,
}, nil
@@ -266,16 +268,40 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
}
return outcome
}
var writes, reasoning *int
switch p.provider {
case "openrouter", "vllm":
var details map[string]json.RawMessage
if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil {
reasoning = optionalInteger(details["reasoning_tokens"])
}
if p.provider == "openrouter" {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens)
} else {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens)
}
default:
if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil {
outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
return outcome
}
}
outcome.ResponseDecoded = true
outcome.Usage = domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning),
}
outcome.UsagePresence = &domain.TokenUsagePresence{
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
}
if p.provider == "openrouter" {
outcome.CostUSD = optionalFloat(wireResp.Usage.Cost)
}
if len(wireResp.Choices) == 0 {
@@ -297,7 +323,7 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
}
return outcome
}
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: outcome.CostUSD}
outcome.GenerationCompleted = true
if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
@@ -540,12 +566,38 @@ type openAIChatResponse struct {
CompletionTokens *int `json:"completion_tokens"`
TotalTokens *int `json:"total_tokens"`
PromptTokensDetails struct {
CachedTokens *int `json:"cached_tokens"`
CachedTokens *int `json:"cached_tokens"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"`
} `json:"prompt_tokens_details"`
CacheWriteTokens *int `json:"cache_write_tokens"`
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
Cost json.RawMessage `json:"cost"`
} `json:"usage"`
}
func optionalInteger(raw json.RawMessage) *int {
if len(raw) == 0 {
return nil
}
var value *int
if json.Unmarshal(raw, &value) != nil {
return nil
}
return value
}
func optionalFloat(raw json.RawMessage) *float64 {
if len(raw) == 0 {
return nil
}
var value *float64
if json.Unmarshal(raw, &value) != nil || (value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0))) {
return nil
}
return value
}
type openAIResponseFormat struct {
Type string `json:"type"`
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`

View File

@@ -882,6 +882,85 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
}
}
func TestProviderSpecificUsageDetails(t *testing.T) {
for _, tc := range []struct {
name, provider, usage string
writes, reasoning int
writesKnown, reasoningKnown bool
}{
{name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true},
{name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true},
{name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true},
{name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`},
{name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`},
{name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`},
{name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`},
} {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`)
request := provider.generateRequest()
request.Target.Provider = tc.provider
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown {
t.Fatalf("response=%+v error=%v", response, err)
}
})
}
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`)
request := provider.generateRequest()
request.Target.Provider = "openrouter"
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.Execute(context.Background())
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens {
t.Fatalf("outcome=%+v", outcome)
}
}
func TestProviderReportedCostIsOptional(t *testing.T) {
for _, tc := range []struct {
name, provider, cost string
known bool
value float64
}{
{"charge", "openrouter", `0.002`, true, .002},
{"zero", "openrouter", `0`, true, 0},
{"null", "openrouter", `null`, false, 0},
{"string", "openrouter", `"0.2"`, false, 0},
{"object", "openrouter", `{}`, false, 0},
{"overflow", "openrouter", `1e9999`, false, 0},
{"generic ignores cost", "openai-compatible", `0.02`, false, 0},
{"vllm ignores cost", "vllm", `0.02`, false, 0},
} {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"cost":`+tc.cost+`},"choices":[{"message":{"content":"ok"}}]}`)
request := provider.generateRequest()
request.Target.Provider = tc.provider
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
if err != nil || response == nil || (response.CostUSD != nil) != tc.known || (response.CostUSD != nil && *response.CostUSD != tc.value) {
t.Fatalf("response=%+v err=%v", response, err)
}
})
}
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"cost":0.02},"choices":[]}`)
request := provider.generateRequest()
request.Target.Provider = "openrouter"
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.Execute(context.Background())
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.CostUSD == nil || *outcome.CostUSD != .02 {
t.Fatalf("outcome=%+v", outcome)
}
}
func checkOptionalResponseFieldOmission(t *testing.T) {
provider := newRecordingProvider(t)
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
@@ -26,8 +27,11 @@ func OpenRouterKeyEndpoint(base string) (string, error) {
return url.JoinPath(base, "key")
}
// ProbeOpenRouterKey executes and interprets an OpenRouter key request.
func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
// ProbeAuthentication executes the selected authentication adapter.
func (c *OpenAICompatibleClient) ProbeAuthentication(ctx context.Context, adapter, endpoint string, timeoutSeconds int, target domain.ExecutionTarget) (AuthenticationEvidence, error) {
if !SupportedAuthenticationAdapter(adapter) {
return AuthenticationEvidence{}, fmt.Errorf("unsupported authentication adapter %q", adapter)
}
parsed, err := url.Parse(endpoint)
if err != nil {
return AuthenticationEvidence{}, err
@@ -69,7 +73,14 @@ func (c *OpenAICompatibleClient) ProbeOpenRouterKey(ctx context.Context, endpoin
if err != nil {
return result, err
}
if recognizedOpenRouterKeyResponse(contents) {
recognized := false
switch adapter {
case "openrouter_key":
recognized = recognizedOpenRouterKeyResponse(contents)
case "vllm_models":
recognized = recognizedVLLMModelsResponse(contents)
}
if recognized {
accepted := true
result.Accepted = &accepted
} else {

View File

@@ -38,7 +38,7 @@ func TestProbeOpenRouterKey(t *testing.T) {
authorization = req.Header.Get("Authorization")
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
})}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://user:password@example.invalid/key", 0, domain.ExecutionTarget{APIKey: tc.key})
if err != nil {
t.Fatal(err)
}
@@ -67,7 +67,7 @@ func TestProbeOpenRouterKeyRetainsBodyFailure(t *testing.T) {
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: authenticationFailingBody{err: want}}, nil
})}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
if !errors.Is(err, want) || !got.HTTP.ResponseReceived {
t.Fatalf("evidence = %+v, error = %v", got, err)
}
@@ -105,7 +105,7 @@ func TestProbeOpenRouterKeyBoundsAndClosesBodies(t *testing.T) {
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: tc.status, ContentLength: 1, Header: make(http.Header), Body: body}, nil
})}})
got, err := client.ProbeOpenRouterKey(context.Background(), "https://example.invalid/key", 0, domain.ExecutionTarget{})
got, err := client.ProbeAuthentication(context.Background(), "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
if err != nil || body.read != tc.wantRead || !body.closed || got.Overflow != (tc.status == http.StatusOK) {
t.Fatalf("evidence = %+v, read = %d, closed = %v, error = %v", got, body.read, body.closed, err)
}
@@ -127,7 +127,7 @@ func TestProbeOpenRouterKeyPreservesCancellation(t *testing.T) {
})}})
ctx, cancel := context.WithCancel(context.Background())
cancel()
got, err := client.ProbeOpenRouterKey(ctx, "https://example.invalid/key", 0, domain.ExecutionTarget{})
got, err := client.ProbeAuthentication(ctx, "openrouter_key", "https://example.invalid/key", 0, domain.ExecutionTarget{})
if !errors.Is(err, context.Canceled) || got.Accepted != nil || got.HTTP.ResponseReceived {
t.Fatalf("evidence = %+v, error = %v", got, err)
}

59
internal/llm/provider.go Normal file
View File

@@ -0,0 +1,59 @@
package llm
import (
"net/url"
"go.maximumdirect.net/promptkit/internal/domain"
)
// EffectiveProvider resolves an omitted provider to the generic conventions.
func EffectiveProvider(provider string) string {
if provider == "" {
return "openai-compatible"
}
return provider
}
// SupportedProvider reports whether a normalized provider selection is supported.
// An empty selection uses the generic OpenAI-compatible conventions.
func SupportedProvider(provider string) bool {
switch provider {
case "", "openai-compatible", "openrouter", "vllm":
return true
default:
return false
}
}
// AuthenticationAdapter selects declared diagnostic metadata or provider defaults.
func AuthenticationAdapter(provider string, smoke *domain.BackendSmokeTestConfig) string {
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Kind != nil {
return *smoke.Authentication.Kind
}
switch provider {
case "openrouter":
return "openrouter_key"
case "vllm":
return "vllm_models"
}
return ""
}
// AuthenticationEndpoint resolves a diagnostic destination without changing configuration.
func AuthenticationEndpoint(adapter, base string, smoke *domain.BackendSmokeTestConfig) (string, error) {
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Endpoint != nil {
return *smoke.Authentication.Endpoint, nil
}
if adapter == "openrouter_key" {
return OpenRouterKeyEndpoint(base)
}
if adapter == "vllm_models" {
return url.JoinPath(base, "models")
}
return base, nil
}
// SupportedAuthenticationAdapter identifies implemented authentication probes.
func SupportedAuthenticationAdapter(adapter string) bool {
return adapter == "openrouter_key" || adapter == "vllm_models"
}

View File

@@ -0,0 +1,34 @@
package llm
import (
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestAuthenticationSelection(t *testing.T) {
none, explicit, models, endpoint := "none", "openrouter_key", "vllm_models", "https://diagnostics.example/probe"
for _, tc := range []struct {
name, provider, adapter, destination string
metadata *domain.BackendSmokeTestConfig
}{
{name: "provider default", provider: "openrouter", adapter: "openrouter_key", destination: "https://example.test/v1/key"},
{name: "cleared block", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
{name: "endpoint only", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "openrouter_key", destination: endpoint},
{name: "explicit adapter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &explicit}}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
{name: "disabled", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
{name: "generic unsupported", destination: "https://example.test/v1"},
{name: "vllm default", provider: "vllm", adapter: "vllm_models", destination: "https://example.test/v1/models"},
{name: "vllm override", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "vllm_models", destination: endpoint},
{name: "explicit vllm on generic", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &models}}, adapter: "vllm_models", destination: "https://example.test/v1/models"},
{name: "disabled vllm", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
} {
t.Run(tc.name, func(t *testing.T) {
kind := AuthenticationAdapter(tc.provider, tc.metadata)
destination, err := AuthenticationEndpoint(kind, "https://example.test/v1", tc.metadata)
if err != nil || kind != tc.adapter || destination != tc.destination {
t.Fatalf("selection=%q destination=%q error=%v", kind, destination, err)
}
})
}
}

View File

@@ -0,0 +1,16 @@
package llm
import "encoding/json"
func recognizedVLLMModelsResponse(contents []byte) bool {
var envelope map[string]json.RawMessage
if json.Unmarshal(contents, &envelope) != nil || envelope == nil {
return false
}
var object string
if json.Unmarshal(envelope["object"], &object) != nil || object != "list" {
return false
}
var data []json.RawMessage
return json.Unmarshal(envelope["data"], &data) == nil && data != nil
}

View File

@@ -0,0 +1,51 @@
package llm
import (
"context"
"io"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestVLLMModelsAuthentication(t *testing.T) {
for _, tc := range []struct {
name, body, key string
status int
accepted *bool
unrecognized, overflow bool
}{
{name: "empty list", status: 200, body: `{"object":"list","data":[]}`, accepted: boolPtr(true)},
{name: "nonempty list", status: 200, body: `{"object":"list","data":[{"id":"model"}]}`, key: "key", accepted: boolPtr(true)},
{name: "wrong object", status: 200, body: `{"object":"model","data":[]}`, unrecognized: true},
{name: "null data", status: 200, body: `{"object":"list","data":null}`, unrecognized: true},
{name: "object data", status: 200, body: `{"object":"list","data":{}}`, unrecognized: true},
{name: "invalid framing", status: 200, body: `{"object":"list","data":[]} {}`, unrecognized: true},
{name: "overflow", status: 200, body: strings.Repeat(" ", maxAuthenticationResponseBytes+1), overflow: true},
{name: "unauthorized", status: 401, accepted: boolPtr(false)},
{name: "other status", status: 503},
} {
t.Run(tc.name, func(t *testing.T) {
var credential string
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
credential = req.Header.Get("Authorization")
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
})}})
got, err := client.ProbeAuthentication(context.Background(), "vllm_models", "https://example.invalid/v1/models", 0, domain.ExecutionTarget{APIKey: tc.key})
if err != nil || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow || (got.Accepted == nil) != (tc.accepted == nil) || (got.Accepted != nil && *got.Accepted != *tc.accepted) {
t.Fatalf("result=%+v err=%v", got, err)
}
want := ""
if tc.key != "" {
want = "Bearer " + tc.key
}
if credential != want {
t.Fatalf("authorization=%q", credential)
}
})
}
}
func boolPtr(value bool) *bool { return &value }

View File

@@ -0,0 +1,67 @@
package observation
import (
"context"
"math"
"go.maximumdirect.net/promptkit/internal/domain"
)
type costContextKey struct{}
// WithCostAccumulator carries one invocation's cost state.
func WithCostAccumulator(ctx context.Context, accumulator *CostAccumulator) context.Context {
return context.WithValue(ctx, costContextKey{}, accumulator)
}
// CostFromContext returns the current invocation's cost state, if installed.
func CostFromContext(ctx context.Context) *CostAccumulator {
value, _ := ctx.Value(costContextKey{}).(*CostAccumulator)
return value
}
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {
return domain.ObservedCost{Value: *reported, Known: true, Complete: true, Source: domain.CostSourceProvider}
}
if zeroCost {
return domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}
}
return domain.ObservedCost{}
}
// CostAccumulator merges dispatched attempts or already aggregated costs.
type CostAccumulator struct {
value domain.ObservedCost
started bool
}
// Add incorporates one contribution.
func (c *CostAccumulator) Add(next domain.ObservedCost) {
if !c.started {
c.started = true
c.value = next
return
}
if next.Known {
c.value.Value += next.Value
if next.Source != domain.CostSourceNone {
if c.value.Source == domain.CostSourceNone {
c.value.Source = next.Source
} else if c.value.Source != next.Source {
c.value.Source = domain.CostSourceMixed
}
}
}
c.value.Known = c.value.Known || next.Known
c.value.Complete = c.value.Complete && next.Complete
}
// Observed returns known complete zero when no contribution has been added.
func (c CostAccumulator) Observed() domain.ObservedCost {
if !c.started {
return domain.ObservedCost{Known: true, Complete: true}
}
return c.value
}

View File

@@ -0,0 +1,63 @@
package observation
import (
"math"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestResolveCostPrecedence(t *testing.T) {
zero, negative, invalid := 0.0, -1.5, math.Inf(1)
for _, tc := range []struct {
name string
reported *float64
fallback bool
want domain.ObservedCost
}{
{"unknown", nil, false, domain.ObservedCost{}},
{"configured zero", nil, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
{"reported zero", &zero, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceProvider}},
{"negative", &negative, false, domain.ObservedCost{Value: -1.5, Known: true, Complete: true, Source: domain.CostSourceProvider}},
{"invalid fallback", &invalid, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
{"invalid unknown", &invalid, false, domain.ObservedCost{}},
} {
t.Run(tc.name, func(t *testing.T) {
if got := ResolveCost(tc.reported, tc.fallback); got != tc.want {
t.Fatalf("got %+v want %+v", got, tc.want)
}
})
}
}
func TestCostAccumulatorMergesSourcesAndCoverage(t *testing.T) {
var acc CostAccumulator
if got := acc.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty: %+v", got)
}
acc.Add(domain.ObservedCost{})
if got := acc.Observed(); got != (domain.ObservedCost{}) {
t.Fatalf("unknown: %+v", got)
}
a, b := .1, .2
acc.Add(ResolveCost(&a, false))
acc.Add(ResolveCost(&b, false))
acc.Add(ResolveCost(nil, true))
got := acc.Observed()
if math.Abs(got.Value-.3) > 1e-12 || !got.Known || got.Complete || got.Source != domain.CostSourceMixed {
t.Fatalf("merged: %+v", got)
}
var zeros CostAccumulator
zero := 0.0
zeros.Add(ResolveCost(&zero, false))
zeros.Add(ResolveCost(nil, true))
if got := zeros.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceMixed}) {
t.Fatalf("zeros: %+v", got)
}
var aggregate CostAccumulator
aggregate.Add(domain.ObservedCost{Known: true, Complete: true})
aggregate.Add(got)
if aggregate.Observed() != got {
t.Fatalf("aggregate: %+v", aggregate.Observed())
}
}

View File

@@ -14,6 +14,7 @@ type Group struct {
terminals int
attempts, retries, repairs uint64
usage domain.ObservedTokenUsage
cost CostAccumulator
}
// Observe enrolls a run and incorporates its first terminal event only.
@@ -36,9 +37,10 @@ func (g *Group) Observe(event domain.ExecutionEvent) {
g.repairs += event.RepairCalls
if g.terminals == 0 {
complete := domain.ObservedTokenCount{Complete: true}
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
}
mergeGroupUsage(&g.usage, event.Usage)
g.cost.Add(event.CostUSD)
g.terminals++
}
@@ -47,12 +49,15 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary {
g.mu.Lock()
defer g.mu.Unlock()
result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage}
if g.terminals > 0 || len(g.members) == 0 {
result.CostUSD = g.cost.Observed()
}
for id, terminal := range g.members {
result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal})
}
if len(g.members) == 0 {
zero := domain.ObservedTokenCount{Known: true, Complete: true}
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, ReasoningTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
}
return result
}
@@ -67,6 +72,7 @@ func mergeGroupUsage(total *domain.ObservedTokenUsage, next domain.ObservedToken
}
merge(&total.PromptTokens, next.PromptTokens)
merge(&total.CompletionTokens, next.CompletionTokens)
merge(&total.ReasoningTokens, next.ReasoningTokens)
merge(&total.TotalTokens, next.TotalTokens)
merge(&total.CachedTokens, next.CachedTokens)
merge(&total.CacheWriteTokens, next.CacheWriteTokens)

View File

@@ -41,6 +41,25 @@ func TestGroupTerminalAccounting(t *testing.T) {
}
}
func TestGroupCostCoverageIsIndependentOfTerminalCoverage(t *testing.T) {
var group Group
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty=%+v", got)
}
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionStarted, RunID: "pending"})
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{}) {
t.Fatalf("pending=%+v", got)
}
charge := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
terminal := domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "done", CostUSD: charge}
group.Observe(terminal)
group.Observe(terminal)
got := group.Snapshot()
if got.TerminalCoverageComplete || got.CostUSD != charge {
t.Fatalf("snapshot=%+v", got)
}
}
func TestGroupConcurrentSnapshots(t *testing.T) {
var group Group
var wg sync.WaitGroup

View File

@@ -30,6 +30,7 @@ type Recorder struct {
generationDuration time.Duration
retryDelay time.Duration
usage UsageAccumulator
cost *CostAccumulator
}
// Selection contains resolved identities only. Unresolved fields stay empty.
@@ -53,6 +54,7 @@ type AttemptData struct {
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
CostUSD domain.ObservedCost
}
// DelayData describes one completed or interrupted retry wait.
@@ -99,6 +101,9 @@ func (r *Recorder) RunID() string { return r.runID }
// SetParentRunID records the immediate source identity before start delivery.
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
// SetCostAccumulator selects the invocation-owned cost state for terminal reporting.
func (r *Recorder) SetCostAccumulator(cost *CostAccumulator) { r.cost = cost }
// Start delivers the opening event with only identities already resolved.
func (r *Recorder) Start() {
event := r.event(domain.ExecutionStarted)
@@ -138,6 +143,7 @@ func (r *Recorder) Attempt(data AttemptData) {
event.StatusCode = data.StatusCode
if data.Dispatched {
event.Usage = observedUsage(data.Usage, data.UsagePresence)
event.CostUSD = data.CostUSD
}
r.deliver(event)
}
@@ -176,6 +182,9 @@ func (r *Recorder) Finish(data FinishData) {
event.Duration = 0
}
event.Usage = r.usage.Observed()
if r.cost != nil {
event.CostUSD = r.cost.Observed()
}
r.deliver(event)
}
@@ -195,6 +204,22 @@ func (r *Recorder) deliver(event domain.ExecutionEvent) {
type contextKey struct{}
type observerKey struct{}
// WithObserver adds an observer to the inherited contextual selection.
func WithObserver(ctx context.Context, observer domain.ExecutionObserver) context.Context {
if observer == nil {
return ctx
}
return context.WithValue(ctx, observerKey{}, Compose(ObserverFromContext(ctx), observer))
}
// ObserverFromContext returns the contextual observer selection, if any.
func ObserverFromContext(ctx context.Context) domain.ExecutionObserver {
observer, _ := ctx.Value(observerKey{}).(domain.ExecutionObserver)
return observer
}
// WithoutExecutionState starts a public operation without inheriting another
// invocation's recorder or prepared-claim sink. Other context values and all
// cancellation/deadline behavior remain with the caller's context.
@@ -206,7 +231,7 @@ type executionBoundary struct{ context.Context }
func (c executionBoundary) Value(key any) any {
switch key.(type) {
case contextKey, claimSinkKey:
case contextKey, claimSinkKey, costContextKey:
return nil
default:
return c.Context.Value(key)
@@ -244,7 +269,7 @@ type usageField struct {
known, complete bool
}
type usageTotals struct {
prompt, completion, total, cached, cacheWrite usageField
prompt, completion, reasoning, total, cached, cacheWrite usageField
}
// UsageAccumulator combines presence-aware usage across dispatched attempts.
@@ -270,7 +295,7 @@ func (u UsageAccumulator) Observed() domain.ObservedTokenUsage {
}
func completeUsageTotals() usageTotals {
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, reasoning: usageField{complete: true},
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
}
@@ -278,6 +303,7 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
known := usagePresence(usage, presence)
u.prompt.add(usage.PromptTokens, known.PromptTokens)
u.completion.add(usage.CompletionTokens, known.CompletionTokens)
u.reasoning.add(usage.ReasoningTokens, known.ReasoningTokens)
u.total.add(usage.TotalTokens, known.TotalTokens)
u.cached.add(usage.CachedTokens, known.CachedTokens)
u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens)
@@ -286,7 +312,8 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage {
return domain.ObservedTokenUsage{
PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts),
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
ReasoningTokens: u.reasoning.snapshot(attempts),
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
CacheWriteTokens: u.cacheWrite.snapshot(attempts),
}
}
@@ -312,6 +339,7 @@ func observedUsage(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
return domain.ObservedTokenUsage{
PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens),
CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens),
ReasoningTokens: observedCount(usage.ReasoningTokens, known.ReasoningTokens),
TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens),
CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens),
CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens),
@@ -324,7 +352,8 @@ func usagePresence(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
}
return domain.TokenUsagePresence{
PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0,
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
ReasoningTokens: usage.ReasoningTokens != 0,
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
CacheWriteTokens: usage.CacheWriteTokens != 0,
}
}

View File

@@ -19,10 +19,15 @@ func TestExecutionBoundaryPreservesCallerContext(t *testing.T) {
recorder := newRecorder("outer", time.Now, nil)
sink := recorder
parent = WithClaimSink(WithRecorder(parent, recorder), &sink)
var cost CostAccumulator
parent = WithCostAccumulator(parent, &cost)
child := WithoutExecutionState(parent)
if FromContext(child) != nil || FromContext(parent) != recorder {
t.Fatal("execution boundary did not isolate recorder state")
}
if CostFromContext(child) != nil || CostFromContext(parent) != &cost {
t.Fatal("execution boundary did not isolate cost state")
}
RegisterClaim(child, nil)
if sink != recorder {
t.Fatal("execution boundary retained the parent claim sink")

View File

@@ -25,3 +25,18 @@ func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) {
t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens)
}
}
func TestReasoningUsagePresence(t *testing.T) {
var usage UsageAccumulator
usage.Add(domain.TokenUsage{}, &domain.TokenUsagePresence{ReasoningTokens: true})
usage.Add(domain.TokenUsage{ReasoningTokens: 4}, nil)
got := usage.Observed().ReasoningTokens
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: true}) {
t.Fatalf("usage=%+v", got)
}
usage.Add(domain.TokenUsage{}, nil)
got = usage.Observed().ReasoningTokens
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}) {
t.Fatalf("partial usage=%+v", got)
}
}

View File

@@ -37,6 +37,10 @@ type CollectionSettings struct {
type BackendSettings struct {
// ID identifies the backend in its resource list.
ID string `yaml:"id"`
// Provider selects response and diagnostic conventions when supplied.
Provider *string `yaml:"provider"`
// ZeroCost enables a configured-zero fallback when no cost is reported.
ZeroCost *bool `yaml:"zero_cost"`
// Endpoint is the backend's absolute base URL.
Endpoint *string `yaml:"endpoint"`
// APIKeyEnv names an environment variable containing a credential.
@@ -163,7 +167,6 @@ type BackendSmokeTestConfig struct {
// diagnostics. A nil endpoint uses the effective backend endpoint.
Reachability *ReachabilityProbeConfig `json:"reachability,omitempty" yaml:"reachability,omitempty"`
// Authentication optionally declares a named authentication diagnostic.
// A supplied object requires Kind.
Authentication *AuthenticationProbeConfig `json:"authentication,omitempty" yaml:"authentication,omitempty"`
}
@@ -178,7 +181,8 @@ type ReachabilityProbeConfig struct {
// AuthenticationProbeConfig configures an optional named authentication diagnostic.
type AuthenticationProbeConfig struct {
// Kind selects the authentication adapter. Supported values are "none" and
// "openrouter_key"; surrounding whitespace and case are normalized.
// "openrouter_key" and "vllm_models"; surrounding whitespace and case are
// normalized. Nil leaves adapter selection to the provider default.
Kind *string `json:"kind,omitempty" yaml:"kind,omitempty"`
// Endpoint is an absolute HTTP(S) URL, including any desired path or query.
// Nil selects the adapter's backend-relative destination; an explicit blank

View File

@@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
return nil, err
}
defer payload.clear()
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := NewOperationScope(ctx, payload.operationTimeout)
defer scope.Close()
defer func() { err = scope.Finalize(err) }()
@@ -117,9 +119,11 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
var recorder *observation.Recorder
var runID string
if payload.observer != nil {
observer := observation.Compose(payload.observer, observation.ObserverFromContext(ctx))
if observer != nil {
var idErr error
recorder, idErr = observation.New(payload.observer)
recorder, idErr = observation.New(observer)
recorder.SetCostAccumulator(cost)
recorder.SetParentRunID(payload.parent.RunID())
observation.RegisterClaim(ctx, recorder)
ctx = observation.WithRecorder(ctx, recorder)

View File

@@ -95,6 +95,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
}
admitter := &fakeRunAdmitter{}
profile := defaultExecutionProfile()
profile.ResolvedTarget.Provider = "vllm"
profile.ResolvedTarget.ZeroCost = true
profile.ResolvedTarget.ExtraParams = map[string]any{
"metadata": map[string]any{"source": "original"},
}
@@ -144,9 +146,11 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
}
renderer.rendered.Messages[0].Content = "source mutation"
profile.ResolvedTarget.Provider = "openrouter"
profile.ResolvedTarget.ZeroCost = false
second := prepared.Details()
if second != first || second.Messages[0].Content != "original message" ||
if second != first || second.Messages[0].Content != "original message" || second.EffectiveModelParams.Provider != "vllm" || !second.EffectiveModelParams.ZeroCost ||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
@@ -313,7 +317,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
responses: []*domain.GenerateResponse{{
Content: `{"repaired":true}`,
Usage: domain.TokenUsage{
PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5,
PromptTokens: 2, CompletionTokens: 3, ReasoningTokens: 2, TotalTokens: 5,
CachedTokens: 7, CacheWriteTokens: 11,
},
}},
@@ -324,7 +328,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
client := &sequenceLLM{responses: []*domain.GenerateResponse{{
Content: `{"broken":true}`,
Usage: domain.TokenUsage{
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
PromptTokens: 13, CompletionTokens: 17, ReasoningTokens: 4, TotalTokens: 19,
CachedTokens: 23, CacheWriteTokens: 29,
},
}}}
@@ -366,7 +370,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
}
wantUsage := domain.TokenUsage{
PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24,
PromptTokens: 15, CompletionTokens: 20, ReasoningTokens: 6, TotalTokens: 24,
CachedTokens: 30, CacheWriteTokens: 40,
}
if result.Usage != wantUsage {

View File

@@ -11,6 +11,7 @@ import (
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
Provider: "openrouter", ZeroCost: true,
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
Temperature: 0, ExtraParams: map[string]any{"configured": true},
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
@@ -35,7 +36,7 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
t.Fatalf("request did not complete partial target: %v", err)
}
if prepared.EffectiveModelParams.Endpoint != "https://runtime.example/v1" ||
prepared.EffectiveModelParams.Model != "runtime-model" || !prepared.TargetPresence.Temperature ||
prepared.EffectiveModelParams.Model != "runtime-model" || prepared.EffectiveModelParams.Provider != "openrouter" || !prepared.EffectiveModelParams.ZeroCost || !prepared.TargetPresence.Temperature ||
prepared.EffectiveModelParams.ExtraParams["request"] != true {
t.Fatalf("request overrides were not applied: %+v", prepared)
}

View File

@@ -137,6 +137,9 @@ func NewRunnerWithRepairer(
}
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
if observation.CostFromContext(ctx) == nil {
ctx = observation.WithCostAccumulator(ctx, &observation.CostAccumulator{})
}
recorder := observation.FromContext(ctx)
runID := ""
if recorder != nil {
@@ -274,7 +277,7 @@ func (r *Runner) executePreparedRun(
exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content)
}
return &domain.RunResult{
result := &domain.RunResult{
Exchange: exchange,
ParentRunID: prepared.ParentRunID,
RunID: runID,
@@ -295,7 +298,11 @@ func (r *Runner) executePreparedRun(
StartTime: start,
EndTime: end,
Duration: end.Sub(start),
}, nil
}
if cost := observation.CostFromContext(ctx); cost != nil {
result.CostUSD = cost.Observed()
}
return result, nil
}
func wrapGenerationError(err error) error {
@@ -309,6 +316,7 @@ func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
return domain.TokenUsage{
PromptTokens: total.PromptTokens + next.PromptTokens,
CompletionTokens: total.CompletionTokens + next.CompletionTokens,
ReasoningTokens: total.ReasoningTokens + next.ReasoningTokens,
TotalTokens: total.TotalTokens + next.TotalTokens,
CachedTokens: total.CachedTokens + next.CachedTokens,
CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens,

View File

@@ -11,7 +11,7 @@ import (
)
type smokeAuthenticationClient interface {
ProbeOpenRouterKey(context.Context, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
ProbeAuthentication(context.Context, string, string, int, domain.ExecutionTarget) (llm.AuthenticationEvidence, error)
}
type authenticationProbe struct{ client smokeAuthenticationClient }
@@ -30,7 +30,8 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
if err != nil || !outcome.supported || (outcome.applicable != nil && !*outcome.applicable) {
return outcome, err
}
result, requestErr := p.client.ProbeOpenRouterKey(ctx, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
result, requestErr := p.client.ProbeAuthentication(ctx, adapter, target.Endpoint, target.EffectiveSettings.TimeoutSeconds, target.EffectiveSettings)
outcome.evidence = result.HTTP
outcome.credentialsAccepted = result.Accepted
if result.Overflow {
@@ -55,22 +56,18 @@ func (p authenticationProbe) run(ctx context.Context, backend domain.Backend, ta
}
func (p authenticationProbe) support(backend domain.Backend) (authenticationProbeOutcome, error) {
if backend.SmokeTests == nil || backend.SmokeTests.Authentication == nil {
adapter := llm.AuthenticationAdapter(backend.Provider, backend.SmokeTests)
if adapter == "" {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
}
authentication := backend.SmokeTests.Authentication
if authentication.Kind == nil {
return authenticationProbeOutcome{}, fmt.Errorf("%w: authentication probe kind is missing", ErrInvalidRequest)
}
switch *authentication.Kind {
case "none":
if adapter == "none" {
return authenticationProbeOutcome{supported: true, applicable: boolPointer(false), reason: domain.SmokeTestReasonAuthenticationNotApplicable}, nil
case "openrouter_key":
if p.client == nil {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
}
return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil
default:
return authenticationProbeOutcome{}, fmt.Errorf("%w: unsupported authentication probe kind %q", ErrInvalidRequest, *authentication.Kind)
}
if !llm.SupportedAuthenticationAdapter(adapter) {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonProbeNotConfigured}, nil
}
if p.client == nil {
return authenticationProbeOutcome{reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable}, nil
}
return authenticationProbeOutcome{supported: true, applicable: boolPointer(true)}, nil
}

View File

@@ -95,6 +95,11 @@ func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
backend: backendWithAuthenticationKind("openrouter_key"),
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
},
{
name: "vllm HTTP adapter unavailable",
backend: domain.Backend{Provider: "vllm"},
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
},
} {
t.Run(tc.name, func(t *testing.T) {
probe := authenticationProbe{}

View File

@@ -110,6 +110,8 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
} else {
settings := defaults.ExecutionTargetDefault()
settings.BackendID = selection.backend.ID
settings.Provider = selection.backend.Provider
settings.ZeroCost = selection.backend.ZeroCost
settings.Endpoint = selection.backend.Endpoint
settings.APIKeyEnv = selection.backend.APIKeyEnv
if len(selection.backend.ExtraParams) > 0 {
@@ -132,7 +134,7 @@ func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTes
if promptInspection != nil {
selection.target.PromptID = promptInspection.PromptID
}
selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend.SmokeTests)
selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend)
if err != nil {
return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err)
}
@@ -165,22 +167,15 @@ func isInferenceSmokeTest(kind domain.SmokeTestKind) bool {
return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput
}
func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, smokeTests *domain.BackendSmokeTestConfig) (string, error) {
func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, backend domain.Backend) (string, error) {
smokeTests := backend.SmokeTests
switch kind {
case domain.SmokeTestReachability:
if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil {
return *smokeTests.Reachability.Endpoint, nil
}
case domain.SmokeTestAuthentication:
if smokeTests != nil && smokeTests.Authentication != nil {
authentication := smokeTests.Authentication
if authentication.Endpoint != nil {
return *authentication.Endpoint, nil
}
if authentication.Kind != nil && *authentication.Kind == "openrouter_key" {
return llm.OpenRouterKeyEndpoint(baseEndpoint)
}
}
return llm.AuthenticationEndpoint(llm.AuthenticationAdapter(backend.Provider, smokeTests), baseEndpoint, smokeTests)
}
return baseEndpoint, nil
}

View File

@@ -0,0 +1,127 @@
package promptkit_test
import (
"context"
"encoding/json"
"io"
"math"
"net/http"
"strings"
"sync"
"sync/atomic"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) {
pendingStarted, release := make(chan struct{}), make(chan struct{})
releasePending := sync.OnceFunc(func() { close(release) })
defer releasePending()
var retryCalls atomic.Int32
transport := costRoundTrip(func(req *http.Request) (*http.Response, error) {
var body struct {
Model string `json:"model"`
}
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
return nil, err
}
status, payload := 200, ""
switch body.Model {
case "router":
if retryCalls.Add(1) == 1 {
status, payload = 503, `{"error":{}}`
} else {
payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`
}
case "failure":
payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}`
case "vllm":
payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}`
case "pending":
close(pendingStarted)
<-release
payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}`
default:
t.Errorf("unexpected model %q", body.Model)
}
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil
})
engine := func(provider, model string) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false))
if err != nil {
t.Fatal(err)
}
return result
}
router, vllm := engine("openrouter", "router"), engine("vllm", "vllm")
var group promptkit.ExecutionGroup
if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty=%+v", empty)
}
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
results := make(chan error, 4)
var completed sync.WaitGroup
var pendingDone sync.WaitGroup
start := func(e *promptkit.Engine, model string, fail bool) {
completed.Add(1)
go func() {
defer completed.Done()
req := promptkit.RunRequest{PromptID: "prompt"}
if model != "" {
req.Execution = &promptkit.ExecutionTargetOverride{Model: model}
}
_, err := e.Run(ctx, req)
if fail && err != nil {
err = nil
} else if fail {
err = io.ErrUnexpectedEOF
}
results <- err
}()
}
pendingDone.Add(1)
go func() {
defer pendingDone.Done()
_, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}})
results <- err
}()
<-pendingStarted
if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete {
t.Fatalf("pending=%+v", pending)
}
start(router, "", false)
start(router, "failure", true)
start(vllm, "", false)
completed.Wait()
for range 3 {
if err := <-results; err != nil {
t.Fatal(err)
}
}
snapshot := group.Snapshot()
if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 {
t.Fatalf("snapshot=%+v", snapshot)
}
if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete {
t.Fatalf("usage=%+v", snapshot.Usage)
}
if _, ok := snapshot.Usage.CacheHitRatio(); ok {
t.Fatal("ratio available despite an unreported retry")
}
releasePending()
if err := <-results; err != nil {
t.Fatal(err)
}
pendingDone.Wait()
if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 {
t.Fatalf("final=%+v", final)
}
}

View File

@@ -19,9 +19,9 @@ func stringPointerForTest(value string) *string {
func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
value := promptkit.GenerateResponse{
Content: "ok",
Usage: promptkit.TokenUsage{TotalTokens: 5},
Usage: promptkit.TokenUsage{TotalTokens: 5, ReasoningTokens: 2},
UsagePresence: &promptkit.TokenUsagePresence{
PromptTokens: true, CompletionTokens: true, TotalTokens: true,
PromptTokens: true, CompletionTokens: true, TotalTokens: true, ReasoningTokens: true,
CachedTokens: true, CacheWriteTokens: true,
},
}
@@ -30,6 +30,9 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
t.Fatal(err)
}
object := decodeJSONObject(t, payload)
if string(decodeJSONObject(t, object["usage"])["reasoning_tokens"]) != "2" {
t.Fatalf("reasoning field missing: %s", payload)
}
if _, ok := object["usage_presence"]; !ok {
t.Fatalf("missing usage_presence: %s", payload)
}
@@ -46,6 +49,34 @@ func TestGenerateResponseUsagePresenceJSON(t *testing.T) {
}
}
func TestCostJSONPreservesUnknownAndReportedZero(t *testing.T) {
var old promptkit.RunResult
if err := json.Unmarshal([]byte(`{"prompt_id":"old"}`), &old); err != nil || old.CostUSD != (promptkit.ObservedCost{}) {
t.Fatalf("old result=%+v err=%v", old, err)
}
value := promptkit.RunResult{CostUSD: promptkit.ObservedCost{Value: .125, Known: true, Complete: false, Source: promptkit.CostSourceProvider}}
data, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
var decoded promptkit.RunResult
if err := json.Unmarshal(data, &decoded); err != nil || decoded.CostUSD != value.CostUSD {
t.Fatalf("round trip=%+v err=%v", decoded, err)
}
zero := 0.0
data, err = json.Marshal(promptkit.GenerateResponse{CostUSD: &zero})
if err != nil || string(decodeJSONObject(t, data)["cost_usd"]) != "0" {
t.Fatalf("reported zero=%s err=%v", data, err)
}
data, err = json.Marshal(promptkit.GenerateResponse{})
if err != nil {
t.Fatal(err)
}
if _, present := decodeJSONObject(t, data)["cost_usd"]; present {
t.Fatalf("unknown cost=%s", data)
}
}
func TestRemovedPromptVersionIsAbsentFromJSON(t *testing.T) {
values := map[string]any{
"prepared": promptkit.PreparedRun{PromptID: "prompt"},
@@ -173,6 +204,7 @@ func TestRunResultJSONContractRoundTripsAllFields(t *testing.T) {
"effective_model_params",
"input_hashes",
"usage",
"cost_usd",
"start_time",
"end_time",
"duration_ms",

View File

@@ -50,10 +50,12 @@ func TestPublicLLMClientAdapterGivesClientOwnedNestedValues(t *testing.T) {
func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
presence := &TokenUsagePresence{PromptTokens: true}
cost := .25
client := &usagePresenceClient{response: &GenerateResponse{
Content: "ok",
Usage: TokenUsage{PromptTokens: 0, TotalTokens: 5},
UsagePresence: presence,
CostUSD: &cost,
}}
adapter := publicLLMClientAdapter{client: client}
got, err := adapter.Generate(context.Background(), domain.GenerateRequest{})
@@ -61,17 +63,23 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
t.Fatal(err)
}
presence.PromptTokens = false
cost = .5
if got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.UsagePresence.TotalTokens || got.Usage.TotalTokens != 5 {
t.Fatalf("copied response = %+v", got)
}
if got.CostUSD == nil || *got.CostUSD != .25 {
t.Fatalf("copied cost=%v", got.CostUSD)
}
}
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {
presence := &TokenUsagePresence{PromptTokens: true}
cost := .125
source := &GenerateResponse{
Content: "returned with error",
Usage: TokenUsage{PromptTokens: 6},
UsagePresence: presence,
CostUSD: &cost,
}
marker := errors.New("injected failure")
adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}}
@@ -82,7 +90,8 @@ func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T
}
source.Content = "mutated after return"
presence.PromptTokens = false
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens {
cost = .5
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.CostUSD == nil || *got.CostUSD != .125 {
t.Fatalf("adapter response was not owned: %+v", got)
}
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"reflect"
"testing"
"time"
@@ -11,6 +12,66 @@ import (
"go.maximumdirect.net/promptkit"
)
func TestContextualObserversSelectIndependentRuns(t *testing.T) {
var order []string
var events []promptkit.ExecutionEvent
observer := func(name string) promptkit.ExecutionObserver {
return func(event promptkit.ExecutionEvent) {
order = append(order, name)
if name == "inherited" {
events = append(events, event)
}
}
}
base := context.Background()
if promptkit.WithExecutionObserver(base, nil) != base {
t.Fatal("nil observer changed context")
}
ctx := promptkit.WithExecutionObserver(base, observer("inherited"))
ctx = promptkit.WithExecutionObserver(ctx, observer("new"))
ctx = promptkit.WithExecutionObserver(ctx, nil)
child := observationEngine(t, observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: "child"}, nil
}))
var childResult *promptkit.RunResult
outer := observationEngine(t, observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
var err error
childResult, err = child.Run(callCtx, promptkit.RunRequest{PromptID: "prompt", Observer: observer("child explicit")})
return &promptkit.GenerateResponse{Content: "outer"}, err
}))
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: observer("outer explicit")})
if err != nil {
t.Fatal(err)
}
want := []string{"outer explicit", "inherited", "new", "child explicit", "inherited", "new", "child explicit", "inherited", "new", "child explicit", "inherited", "new", "outer explicit", "inherited", "new", "outer explicit", "inherited", "new"}
if !reflect.DeepEqual(order, want) {
t.Fatalf("observer order = %v", order)
}
if childResult.RunID == "" || result.RunID == "" || childResult.RunID == result.RunID {
t.Fatalf("run IDs: outer=%q child=%q", result.RunID, childResult.RunID)
}
counts := map[string]int{}
for _, event := range events {
counts[event.RunID]++
}
if counts[result.RunID] != 3 || counts[childResult.RunID] != 3 || len(counts) != 2 {
t.Fatalf("events = %+v", events)
}
order = nil
if _, err := child.Run(base, promptkit.RunRequest{PromptID: "prompt"}); err != nil {
t.Fatal(err)
}
if len(order) != 0 {
t.Fatalf("unrelated context selected observer: %v", order)
}
if _, err := child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"inherited", "new", "inherited", "new", "inherited", "new"}) {
t.Fatalf("context-only observer order = %v", order)
}
}
type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
func (f observationClientFunc) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {

View File

@@ -16,6 +16,19 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
}
func TestInjectedReasoningPresenceReachesObservation(t *testing.T) {
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok", UsagePresence: &promptkit.TokenUsagePresence{ReasoningTokens: true}}})
var events []promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
if err != nil || result == nil || len(events) != 3 {
t.Fatalf("result=%+v events=%+v error=%v", result, events, err)
}
want := promptkit.ObservedTokenCount{Known: true, Complete: true}
if events[1].Usage.ReasoningTokens != want || events[2].Usage.ReasoningTokens != want {
t.Fatalf("events=%+v", events)
}
}
func TestContinuationObservationAncestry(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}}
engine := observationEngine(t, client)

View File

@@ -17,6 +17,20 @@ import (
// event, and Promptkit keeps no asynchronous queue or durable history.
type ExecutionObserver func(ExecutionEvent)
// WithExecutionObserver adds an observer to executions using the returned
// context. Request observers run first, then inherited contextual observers
// in registration order. A nil observer leaves the context unchanged.
// Prepared executions use their execution context, not their preparation
// context. Callers own context propagation and synchronization of observers.
func WithExecutionObserver(ctx context.Context, observer ExecutionObserver) context.Context {
if observer == nil {
return ctx
}
return observation.WithObserver(ctx, func(event domain.ExecutionEvent) {
observer(fromDomainExecutionEvent(event))
})
}
// ComposeObservers returns a synchronous ordered fan-out of non-nil observers.
// It copies the input slice; a panic propagates and prevents later callbacks.
func ComposeObservers(observers ...ExecutionObserver) ExecutionObserver {
@@ -83,11 +97,24 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount
}
// CacheHitRatio returns the fraction of input tokens served from cache when
// both counts are known, complete, and consistent, with positive input and
// reads between zero and input inclusive. Otherwise it returns (0, false).
// Cache writes are independent; aggregate ratios use summed counts.
func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) {
input, reads := u.PromptTokens, u.CachedTokens
if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value {
return 0, false
}
return float64(reads.Value) / float64(input.Value), true
}
// ExecutionEvent contains metadata only, not provider text, request content,
// credentials, URLs, headers, or error objects. It has no stable JSON contract.
// Identifiers themselves may be sensitive. Nonterminal aggregate counters are
@@ -127,6 +154,7 @@ type ExecutionEvent struct {
StatusCode int
RetriesExhausted bool
Usage ObservedTokenUsage
CostUSD ObservedCost
Attempts uint64
TransientRetries uint64
RepairCalls uint64
@@ -144,8 +172,9 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
GenerationDuration: value.GenerationDuration, RetryDelay: value.RetryDelay, Duration: value.Duration,
Outcome: ExecutionOutcome(value.Outcome), Failure: ExecutionFailureKind(value.Failure),
StatusCode: value.StatusCode, RetriesExhausted: value.RetriesExhausted,
CostUSD: fromDomainCost(value.CostUSD),
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens),
TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
TotalTokens: count(value.Usage.TotalTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)},
Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
}
}

View File

@@ -15,6 +15,8 @@ const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
// A successful PrepareExecution may retain an Observer function in this
// handle's private payload. The callback and other private execution settings
// are cleared on claim completion or Discard. Details never exposes them.
// Contextual observers are selected from the RunPrepared context, not retained
// from preparation.
// It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for
// execution of the frozen snapshot. Only RunPrepared can produce its completed
// [Exchange]; preparation alone does not.

View File

@@ -3,6 +3,7 @@ package promptkit_test
import (
"context"
"errors"
"reflect"
"sync"
"testing"
@@ -10,6 +11,50 @@ import (
"go.maximumdirect.net/promptkit"
)
func TestPreparedObservationUsesExecutionContext(t *testing.T) {
engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
foreign := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
var order []string
observe := func(name string) promptkit.ExecutionObserver {
return func(promptkit.ExecutionEvent) { order = append(order, name) }
}
prepareCtx := promptkit.WithExecutionObserver(context.Background(), observe("preparation"))
executeCtx := promptkit.WithExecutionObserver(context.Background(), observe("execution"))
handle, err := engine.PrepareExecution(prepareCtx, promptkit.RunRequest{PromptID: "prepared", Observer: observe("explicit")})
if err != nil {
t.Fatal(err)
}
if len(order) != 0 {
t.Fatalf("preparation emitted events: %v", order)
}
if _, err := foreign.RunPrepared(executeCtx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
t.Fatalf("foreign claim: %v, %v", err, order)
}
if _, err := engine.RunPrepared(executeCtx, handle); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"explicit", "execution", "explicit", "execution", "explicit", "execution"}) {
t.Fatalf("observer order: %v", order)
}
order = nil
if _, err := engine.RunPrepared(executeCtx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
t.Fatalf("reused claim: %v, %v", err, order)
}
if _, err := engine.RunPrepared(executeCtx, nil); !errors.Is(err, promptkit.ErrInvalidRequest) || len(order) != 0 {
t.Fatalf("nil claim: %v, %v", err, order)
}
handle, err = engine.PrepareExecution(prepareCtx, promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatal(err)
}
if _, err := engine.RunPrepared(executeCtx, handle); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(order, []string{"execution", "execution", "execution"}) {
t.Fatalf("context-only order: %v", order)
}
}
func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) {
engine := newPreparedContractEngine(t, &preparedRecordingClient{response: &promptkit.GenerateResponse{Content: "ok"}}, "hello")
var events []promptkit.ExecutionEvent

View File

@@ -28,7 +28,11 @@ type ProfileInspection struct {
// caller-owned. Nil pointers denote absent fields; non-nil pointers preserve
// explicit zero values and empty collections in JSON and YAML.
type BackendInspection struct {
BackendID string `json:"id" yaml:"id"`
BackendID string `json:"id" yaml:"id"`
// Provider is the normalized declaration, not a manufactured runtime default.
Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"`
// ZeroCost preserves an explicitly declared false as well as true.
ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"`
Endpoint string `json:"endpoint" yaml:"endpoint"`
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
ConcurrencyLimit *int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit,omitempty"`

View File

@@ -13,7 +13,7 @@ import (
func TestResourceInspectionGenericEncoding(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
"backends": []any{map[string]any{"id": "local", "provider": " VLLM ", "zero_cost": false, "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}},
"prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}}})
@@ -77,7 +77,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
}
}
b := decode(backend)
if b["id"] != "local" || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
if b["id"] != "local" || b["provider"] != "vllm" || b["zero_cost"] != false || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
t.Errorf("backend = %#v", b)
}
params := b["extra_params"].(map[string]any)
@@ -92,7 +92,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
t.Errorf("reachability = %#v", reach)
}
absent(b, "extends")
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit")
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit", "provider", "zero_cost")
p := decode(profile)
if p["id"] != "profile" || p["backend"] != "local" || p["model"] != "model" || p["endpoint"] != "" || p["api_key_env"] != "" || p["service_tier"] != "" || p["reasoning_effort"] != "" || p["api_key_required"] != false || fmt.Sprint(p["temperature"]) != "0" || fmt.Sprint(p["top_p"]) != "0" || fmt.Sprint(p["max_tokens"]) != "0" || fmt.Sprint(p["timeout_seconds"]) != "0" || len(p["extra_params"].(map[string]any)) != 0 {
t.Errorf("profile = %#v", p)

View File

@@ -65,3 +65,21 @@ func TestSharedSmokeSettingsWorkWithBackendSource(t *testing.T) {
t.Fatalf("backend source = %+v, %v", result, err)
}
}
func TestBackendSourcePreservesProviderFieldPresence(t *testing.T) {
blank, disabled := "", false
result, err := configkit.Load[promptkit.Settings](promptkit.BackendSource("backends", []promptkit.BackendConfig{{ID: "omitted"}, {ID: "supplied", Provider: &blank, ZeroCost: &disabled}}))
if err != nil {
t.Fatal(err)
}
if result.Config.Backends[0].Provider != nil || result.Config.Backends[0].ZeroCost != nil {
t.Fatalf("omitted values: %+v", result.Config.Backends[0])
}
got := result.Config.Backends[1]
if got.Provider == nil || *got.Provider != "" || got.ZeroCost == nil || *got.ZeroCost {
t.Fatalf("supplied values: %+v", got)
}
if local := promptkit.LocalBackend("https://example.invalid/v1", 0); local.Provider != nil || local.ZeroCost != nil {
t.Fatalf("local defaults: %+v", local)
}
}

View File

@@ -368,6 +368,7 @@ func fromDomainObservedTokenUsage(usage domain.ObservedTokenUsage) ObservedToken
return ObservedTokenUsage{
PromptTokens: fromDomainObservedTokenCount(usage.PromptTokens),
CompletionTokens: fromDomainObservedTokenCount(usage.CompletionTokens),
ReasoningTokens: fromDomainObservedTokenCount(usage.ReasoningTokens),
TotalTokens: fromDomainObservedTokenCount(usage.TotalTokens),
CachedTokens: fromDomainObservedTokenCount(usage.CachedTokens),
CacheWriteTokens: fromDomainObservedTokenCount(usage.CacheWriteTokens),

View File

@@ -5,13 +5,40 @@ import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestSmokeAuthenticationUsesProviderDefault(t *testing.T) {
var path string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path = r.URL.Path
fmt.Fprint(w, `{"data":{"label":"member","is_free_tier":false,"usage":0}}`)
}))
defer server.Close()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("diagnostic", map[string]any{
"backends": []any{map[string]any{"id": "renamed", "provider": "openrouter", "endpoint": server.URL + "/v1"}},
})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithBundledOpenRouter(false))
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectBackend(context.Background(), "renamed")
if err != nil || inspection.SmokeTests != nil {
t.Fatalf("declared metadata: %+v, %v", inspection, err)
}
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestAuthentication, BackendID: "renamed"})
if err != nil || !report.Supported || !report.Completed || path != "/v1/key" || report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted {
t.Fatalf("report=%+v path=%q err=%v", report, path, err)
}
}
type gatedSmokeClient struct {
started chan promptkit.GenerateRequest
release <-chan struct{}

View File

@@ -298,6 +298,8 @@ type RunResult struct {
InputHashes map[string]string `json:"input_hashes,omitempty"`
// Usage is the token accounting reported by the LLM client.
Usage TokenUsage `json:"usage"`
// CostUSD is the observed charge subtotal across dispatched attempts.
CostUSD ObservedCost `json:"cost_usd"`
// StartTime is the UTC time immediately before ordinary Run preparation or
// after RunPrepared claims its handle.
StartTime time.Time `json:"start_time,omitempty"`
@@ -568,9 +570,11 @@ type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"`
// CompletionTokens is the reported generated-token count.
CompletionTokens int `json:"completion_tokens"`
// ReasoningTokens is the reported reasoning subset of completion tokens.
ReasoningTokens int `json:"reasoning_tokens"`
// TotalTokens is the reported total-token count.
TotalTokens int `json:"total_tokens"`
// CachedTokens is the reported cached-input-token count.
// CachedTokens is the reported input-token cache-read count.
CachedTokens int `json:"cached_tokens"`
// CacheWriteTokens is the reported cache-write-token count.
CacheWriteTokens int `json:"cache_write_tokens"`
@@ -656,7 +660,9 @@ type StructuredOutputJSONSpec struct {
// treated as known and zero counts as unknown for observation only; the legacy
// interface cannot distinguish omitted zero from measured zero. An explicit
// UsagePresence can report known zero counts. Result usage sums the supplied
// Usage values independently of presence.
// Usage values independently of presence. A non-nil CostUSD is copied and
// retained for attempt accounting even when returned alongside an error;
// non-finite values are treated as unknown before configured-zero fallback.
type LLMClient interface {
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
}
@@ -690,6 +696,8 @@ type GenerateResponse struct {
// UsagePresence states which usage fields were reported, including zeros.
// Nil uses the legacy observation fallback described by LLMClient.
UsagePresence *TokenUsagePresence `json:"usage_presence,omitempty"`
// CostUSD is an optional reported charge; a pointer to zero reports zero.
CostUSD *float64 `json:"cost_usd,omitempty"`
}
// TokenUsagePresence identifies provider-reported token counts for observation.
@@ -697,6 +705,7 @@ type GenerateResponse struct {
type TokenUsagePresence struct {
PromptTokens bool `json:"prompt_tokens"`
CompletionTokens bool `json:"completion_tokens"`
ReasoningTokens bool `json:"reasoning_tokens"`
TotalTokens bool `json:"total_tokens"`
CachedTokens bool `json:"cached_tokens"`
CacheWriteTokens bool `json:"cache_write_tokens"`