Capture optional provider reported costs

This commit is contained in:
2026-11-22 18:25:32 +00:00
parent 6ed0e037c4
commit cd0e1f5d39
5 changed files with 66 additions and 2 deletions

View File

@@ -170,6 +170,9 @@ come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed, its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
fractional, or unrepresentable new details remain unknown without invalidating fractional, or unrepresentable new details remain unknown without invalidating
otherwise usable content. Existing common count decoding remains strict. otherwise usable content. Existing common count decoding remains strict.
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
amounts are ignored without invalidating content. The generic and vLLM
providers do not interpret that field.
Each common token field retains whether the provider reported a Each common token field retains whether the provider reported a
non-null integer, including zero. Omitted and `null` fields are unknown for non-null integer, including zero. Omitted and `null` fields are unknown for

View File

@@ -304,6 +304,8 @@ Validation: pure accounting and public serialization tests; no pricing library.
## Stage 10 — Optional Cost At The Model-Client Boundary ## Stage 10 — Optional Cost At The Model-Client Boundary
**Completed.**
- Decode OpenRouter `usage.cost` independently using local tolerant finite-float - Decode OpenRouter `usage.cost` independently using local tolerant finite-float
handling. Missing/null/string/object/overflow values are unknown; invalid handling. Missing/null/string/object/overflow values are unknown; invalid
framing or common envelope fields retain existing failures. Generic/vLLM do framing or common envelope fields retain existing failures. Generic/vLLM do

View File

@@ -105,6 +105,7 @@ type AttemptOutcome struct {
StatusCode int StatusCode int
Usage domain.TokenUsage Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence UsagePresence *domain.TokenUsagePresence
CostUSD *float64
RetryNotBefore time.Time RetryNotBefore time.Time
Evidence HTTPExecutionEvidence Evidence HTTPExecutionEvidence
StructuredOutputRequested *bool StructuredOutputRequested *bool
@@ -299,6 +300,9 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil, TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil, CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
} }
if p.provider == "openrouter" {
outcome.CostUSD = optionalFloat(wireResp.Usage.Cost)
}
if len(wireResp.Choices) == 0 { if len(wireResp.Choices) == 0 {
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse) outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
@@ -319,7 +323,7 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
} }
return outcome return outcome
} }
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence} outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: outcome.CostUSD}
outcome.GenerationCompleted = true outcome.GenerationCompleted = true
if captureEvidence { if captureEvidence {
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil { if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
@@ -568,6 +572,7 @@ type openAIChatResponse struct {
} `json:"prompt_tokens_details"` } `json:"prompt_tokens_details"`
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"` CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"` CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
Cost json.RawMessage `json:"cost"`
} `json:"usage"` } `json:"usage"`
} }
@@ -582,6 +587,17 @@ func optionalInteger(raw json.RawMessage) *int {
return value return value
} }
func optionalFloat(raw json.RawMessage) *float64 {
if len(raw) == 0 {
return nil
}
var value *float64
if json.Unmarshal(raw, &value) != nil || (value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0))) {
return nil
}
return value
}
type openAIResponseFormat struct { type openAIResponseFormat struct {
Type string `json:"type"` Type string `json:"type"`
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"` JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`

View File

@@ -921,6 +921,46 @@ func TestProviderSpecificUsageDetails(t *testing.T) {
} }
} }
func TestProviderReportedCostIsOptional(t *testing.T) {
for _, tc := range []struct {
name, provider, cost string
known bool
value float64
}{
{"charge", "openrouter", `0.002`, true, .002},
{"zero", "openrouter", `0`, true, 0},
{"null", "openrouter", `null`, false, 0},
{"string", "openrouter", `"0.2"`, false, 0},
{"object", "openrouter", `{}`, false, 0},
{"overflow", "openrouter", `1e9999`, false, 0},
{"generic ignores cost", "openai-compatible", `0.02`, false, 0},
{"vllm ignores cost", "vllm", `0.02`, false, 0},
} {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"cost":`+tc.cost+`},"choices":[{"message":{"content":"ok"}}]}`)
request := provider.generateRequest()
request.Target.Provider = tc.provider
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
if err != nil || response == nil || (response.CostUSD != nil) != tc.known || (response.CostUSD != nil && *response.CostUSD != tc.value) {
t.Fatalf("response=%+v err=%v", response, err)
}
})
}
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"cost":0.02},"choices":[]}`)
request := provider.generateRequest()
request.Target.Provider = "openrouter"
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.Execute(context.Background())
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.CostUSD == nil || *outcome.CostUSD != .02 {
t.Fatalf("outcome=%+v", outcome)
}
}
func checkOptionalResponseFieldOmission(t *testing.T) { func checkOptionalResponseFieldOmission(t *testing.T) {
provider := newRecordingProvider(t) provider := newRecordingProvider(t)
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{}) client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})

View File

@@ -74,10 +74,12 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) { func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {
presence := &TokenUsagePresence{PromptTokens: true} presence := &TokenUsagePresence{PromptTokens: true}
cost := .125
source := &GenerateResponse{ source := &GenerateResponse{
Content: "returned with error", Content: "returned with error",
Usage: TokenUsage{PromptTokens: 6}, Usage: TokenUsage{PromptTokens: 6},
UsagePresence: presence, UsagePresence: presence,
CostUSD: &cost,
} }
marker := errors.New("injected failure") marker := errors.New("injected failure")
adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}} adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}}
@@ -88,7 +90,8 @@ func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T
} }
source.Content = "mutated after return" source.Content = "mutated after return"
presence.PromptTokens = false presence.PromptTokens = false
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens { cost = .5
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.CostUSD == nil || *got.CostUSD != .125 {
t.Fatalf("adapter response was not owned: %+v", got) t.Fatalf("adapter response was not owned: %+v", got)
} }
} }