Capture optional provider reported costs
This commit is contained in:
@@ -170,6 +170,9 @@ come from `usage.prompt_tokens_details.cache_write_tokens`; vLLM uses
|
|||||||
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
|
its top-level `usage.cache_write_tokens` mapping. Missing, null, malformed,
|
||||||
fractional, or unrepresentable new details remain unknown without invalidating
|
fractional, or unrepresentable new details remain unknown without invalidating
|
||||||
otherwise usable content. Existing common count decoding remains strict.
|
otherwise usable content. Existing common count decoding remains strict.
|
||||||
|
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
|
||||||
|
amounts are ignored without invalidating content. The generic and vLLM
|
||||||
|
providers do not interpret that field.
|
||||||
|
|
||||||
Each common token field retains whether the provider reported a
|
Each common token field retains whether the provider reported a
|
||||||
non-null integer, including zero. Omitted and `null` fields are unknown for
|
non-null integer, including zero. Omitted and `null` fields are unknown for
|
||||||
|
|||||||
@@ -304,6 +304,8 @@ Validation: pure accounting and public serialization tests; no pricing library.
|
|||||||
|
|
||||||
## Stage 10 — Optional Cost At The Model-Client Boundary
|
## Stage 10 — Optional Cost At The Model-Client Boundary
|
||||||
|
|
||||||
|
**Completed.**
|
||||||
|
|
||||||
- Decode OpenRouter `usage.cost` independently using local tolerant finite-float
|
- Decode OpenRouter `usage.cost` independently using local tolerant finite-float
|
||||||
handling. Missing/null/string/object/overflow values are unknown; invalid
|
handling. Missing/null/string/object/overflow values are unknown; invalid
|
||||||
framing or common envelope fields retain existing failures. Generic/vLLM do
|
framing or common envelope fields retain existing failures. Generic/vLLM do
|
||||||
|
|||||||
@@ -105,6 +105,7 @@ type AttemptOutcome struct {
|
|||||||
StatusCode int
|
StatusCode int
|
||||||
Usage domain.TokenUsage
|
Usage domain.TokenUsage
|
||||||
UsagePresence *domain.TokenUsagePresence
|
UsagePresence *domain.TokenUsagePresence
|
||||||
|
CostUSD *float64
|
||||||
RetryNotBefore time.Time
|
RetryNotBefore time.Time
|
||||||
Evidence HTTPExecutionEvidence
|
Evidence HTTPExecutionEvidence
|
||||||
StructuredOutputRequested *bool
|
StructuredOutputRequested *bool
|
||||||
@@ -299,6 +300,9 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
|
|||||||
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
|
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
|
||||||
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
|
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
|
||||||
}
|
}
|
||||||
|
if p.provider == "openrouter" {
|
||||||
|
outcome.CostUSD = optionalFloat(wireResp.Usage.Cost)
|
||||||
|
}
|
||||||
|
|
||||||
if len(wireResp.Choices) == 0 {
|
if len(wireResp.Choices) == 0 {
|
||||||
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||||
@@ -319,7 +323,7 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
|
|||||||
}
|
}
|
||||||
return outcome
|
return outcome
|
||||||
}
|
}
|
||||||
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
|
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: outcome.CostUSD}
|
||||||
outcome.GenerationCompleted = true
|
outcome.GenerationCompleted = true
|
||||||
if captureEvidence {
|
if captureEvidence {
|
||||||
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
|
if contextErr := observedContextError(requestContext, ctx, nil); contextErr != nil {
|
||||||
@@ -568,6 +572,7 @@ type openAIChatResponse struct {
|
|||||||
} `json:"prompt_tokens_details"`
|
} `json:"prompt_tokens_details"`
|
||||||
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
|
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
|
||||||
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
||||||
|
Cost json.RawMessage `json:"cost"`
|
||||||
} `json:"usage"`
|
} `json:"usage"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -582,6 +587,17 @@ func optionalInteger(raw json.RawMessage) *int {
|
|||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func optionalFloat(raw json.RawMessage) *float64 {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var value *float64
|
||||||
|
if json.Unmarshal(raw, &value) != nil || (value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0))) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
type openAIResponseFormat struct {
|
type openAIResponseFormat struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
||||||
|
|||||||
@@ -921,6 +921,46 @@ func TestProviderSpecificUsageDetails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProviderReportedCostIsOptional(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, provider, cost string
|
||||||
|
known bool
|
||||||
|
value float64
|
||||||
|
}{
|
||||||
|
{"charge", "openrouter", `0.002`, true, .002},
|
||||||
|
{"zero", "openrouter", `0`, true, 0},
|
||||||
|
{"null", "openrouter", `null`, false, 0},
|
||||||
|
{"string", "openrouter", `"0.2"`, false, 0},
|
||||||
|
{"object", "openrouter", `{}`, false, 0},
|
||||||
|
{"overflow", "openrouter", `1e9999`, false, 0},
|
||||||
|
{"generic ignores cost", "openai-compatible", `0.02`, false, 0},
|
||||||
|
{"vllm ignores cost", "vllm", `0.02`, false, 0},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
provider := newRecordingProvider(t)
|
||||||
|
provider.respond(http.StatusOK, `{"usage":{"cost":`+tc.cost+`},"choices":[{"message":{"content":"ok"}}]}`)
|
||||||
|
request := provider.generateRequest()
|
||||||
|
request.Target.Provider = tc.provider
|
||||||
|
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
|
||||||
|
if err != nil || response == nil || (response.CostUSD != nil) != tc.known || (response.CostUSD != nil && *response.CostUSD != tc.value) {
|
||||||
|
t.Fatalf("response=%+v err=%v", response, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
provider := newRecordingProvider(t)
|
||||||
|
provider.respond(http.StatusOK, `{"usage":{"cost":0.02},"choices":[]}`)
|
||||||
|
request := provider.generateRequest()
|
||||||
|
request.Target.Provider = "openrouter"
|
||||||
|
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
outcome := prepared.Execute(context.Background())
|
||||||
|
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.CostUSD == nil || *outcome.CostUSD != .02 {
|
||||||
|
t.Fatalf("outcome=%+v", outcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||||
|
|||||||
@@ -74,10 +74,12 @@ func TestPublicLLMClientAdapterCopiesUsagePresence(t *testing.T) {
|
|||||||
|
|
||||||
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {
|
func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T) {
|
||||||
presence := &TokenUsagePresence{PromptTokens: true}
|
presence := &TokenUsagePresence{PromptTokens: true}
|
||||||
|
cost := .125
|
||||||
source := &GenerateResponse{
|
source := &GenerateResponse{
|
||||||
Content: "returned with error",
|
Content: "returned with error",
|
||||||
Usage: TokenUsage{PromptTokens: 6},
|
Usage: TokenUsage{PromptTokens: 6},
|
||||||
UsagePresence: presence,
|
UsagePresence: presence,
|
||||||
|
CostUSD: &cost,
|
||||||
}
|
}
|
||||||
marker := errors.New("injected failure")
|
marker := errors.New("injected failure")
|
||||||
adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}}
|
adapter := publicLLMClientAdapter{client: responseAndErrorClient{response: source, err: marker}}
|
||||||
@@ -88,7 +90,8 @@ func TestPublicLLMClientAdapterPreservesOwnedResponseAlongsideError(t *testing.T
|
|||||||
}
|
}
|
||||||
source.Content = "mutated after return"
|
source.Content = "mutated after return"
|
||||||
presence.PromptTokens = false
|
presence.PromptTokens = false
|
||||||
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens {
|
cost = .5
|
||||||
|
if got.Content != "returned with error" || got.UsagePresence == nil || !got.UsagePresence.PromptTokens || got.CostUSD == nil || *got.CostUSD != .125 {
|
||||||
t.Fatalf("adapter response was not owned: %+v", got)
|
t.Fatalf("adapter response was not owned: %+v", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user