Decode provider reasoning and cache usage details

This commit is contained in:
2026-11-22 18:20:11 +00:00
parent c3257b63f0
commit db1cad6d2d
11 changed files with 108 additions and 9 deletions

View File

@@ -334,6 +334,7 @@ type GenerateResponse struct {
type TokenUsagePresence struct {
PromptTokens bool
CompletionTokens bool
ReasoningTokens bool
TotalTokens bool
CachedTokens bool
CacheWriteTokens bool
@@ -343,6 +344,7 @@ type TokenUsagePresence struct {
type TokenUsage struct {
PromptTokens int
CompletionTokens int
ReasoningTokens int
TotalTokens int
CachedTokens int
CacheWriteTokens int

View File

@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
type ObservedTokenUsage struct {
PromptTokens ObservedTokenCount
CompletionTokens ObservedTokenCount
ReasoningTokens ObservedTokenCount
TotalTokens ObservedTokenCount
CachedTokens ObservedTokenCount
CacheWriteTokens ObservedTokenCount

View File

@@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
// PreparedGeneration holds one encoded request and resolved authentication for
// repeated attempts. Each execution creates a fresh HTTP request and context.
type PreparedGeneration struct {
provider string
endpoint string
payload []byte
apiKey string
@@ -156,7 +157,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
}
return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey,
endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider),
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
structuredOutputRequested: wirePayload["response_format"] != nil,
}, nil
@@ -266,16 +267,37 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
}
return outcome
}
var writes, reasoning *int
switch p.provider {
case "openrouter", "vllm":
var details map[string]json.RawMessage
if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil {
reasoning = optionalInteger(details["reasoning_tokens"])
}
if p.provider == "openrouter" {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens)
} else {
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens)
}
default:
if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil {
outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
outcome.Err = contextErr
}
return outcome
}
}
outcome.ResponseDecoded = true
outcome.Usage = domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning),
}
outcome.UsagePresence = &domain.TokenUsagePresence{
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
}
if len(wireResp.Choices) == 0 {
@@ -540,12 +562,26 @@ type openAIChatResponse struct {
CompletionTokens *int `json:"completion_tokens"`
TotalTokens *int `json:"total_tokens"`
PromptTokensDetails struct {
CachedTokens *int `json:"cached_tokens"`
CachedTokens *int `json:"cached_tokens"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"`
} `json:"prompt_tokens_details"`
CacheWriteTokens *int `json:"cache_write_tokens"`
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
} `json:"usage"`
}
func optionalInteger(raw json.RawMessage) *int {
if len(raw) == 0 {
return nil
}
var value *int
if json.Unmarshal(raw, &value) != nil {
return nil
}
return value
}
type openAIResponseFormat struct {
Type string `json:"type"`
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`

View File

@@ -882,6 +882,45 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
}
}
func TestProviderSpecificUsageDetails(t *testing.T) {
for _, tc := range []struct {
name, provider, usage string
writes, reasoning int
writesKnown, reasoningKnown bool
}{
{name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true},
{name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true},
{name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true},
{name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`},
{name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`},
{name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`},
{name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`},
} {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`)
request := provider.generateRequest()
request.Target.Provider = tc.provider
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown {
t.Fatalf("response=%+v error=%v", response, err)
}
})
}
provider := newRecordingProvider(t)
provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`)
request := provider.generateRequest()
request.Target.Provider = "openrouter"
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
if err != nil {
t.Fatal(err)
}
outcome := prepared.Execute(context.Background())
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens {
t.Fatalf("outcome=%+v", outcome)
}
}
func checkOptionalResponseFieldOmission(t *testing.T) {
provider := newRecordingProvider(t)
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})