Decode provider reasoning and cache usage details
This commit is contained in:
@@ -334,6 +334,7 @@ type GenerateResponse struct {
|
||||
type TokenUsagePresence struct {
|
||||
PromptTokens bool
|
||||
CompletionTokens bool
|
||||
ReasoningTokens bool
|
||||
TotalTokens bool
|
||||
CachedTokens bool
|
||||
CacheWriteTokens bool
|
||||
@@ -343,6 +344,7 @@ type TokenUsagePresence struct {
|
||||
type TokenUsage struct {
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
ReasoningTokens int
|
||||
TotalTokens int
|
||||
CachedTokens int
|
||||
CacheWriteTokens int
|
||||
|
||||
@@ -56,6 +56,7 @@ type ObservedTokenCount struct {
|
||||
type ObservedTokenUsage struct {
|
||||
PromptTokens ObservedTokenCount
|
||||
CompletionTokens ObservedTokenCount
|
||||
ReasoningTokens ObservedTokenCount
|
||||
TotalTokens ObservedTokenCount
|
||||
CachedTokens ObservedTokenCount
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
|
||||
@@ -88,6 +88,7 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
||||
// PreparedGeneration holds one encoded request and resolved authentication for
|
||||
// repeated attempts. Each execution creates a fresh HTTP request and context.
|
||||
type PreparedGeneration struct {
|
||||
provider string
|
||||
endpoint string
|
||||
payload []byte
|
||||
apiKey string
|
||||
@@ -156,7 +157,7 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
|
||||
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
|
||||
}
|
||||
return &PreparedGeneration{
|
||||
endpoint: endpoint, payload: payload, apiKey: apiKey,
|
||||
endpoint: endpoint, payload: payload, apiKey: apiKey, provider: EffectiveProvider(req.Target.Provider),
|
||||
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
|
||||
structuredOutputRequested: wirePayload["response_format"] != nil,
|
||||
}, nil
|
||||
@@ -266,16 +267,37 @@ func (p *PreparedGeneration) execute(ctx context.Context, captureEvidence bool)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
var writes, reasoning *int
|
||||
switch p.provider {
|
||||
case "openrouter", "vllm":
|
||||
var details map[string]json.RawMessage
|
||||
if json.Unmarshal(wireResp.Usage.CompletionTokensDetails, &details) == nil {
|
||||
reasoning = optionalInteger(details["reasoning_tokens"])
|
||||
}
|
||||
if p.provider == "openrouter" {
|
||||
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CacheWriteTokens)
|
||||
} else {
|
||||
writes = optionalInteger(wireResp.Usage.PromptTokensDetails.CreatedCacheTokens)
|
||||
}
|
||||
default:
|
||||
if len(wireResp.Usage.CacheWriteTokens) > 0 && json.Unmarshal(wireResp.Usage.CacheWriteTokens, &writes) != nil {
|
||||
outcome.Err = fmt.Errorf("%w: failed to decode response", ErrMalformedResponse)
|
||||
if contextErr := observedContextError(requestContext, ctx, outcome.Err); contextErr != nil {
|
||||
outcome.Err = contextErr
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
}
|
||||
outcome.ResponseDecoded = true
|
||||
outcome.Usage = domain.TokenUsage{
|
||||
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
|
||||
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
|
||||
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
|
||||
CacheWriteTokens: tokenCount(writes), ReasoningTokens: tokenCount(reasoning),
|
||||
}
|
||||
outcome.UsagePresence = &domain.TokenUsagePresence{
|
||||
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
|
||||
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
|
||||
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
|
||||
CacheWriteTokens: writes != nil, ReasoningTokens: reasoning != nil,
|
||||
}
|
||||
|
||||
if len(wireResp.Choices) == 0 {
|
||||
@@ -540,12 +562,26 @@ type openAIChatResponse struct {
|
||||
CompletionTokens *int `json:"completion_tokens"`
|
||||
TotalTokens *int `json:"total_tokens"`
|
||||
PromptTokensDetails struct {
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
CachedTokens *int `json:"cached_tokens"`
|
||||
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
||||
CreatedCacheTokens json.RawMessage `json:"created_cache_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
CacheWriteTokens *int `json:"cache_write_tokens"`
|
||||
CompletionTokensDetails json.RawMessage `json:"completion_tokens_details"`
|
||||
CacheWriteTokens json.RawMessage `json:"cache_write_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
|
||||
func optionalInteger(raw json.RawMessage) *int {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
var value *int
|
||||
if json.Unmarshal(raw, &value) != nil {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type openAIResponseFormat struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
||||
|
||||
@@ -882,6 +882,45 @@ func TestOpenAICompatibleUsagePresence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderSpecificUsageDetails(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, provider, usage string
|
||||
writes, reasoning int
|
||||
writesKnown, reasoningKnown bool
|
||||
}{
|
||||
{name: "openrouter nested", provider: "openrouter", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 3, reasoning: 4, writesKnown: true, reasoningKnown: true},
|
||||
{name: "vllm nested", provider: "vllm", usage: `"prompt_tokens_details":{"cached_tokens":2,"cache_write_tokens":3,"created_cache_tokens":7},"completion_tokens_details":{"reasoning_tokens":0},"cache_write_tokens":9`, writes: 7, writesKnown: true, reasoningKnown: true},
|
||||
{name: "generic ignores details", provider: "openai-compatible", usage: `"prompt_tokens_details":{"cache_write_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"cache_write_tokens":9`, writes: 9, writesKnown: true},
|
||||
{name: "missing selected field", provider: "openrouter", usage: `"prompt_tokens_details":{"created_cache_tokens":7},"cache_write_tokens":9`},
|
||||
{name: "invalid optional fields", provider: "vllm", usage: `"prompt_tokens_details":{"created_cache_tokens":"7","cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":2.5},"cache_write_tokens":{}`},
|
||||
{name: "overflow and null", provider: "openrouter", usage: `"prompt_tokens_details":{"cache_write_tokens":999999999999999999999999},"completion_tokens_details":{"reasoning_tokens":null}`},
|
||||
{name: "vllm disabled details", provider: "vllm", usage: `"prompt_tokens":12`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{`+tc.usage+`},"choices":[{"message":{"content":"ok"}}]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = tc.provider
|
||||
response, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).Generate(context.Background(), request)
|
||||
if err != nil || response.Usage.CacheWriteTokens != tc.writes || response.Usage.ReasoningTokens != tc.reasoning || response.UsagePresence.CacheWriteTokens != tc.writesKnown || response.UsagePresence.ReasoningTokens != tc.reasoningKnown {
|
||||
t.Fatalf("response=%+v error=%v", response, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
provider := newRecordingProvider(t)
|
||||
provider.respond(http.StatusOK, `{"usage":{"completion_tokens_details":{"reasoning_tokens":5}},"choices":[]}`)
|
||||
request := provider.generateRequest()
|
||||
request.Target.Provider = "openrouter"
|
||||
prepared, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{}).PrepareGeneration(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcome := prepared.Execute(context.Background())
|
||||
if !errors.Is(outcome.Err, ErrMalformedResponse) || outcome.Usage.ReasoningTokens != 5 || !outcome.UsagePresence.ReasoningTokens {
|
||||
t.Fatalf("outcome=%+v", outcome)
|
||||
}
|
||||
}
|
||||
|
||||
func checkOptionalResponseFieldOmission(t *testing.T) {
|
||||
provider := newRecordingProvider(t)
|
||||
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{})
|
||||
|
||||
Reference in New Issue
Block a user