Aggregate reasoning usage and derive cache hit ratios
This commit is contained in:
@@ -36,7 +36,7 @@ func (g *Group) Observe(event domain.ExecutionEvent) {
|
||||
g.repairs += event.RepairCalls
|
||||
if g.terminals == 0 {
|
||||
complete := domain.ObservedTokenCount{Complete: true}
|
||||
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||
}
|
||||
mergeGroupUsage(&g.usage, event.Usage)
|
||||
g.terminals++
|
||||
@@ -52,7 +52,7 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary {
|
||||
}
|
||||
if len(g.members) == 0 {
|
||||
zero := domain.ObservedTokenCount{Known: true, Complete: true}
|
||||
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
|
||||
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, ReasoningTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -67,6 +67,7 @@ func mergeGroupUsage(total *domain.ObservedTokenUsage, next domain.ObservedToken
|
||||
}
|
||||
merge(&total.PromptTokens, next.PromptTokens)
|
||||
merge(&total.CompletionTokens, next.CompletionTokens)
|
||||
merge(&total.ReasoningTokens, next.ReasoningTokens)
|
||||
merge(&total.TotalTokens, next.TotalTokens)
|
||||
merge(&total.CachedTokens, next.CachedTokens)
|
||||
merge(&total.CacheWriteTokens, next.CacheWriteTokens)
|
||||
|
||||
@@ -260,7 +260,7 @@ type usageField struct {
|
||||
known, complete bool
|
||||
}
|
||||
type usageTotals struct {
|
||||
prompt, completion, total, cached, cacheWrite usageField
|
||||
prompt, completion, reasoning, total, cached, cacheWrite usageField
|
||||
}
|
||||
|
||||
// UsageAccumulator combines presence-aware usage across dispatched attempts.
|
||||
@@ -286,7 +286,7 @@ func (u UsageAccumulator) Observed() domain.ObservedTokenUsage {
|
||||
}
|
||||
|
||||
func completeUsageTotals() usageTotals {
|
||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
|
||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, reasoning: usageField{complete: true},
|
||||
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
|
||||
}
|
||||
|
||||
@@ -294,6 +294,7 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
|
||||
known := usagePresence(usage, presence)
|
||||
u.prompt.add(usage.PromptTokens, known.PromptTokens)
|
||||
u.completion.add(usage.CompletionTokens, known.CompletionTokens)
|
||||
u.reasoning.add(usage.ReasoningTokens, known.ReasoningTokens)
|
||||
u.total.add(usage.TotalTokens, known.TotalTokens)
|
||||
u.cached.add(usage.CachedTokens, known.CachedTokens)
|
||||
u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens)
|
||||
@@ -302,7 +303,8 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
|
||||
func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage {
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts),
|
||||
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
|
||||
ReasoningTokens: u.reasoning.snapshot(attempts),
|
||||
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
|
||||
CacheWriteTokens: u.cacheWrite.snapshot(attempts),
|
||||
}
|
||||
}
|
||||
@@ -328,6 +330,7 @@ func observedUsage(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens),
|
||||
CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens),
|
||||
ReasoningTokens: observedCount(usage.ReasoningTokens, known.ReasoningTokens),
|
||||
TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens),
|
||||
CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens),
|
||||
CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens),
|
||||
@@ -340,7 +343,8 @@ func usagePresence(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
|
||||
}
|
||||
return domain.TokenUsagePresence{
|
||||
PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0,
|
||||
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
|
||||
ReasoningTokens: usage.ReasoningTokens != 0,
|
||||
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
|
||||
CacheWriteTokens: usage.CacheWriteTokens != 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,3 +25,18 @@ func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) {
|
||||
t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReasoningUsagePresence(t *testing.T) {
|
||||
var usage UsageAccumulator
|
||||
usage.Add(domain.TokenUsage{}, &domain.TokenUsagePresence{ReasoningTokens: true})
|
||||
usage.Add(domain.TokenUsage{ReasoningTokens: 4}, nil)
|
||||
got := usage.Observed().ReasoningTokens
|
||||
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: true}) {
|
||||
t.Fatalf("usage=%+v", got)
|
||||
}
|
||||
usage.Add(domain.TokenUsage{}, nil)
|
||||
got = usage.Observed().ReasoningTokens
|
||||
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}) {
|
||||
t.Fatalf("partial usage=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,7 +317,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
responses: []*domain.GenerateResponse{{
|
||||
Content: `{"repaired":true}`,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5,
|
||||
PromptTokens: 2, CompletionTokens: 3, ReasoningTokens: 2, TotalTokens: 5,
|
||||
CachedTokens: 7, CacheWriteTokens: 11,
|
||||
},
|
||||
}},
|
||||
@@ -328,7 +328,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
client := &sequenceLLM{responses: []*domain.GenerateResponse{{
|
||||
Content: `{"broken":true}`,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
|
||||
PromptTokens: 13, CompletionTokens: 17, ReasoningTokens: 4, TotalTokens: 19,
|
||||
CachedTokens: 23, CacheWriteTokens: 29,
|
||||
},
|
||||
}}}
|
||||
@@ -370,7 +370,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
||||
}
|
||||
wantUsage := domain.TokenUsage{
|
||||
PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24,
|
||||
PromptTokens: 15, CompletionTokens: 20, ReasoningTokens: 6, TotalTokens: 24,
|
||||
CachedTokens: 30, CacheWriteTokens: 40,
|
||||
}
|
||||
if result.Usage != wantUsage {
|
||||
|
||||
@@ -309,6 +309,7 @@ func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: total.PromptTokens + next.PromptTokens,
|
||||
CompletionTokens: total.CompletionTokens + next.CompletionTokens,
|
||||
ReasoningTokens: total.ReasoningTokens + next.ReasoningTokens,
|
||||
TotalTokens: total.TotalTokens + next.TotalTokens,
|
||||
CachedTokens: total.CachedTokens + next.CachedTokens,
|
||||
CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens,
|
||||
|
||||
Reference in New Issue
Block a user