Aggregate reasoning usage and derive cache hit ratios
This commit is contained in:
53
cache_hit_contract_test.go
Normal file
53
cache_hit_contract_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) {
|
||||
known := func(v int) promptkit.ObservedTokenCount {
|
||||
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
input, reads promptkit.ObservedTokenCount
|
||||
ratio float64
|
||||
ok bool
|
||||
}{
|
||||
{name: "fraction", input: known(8), reads: known(3), ratio: .375, ok: true},
|
||||
{name: "zero reads", input: known(8), reads: known(0), ok: true},
|
||||
{name: "zero input", input: known(0), reads: known(0)},
|
||||
{name: "negative reads", input: known(8), reads: known(-1)},
|
||||
{name: "reads exceed input", input: known(8), reads: known(9)},
|
||||
{name: "unknown", input: known(8)},
|
||||
{name: "incomplete", input: known(8), reads: promptkit.ObservedTokenCount{Value: 3, Known: true}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
usage := promptkit.ObservedTokenUsage{PromptTokens: tc.input, CachedTokens: tc.reads}
|
||||
got, ok := usage.CacheHitRatio()
|
||||
if ok != tc.ok || math.Abs(got-tc.ratio) > 1e-12 {
|
||||
t.Fatalf("ratio=%v available=%t", got, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupCacheRatioUsesSummedTokensAndDeduplicatesTerminals(t *testing.T) {
|
||||
known := func(v int) promptkit.ObservedTokenCount {
|
||||
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
|
||||
}
|
||||
var group promptkit.ExecutionGroup
|
||||
first := promptkit.ExecutionEvent{RunID: "a", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(10), CachedTokens: known(10), ReasoningTokens: known(1)}}
|
||||
second := promptkit.ExecutionEvent{RunID: "b", Kind: promptkit.ExecutionFinished, Usage: promptkit.ObservedTokenUsage{PromptTokens: known(90), CachedTokens: known(0), ReasoningTokens: known(2)}}
|
||||
group.Observe(first)
|
||||
group.Observe(first)
|
||||
group.Observe(second)
|
||||
snapshot := group.Snapshot()
|
||||
ratio, ok := snapshot.Usage.CacheHitRatio()
|
||||
if !ok || math.Abs(ratio-.1) > 1e-12 || snapshot.Usage.ReasoningTokens != known(3) {
|
||||
t.Fatalf("summary=%+v ratio=%v available=%t", snapshot, ratio, ok)
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,10 @@ attempts above the first in a generation, and repair generations with at least
|
||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||
attempt events.
|
||||
|
||||
Reasoning counts use the same per-field rules and are a subset of completion
|
||||
tokens. The public cache-hit helper derives a ratio from complete, consistent
|
||||
summed input and cache-read counts; it does not aggregate ratios or use writes.
|
||||
|
||||
An execution group is a zero-ready synchronized collector of selected
|
||||
events. It enrolls by run ID and aggregates each run's first terminal counters and
|
||||
presence-aware usage exactly once. Snapshots copy members and totals under one
|
||||
|
||||
@@ -254,6 +254,8 @@ Validation: model-client parsing tests and public token JSON round trips.
|
||||
|
||||
## Stage 8 — Token Aggregation And Cache-Hit Helper
|
||||
|
||||
**Completed.**
|
||||
|
||||
Touch observation usage accumulators/recorder/group, runner `addTokenUsage`,
|
||||
public event/group conversions, and related fixtures.
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ func (g *ExecutionGroup) Observe(event ExecutionEvent) {
|
||||
return domain.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
g.group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
|
||||
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
|
||||
Usage: domain.ObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), ReasoningTokens: count(event.Usage.ReasoningTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)}})
|
||||
}
|
||||
|
||||
// Snapshot returns an internally consistent, caller-owned summary. Coverage
|
||||
@@ -52,5 +52,5 @@ func (g *ExecutionGroup) Snapshot() ExecutionGroupSummary {
|
||||
members[i] = ExecutionGroupMember{RunID: member.RunID, TerminalObserved: member.TerminalObserved}
|
||||
}
|
||||
return ExecutionGroupSummary{Members: members, TerminalCoverageComplete: value.TerminalCoverageComplete, Attempts: value.Attempts, TransientRetries: value.TransientRetries, RepairCalls: value.RepairCalls,
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
|
||||
Usage: ObservedTokenUsage{PromptTokens: count(value.Usage.PromptTokens), CompletionTokens: count(value.Usage.CompletionTokens), ReasoningTokens: count(value.Usage.ReasoningTokens), TotalTokens: count(value.Usage.TotalTokens), CachedTokens: count(value.Usage.CachedTokens), CacheWriteTokens: count(value.Usage.CacheWriteTokens)}}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ func (g *Group) Observe(event domain.ExecutionEvent) {
|
||||
g.repairs += event.RepairCalls
|
||||
if g.terminals == 0 {
|
||||
complete := domain.ObservedTokenCount{Complete: true}
|
||||
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||
}
|
||||
mergeGroupUsage(&g.usage, event.Usage)
|
||||
g.terminals++
|
||||
@@ -52,7 +52,7 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary {
|
||||
}
|
||||
if len(g.members) == 0 {
|
||||
zero := domain.ObservedTokenCount{Known: true, Complete: true}
|
||||
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
|
||||
result.Usage = domain.ObservedTokenUsage{PromptTokens: zero, CompletionTokens: zero, ReasoningTokens: zero, TotalTokens: zero, CachedTokens: zero, CacheWriteTokens: zero}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -67,6 +67,7 @@ func mergeGroupUsage(total *domain.ObservedTokenUsage, next domain.ObservedToken
|
||||
}
|
||||
merge(&total.PromptTokens, next.PromptTokens)
|
||||
merge(&total.CompletionTokens, next.CompletionTokens)
|
||||
merge(&total.ReasoningTokens, next.ReasoningTokens)
|
||||
merge(&total.TotalTokens, next.TotalTokens)
|
||||
merge(&total.CachedTokens, next.CachedTokens)
|
||||
merge(&total.CacheWriteTokens, next.CacheWriteTokens)
|
||||
|
||||
@@ -260,7 +260,7 @@ type usageField struct {
|
||||
known, complete bool
|
||||
}
|
||||
type usageTotals struct {
|
||||
prompt, completion, total, cached, cacheWrite usageField
|
||||
prompt, completion, reasoning, total, cached, cacheWrite usageField
|
||||
}
|
||||
|
||||
// UsageAccumulator combines presence-aware usage across dispatched attempts.
|
||||
@@ -286,7 +286,7 @@ func (u UsageAccumulator) Observed() domain.ObservedTokenUsage {
|
||||
}
|
||||
|
||||
func completeUsageTotals() usageTotals {
|
||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true},
|
||||
return usageTotals{prompt: usageField{complete: true}, completion: usageField{complete: true}, reasoning: usageField{complete: true},
|
||||
total: usageField{complete: true}, cached: usageField{complete: true}, cacheWrite: usageField{complete: true}}
|
||||
}
|
||||
|
||||
@@ -294,6 +294,7 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
|
||||
known := usagePresence(usage, presence)
|
||||
u.prompt.add(usage.PromptTokens, known.PromptTokens)
|
||||
u.completion.add(usage.CompletionTokens, known.CompletionTokens)
|
||||
u.reasoning.add(usage.ReasoningTokens, known.ReasoningTokens)
|
||||
u.total.add(usage.TotalTokens, known.TotalTokens)
|
||||
u.cached.add(usage.CachedTokens, known.CachedTokens)
|
||||
u.cacheWrite.add(usage.CacheWriteTokens, known.CacheWriteTokens)
|
||||
@@ -302,7 +303,8 @@ func (u *usageTotals) add(usage domain.TokenUsage, presence *domain.TokenUsagePr
|
||||
func (u usageTotals) snapshot(attempts uint64) domain.ObservedTokenUsage {
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: u.prompt.snapshot(attempts), CompletionTokens: u.completion.snapshot(attempts),
|
||||
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
|
||||
ReasoningTokens: u.reasoning.snapshot(attempts),
|
||||
TotalTokens: u.total.snapshot(attempts), CachedTokens: u.cached.snapshot(attempts),
|
||||
CacheWriteTokens: u.cacheWrite.snapshot(attempts),
|
||||
}
|
||||
}
|
||||
@@ -328,6 +330,7 @@ func observedUsage(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
|
||||
return domain.ObservedTokenUsage{
|
||||
PromptTokens: observedCount(usage.PromptTokens, known.PromptTokens),
|
||||
CompletionTokens: observedCount(usage.CompletionTokens, known.CompletionTokens),
|
||||
ReasoningTokens: observedCount(usage.ReasoningTokens, known.ReasoningTokens),
|
||||
TotalTokens: observedCount(usage.TotalTokens, known.TotalTokens),
|
||||
CachedTokens: observedCount(usage.CachedTokens, known.CachedTokens),
|
||||
CacheWriteTokens: observedCount(usage.CacheWriteTokens, known.CacheWriteTokens),
|
||||
@@ -340,7 +343,8 @@ func usagePresence(usage domain.TokenUsage, presence *domain.TokenUsagePresence)
|
||||
}
|
||||
return domain.TokenUsagePresence{
|
||||
PromptTokens: usage.PromptTokens != 0, CompletionTokens: usage.CompletionTokens != 0,
|
||||
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
|
||||
ReasoningTokens: usage.ReasoningTokens != 0,
|
||||
TotalTokens: usage.TotalTokens != 0, CachedTokens: usage.CachedTokens != 0,
|
||||
CacheWriteTokens: usage.CacheWriteTokens != 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,3 +25,18 @@ func TestUsageAccumulatorPreservesPresenceAndNoAttemptSemantics(t *testing.T) {
|
||||
t.Fatalf("unreported completion usage = %+v", observed.CompletionTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReasoningUsagePresence(t *testing.T) {
|
||||
var usage UsageAccumulator
|
||||
usage.Add(domain.TokenUsage{}, &domain.TokenUsagePresence{ReasoningTokens: true})
|
||||
usage.Add(domain.TokenUsage{ReasoningTokens: 4}, nil)
|
||||
got := usage.Observed().ReasoningTokens
|
||||
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: true}) {
|
||||
t.Fatalf("usage=%+v", got)
|
||||
}
|
||||
usage.Add(domain.TokenUsage{}, nil)
|
||||
got = usage.Observed().ReasoningTokens
|
||||
if got != (domain.ObservedTokenCount{Value: 4, Known: true, Complete: false}) {
|
||||
t.Fatalf("partial usage=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,7 +317,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
responses: []*domain.GenerateResponse{{
|
||||
Content: `{"repaired":true}`,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5,
|
||||
PromptTokens: 2, CompletionTokens: 3, ReasoningTokens: 2, TotalTokens: 5,
|
||||
CachedTokens: 7, CacheWriteTokens: 11,
|
||||
},
|
||||
}},
|
||||
@@ -328,7 +328,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
client := &sequenceLLM{responses: []*domain.GenerateResponse{{
|
||||
Content: `{"broken":true}`,
|
||||
Usage: domain.TokenUsage{
|
||||
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
|
||||
PromptTokens: 13, CompletionTokens: 17, ReasoningTokens: 4, TotalTokens: 19,
|
||||
CachedTokens: 23, CacheWriteTokens: 29,
|
||||
},
|
||||
}}}
|
||||
@@ -370,7 +370,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
||||
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
||||
}
|
||||
wantUsage := domain.TokenUsage{
|
||||
PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24,
|
||||
PromptTokens: 15, CompletionTokens: 20, ReasoningTokens: 6, TotalTokens: 24,
|
||||
CachedTokens: 30, CacheWriteTokens: 40,
|
||||
}
|
||||
if result.Usage != wantUsage {
|
||||
|
||||
@@ -309,6 +309,7 @@ func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
||||
return domain.TokenUsage{
|
||||
PromptTokens: total.PromptTokens + next.PromptTokens,
|
||||
CompletionTokens: total.CompletionTokens + next.CompletionTokens,
|
||||
ReasoningTokens: total.ReasoningTokens + next.ReasoningTokens,
|
||||
TotalTokens: total.TotalTokens + next.TotalTokens,
|
||||
CachedTokens: total.CachedTokens + next.CachedTokens,
|
||||
CacheWriteTokens: total.CacheWriteTokens + next.CacheWriteTokens,
|
||||
|
||||
@@ -16,6 +16,19 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi
|
||||
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
|
||||
}
|
||||
|
||||
func TestInjectedReasoningPresenceReachesObservation(t *testing.T) {
|
||||
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok", UsagePresence: &promptkit.TokenUsagePresence{ReasoningTokens: true}}})
|
||||
var events []promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(event promptkit.ExecutionEvent) { events = append(events, event) }})
|
||||
if err != nil || result == nil || len(events) != 3 {
|
||||
t.Fatalf("result=%+v events=%+v error=%v", result, events, err)
|
||||
}
|
||||
want := promptkit.ObservedTokenCount{Known: true, Complete: true}
|
||||
if events[1].Usage.ReasoningTokens != want || events[2].Usage.ReasoningTokens != want {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinuationObservationAncestry(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}}
|
||||
engine := observationEngine(t, client)
|
||||
|
||||
@@ -101,6 +101,16 @@ type ObservedTokenUsage struct {
|
||||
CacheWriteTokens ObservedTokenCount
|
||||
}
|
||||
|
||||
// CacheHitRatio returns the fraction of input tokens served from cache when
|
||||
// both counts are known, complete, and consistent. Cache writes are independent.
|
||||
func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) {
|
||||
input, reads := u.PromptTokens, u.CachedTokens
|
||||
if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value {
|
||||
return 0, false
|
||||
}
|
||||
return float64(reads.Value) / float64(input.Value), true
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only, not provider text, request content,
|
||||
// credentials, URLs, headers, or error objects. It has no stable JSON contract.
|
||||
// Identifiers themselves may be sensitive. Nonterminal aggregate counters are
|
||||
|
||||
Reference in New Issue
Block a user