Collect costs for dispatched model attempts
This commit is contained in:
@@ -130,7 +130,7 @@ func (c *Coordinator) generate(
|
||||
if result.Dispatched {
|
||||
attempts++
|
||||
}
|
||||
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
|
||||
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal)
|
||||
if result.Outcome.Err == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if captureEvidence {
|
||||
@@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
||||
generation = 1
|
||||
}
|
||||
result := c.executeInjected(ctx, req)
|
||||
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
|
||||
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1)
|
||||
if captureEvidence {
|
||||
evidence.Usage = usage.Observed()
|
||||
}
|
||||
@@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe
|
||||
requested := req.StructuredOutput != nil
|
||||
result.Outcome.StructuredOutputRequested = &requested
|
||||
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
|
||||
if result.Outcome.Response != nil {
|
||||
result.Outcome.CostUSD = result.Outcome.Response.CostUSD
|
||||
}
|
||||
result.RequestDuration = elapsed(start, c.now())
|
||||
if result.Outcome.Err == nil && result.Outcome.Response == nil {
|
||||
result.Outcome.Err = llm.ErrMalformedResponse
|
||||
@@ -240,8 +243,15 @@ type attemptResult struct {
|
||||
|
||||
// recordAttempt runs after the attempt has released its permit, including when
|
||||
// the recorder calls application code. Injected clients expose no HTTP facts.
|
||||
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
|
||||
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) {
|
||||
outcome := result.Outcome
|
||||
var resolved domain.ObservedCost
|
||||
if result.Dispatched {
|
||||
resolved = observation.ResolveCost(outcome.CostUSD, zeroCost)
|
||||
if cost != nil {
|
||||
cost.Add(resolved)
|
||||
}
|
||||
}
|
||||
failure := ClassifyFailure(outcome.Err)
|
||||
if capture {
|
||||
attempt := AttemptEvidence{
|
||||
@@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
|
||||
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
|
||||
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,60 @@ func TestObservedRetryAndExhaustion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatchedAttemptCostsAreCollectedWithoutObserver(t *testing.T) {
|
||||
calls := 0
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return response(503, `{"error":{}}`), nil
|
||||
}
|
||||
return response(200, `{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`), nil
|
||||
})
|
||||
var cost observation.CostAccumulator
|
||||
ctx := observation.WithCostAccumulator(context.Background(), &cost)
|
||||
req := testRequest(1)
|
||||
req.Target.Provider = "openrouter"
|
||||
if _, err := coordinator.Generate(ctx, req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := cost.Observed()
|
||||
if got.Value != .02 || !got.Known || got.Complete || got.Source != domain.CostSourceProvider {
|
||||
t.Fatalf("cost=%+v", got)
|
||||
}
|
||||
|
||||
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { return nil, errors.New("transport failed") })
|
||||
var zero observation.CostAccumulator
|
||||
req.Target.ZeroCost = true
|
||||
if _, err := configured.Generate(observation.WithCostAccumulator(context.Background(), &zero), req); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if got := zero.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
|
||||
t.Fatalf("fallback=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) {
|
||||
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
|
||||
charge := .25
|
||||
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{CostUSD: &charge}, err: errors.New("failed")})
|
||||
var cost observation.CostAccumulator
|
||||
var events []domain.ExecutionEvent
|
||||
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder.SetCostAccumulator(&cost)
|
||||
ctx := observation.WithRecorder(observation.WithCostAccumulator(context.Background(), &cost), recorder)
|
||||
if _, err := coordinator.Generate(ctx, testRequest(0)); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
recorder.Finish(observation.FinishData{})
|
||||
want := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
|
||||
if len(events) != 2 || events[0].CostUSD != want || events[1].CostUSD != want || cost.Observed() != want {
|
||||
t.Fatalf("events=%+v cost=%+v", events, cost.Observed())
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
|
||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
@@ -1,11 +1,25 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
type costContextKey struct{}
|
||||
|
||||
// WithCostAccumulator carries one invocation's cost state.
|
||||
func WithCostAccumulator(ctx context.Context, accumulator *CostAccumulator) context.Context {
|
||||
return context.WithValue(ctx, costContextKey{}, accumulator)
|
||||
}
|
||||
|
||||
// CostFromContext returns the current invocation's cost state, if installed.
|
||||
func CostFromContext(ctx context.Context) *CostAccumulator {
|
||||
value, _ := ctx.Value(costContextKey{}).(*CostAccumulator)
|
||||
return value
|
||||
}
|
||||
|
||||
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
|
||||
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
|
||||
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {
|
||||
|
||||
@@ -14,6 +14,7 @@ type Group struct {
|
||||
terminals int
|
||||
attempts, retries, repairs uint64
|
||||
usage domain.ObservedTokenUsage
|
||||
cost CostAccumulator
|
||||
}
|
||||
|
||||
// Observe enrolls a run and incorporates its first terminal event only.
|
||||
@@ -39,6 +40,7 @@ func (g *Group) Observe(event domain.ExecutionEvent) {
|
||||
g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete}
|
||||
}
|
||||
mergeGroupUsage(&g.usage, event.Usage)
|
||||
g.cost.Add(event.CostUSD)
|
||||
g.terminals++
|
||||
}
|
||||
|
||||
@@ -47,6 +49,9 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage}
|
||||
if g.terminals > 0 || len(g.members) == 0 {
|
||||
result.CostUSD = g.cost.Observed()
|
||||
}
|
||||
for id, terminal := range g.members {
|
||||
result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal})
|
||||
}
|
||||
|
||||
@@ -41,6 +41,25 @@ func TestGroupTerminalAccounting(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupCostCoverageIsIndependentOfTerminalCoverage(t *testing.T) {
|
||||
var group Group
|
||||
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("empty=%+v", got)
|
||||
}
|
||||
group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionStarted, RunID: "pending"})
|
||||
if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{}) {
|
||||
t.Fatalf("pending=%+v", got)
|
||||
}
|
||||
charge := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
|
||||
terminal := domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "done", CostUSD: charge}
|
||||
group.Observe(terminal)
|
||||
group.Observe(terminal)
|
||||
got := group.Snapshot()
|
||||
if got.TerminalCoverageComplete || got.CostUSD != charge {
|
||||
t.Fatalf("snapshot=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupConcurrentSnapshots(t *testing.T) {
|
||||
var group Group
|
||||
var wg sync.WaitGroup
|
||||
|
||||
@@ -30,6 +30,7 @@ type Recorder struct {
|
||||
generationDuration time.Duration
|
||||
retryDelay time.Duration
|
||||
usage UsageAccumulator
|
||||
cost *CostAccumulator
|
||||
}
|
||||
|
||||
// Selection contains resolved identities only. Unresolved fields stay empty.
|
||||
@@ -53,6 +54,7 @@ type AttemptData struct {
|
||||
StatusCode int
|
||||
Usage domain.TokenUsage
|
||||
UsagePresence *domain.TokenUsagePresence
|
||||
CostUSD domain.ObservedCost
|
||||
}
|
||||
|
||||
// DelayData describes one completed or interrupted retry wait.
|
||||
@@ -99,6 +101,9 @@ func (r *Recorder) RunID() string { return r.runID }
|
||||
// SetParentRunID records the immediate source identity before start delivery.
|
||||
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
|
||||
|
||||
// SetCostAccumulator selects the invocation-owned cost state for terminal reporting.
|
||||
func (r *Recorder) SetCostAccumulator(cost *CostAccumulator) { r.cost = cost }
|
||||
|
||||
// Start delivers the opening event with only identities already resolved.
|
||||
func (r *Recorder) Start() {
|
||||
event := r.event(domain.ExecutionStarted)
|
||||
@@ -138,6 +143,7 @@ func (r *Recorder) Attempt(data AttemptData) {
|
||||
event.StatusCode = data.StatusCode
|
||||
if data.Dispatched {
|
||||
event.Usage = observedUsage(data.Usage, data.UsagePresence)
|
||||
event.CostUSD = data.CostUSD
|
||||
}
|
||||
r.deliver(event)
|
||||
}
|
||||
@@ -176,6 +182,9 @@ func (r *Recorder) Finish(data FinishData) {
|
||||
event.Duration = 0
|
||||
}
|
||||
event.Usage = r.usage.Observed()
|
||||
if r.cost != nil {
|
||||
event.CostUSD = r.cost.Observed()
|
||||
}
|
||||
r.deliver(event)
|
||||
}
|
||||
|
||||
@@ -222,7 +231,7 @@ type executionBoundary struct{ context.Context }
|
||||
|
||||
func (c executionBoundary) Value(key any) any {
|
||||
switch key.(type) {
|
||||
case contextKey, claimSinkKey:
|
||||
case contextKey, claimSinkKey, costContextKey:
|
||||
return nil
|
||||
default:
|
||||
return c.Context.Value(key)
|
||||
|
||||
@@ -19,10 +19,15 @@ func TestExecutionBoundaryPreservesCallerContext(t *testing.T) {
|
||||
recorder := newRecorder("outer", time.Now, nil)
|
||||
sink := recorder
|
||||
parent = WithClaimSink(WithRecorder(parent, recorder), &sink)
|
||||
var cost CostAccumulator
|
||||
parent = WithCostAccumulator(parent, &cost)
|
||||
child := WithoutExecutionState(parent)
|
||||
if FromContext(child) != nil || FromContext(parent) != recorder {
|
||||
t.Fatal("execution boundary did not isolate recorder state")
|
||||
}
|
||||
if CostFromContext(child) != nil || CostFromContext(parent) != &cost {
|
||||
t.Fatal("execution boundary did not isolate cost state")
|
||||
}
|
||||
RegisterClaim(child, nil)
|
||||
if sink != recorder {
|
||||
t.Fatal("execution boundary retained the parent claim sink")
|
||||
|
||||
Reference in New Issue
Block a user