From 81a2a7591d9cbd19450cb23eac36c26ce5f98436 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 18:27:01 +0000 Subject: [PATCH] Collect costs for dispatched model attempts --- docs/internal/observation.md | 7 +++- docs/roadmap/implementation.md | 2 + internal/generation/coordinator.go | 18 +++++++-- internal/generation/observation_test.go | 54 +++++++++++++++++++++++++ internal/observation/cost.go | 14 +++++++ internal/observation/group.go | 5 +++ internal/observation/group_test.go | 19 +++++++++ internal/observation/recorder.go | 11 ++++- internal/observation/recorder_test.go | 5 +++ 9 files changed, 129 insertions(+), 6 deletions(-) diff --git a/docs/internal/observation.md b/docs/internal/observation.md index 732be3c..aecb444 100644 --- a/docs/internal/observation.md +++ b/docs/internal/observation.md @@ -18,7 +18,7 @@ Timeout attribution is finalized and the child scope released before terminal delivery, so observer latency cannot change the already selected outcome. Prepared-content notification is a separate opt-in callback, not an event. -The facade masks inherited recorder and prepared-claim state when entering +The facade masks inherited recorder, cost, and prepared-claim state when entering preparation or execution. Delegated calls from an injected collaborator retain the caller's cancellation, deadline, and unrelated context values, but do not reuse the enclosing invocation's accounting. Ordinary runs compose the request @@ -56,6 +56,11 @@ attempts above the first in a generation, and repair generations with at least one dispatch. A terminal event is an aggregate and must not be added to its attempt events. +The generation coordinator resolves cost once per dispatched attempt and adds +it to invocation-local accounting independently of event delivery. A recorder +can read that same accumulator at termination; groups merge first terminal +costs independently of terminal coverage. + Reasoning counts use the same per-field rules and are a subset of completion tokens. The public cache-hit helper derives a ratio from complete, consistent summed input and cache-read counts; it does not aggregate ratios or use writes. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 0950ac4..922510a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -325,6 +325,8 @@ Validation: provider decoder and injected-adapter tests. ## Stage 11 — Per-Attempt Cost Collection And Observation +**Completed.** + Touch generation `recordAttempt` and built-in/injected execution paths, observation attempt data/events, recorder terminal cost, and group aggregation. diff --git a/internal/generation/coordinator.go b/internal/generation/coordinator.go index 03726cb..1e1f415 100644 --- a/internal/generation/coordinator.go +++ b/internal/generation/coordinator.go @@ -130,7 +130,7 @@ func (c *Coordinator) generate( if result.Dispatched { attempts++ } - recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal) + recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal) if result.Outcome.Err == nil { if err := ctx.Err(); err != nil { if captureEvidence { @@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR generation = 1 } result := c.executeInjected(ctx, req) - recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1) + recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1) if captureEvidence { evidence.Usage = usage.Observed() } @@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe requested := req.StructuredOutput != nil result.Outcome.StructuredOutputRequested = &requested result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req) + if result.Outcome.Response != nil { + result.Outcome.CostUSD = result.Outcome.Response.CostUSD + } result.RequestDuration = elapsed(start, c.now()) if result.Outcome.Err == nil && result.Outcome.Response == nil { result.Outcome.Err = llm.ErrMalformedResponse @@ -240,8 +243,15 @@ type attemptResult struct { // recordAttempt runs after the attempt has released its permit, including when // the recorder calls application code. Injected clients expose no HTTP facts. -func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) { +func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) { outcome := result.Outcome + var resolved domain.ObservedCost + if result.Dispatched { + resolved = observation.ResolveCost(outcome.CostUSD, zeroCost) + if cost != nil { + cost.Add(resolved) + } + } failure := ClassifyFailure(outcome.Err) if capture { attempt := AttemptEvidence{ @@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal, Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration, Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode, - Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}) + Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved}) } } diff --git a/internal/generation/observation_test.go b/internal/generation/observation_test.go index ce4752a..a9e5b56 100644 --- a/internal/generation/observation_test.go +++ b/internal/generation/observation_test.go @@ -58,6 +58,60 @@ func TestObservedRetryAndExhaustion(t *testing.T) { } } +func TestDispatchedAttemptCostsAreCollectedWithoutObserver(t *testing.T) { + calls := 0 + coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return response(503, `{"error":{}}`), nil + } + return response(200, `{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`), nil + }) + var cost observation.CostAccumulator + ctx := observation.WithCostAccumulator(context.Background(), &cost) + req := testRequest(1) + req.Target.Provider = "openrouter" + if _, err := coordinator.Generate(ctx, req); err != nil { + t.Fatal(err) + } + got := cost.Observed() + if got.Value != .02 || !got.Known || got.Complete || got.Source != domain.CostSourceProvider { + t.Fatalf("cost=%+v", got) + } + + configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { return nil, errors.New("transport failed") }) + var zero observation.CostAccumulator + req.Target.ZeroCost = true + if _, err := configured.Generate(observation.WithCostAccumulator(context.Background(), &zero), req); err == nil { + t.Fatal("expected error") + } + if got := zero.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) { + t.Fatalf("fallback=%+v", got) + } +} + +func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) { + configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil }) + charge := .25 + coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{CostUSD: &charge}, err: errors.New("failed")}) + var cost observation.CostAccumulator + var events []domain.ExecutionEvent + recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) }) + if err != nil { + t.Fatal(err) + } + recorder.SetCostAccumulator(&cost) + ctx := observation.WithRecorder(observation.WithCostAccumulator(context.Background(), &cost), recorder) + if _, err := coordinator.Generate(ctx, testRequest(0)); err == nil { + t.Fatal("expected error") + } + recorder.Finish(observation.FinishData{}) + want := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider} + if len(events) != 2 || events[0].CostUSD != want || events[1].CostUSD != want || cost.Observed() != want { + t.Fatalf("events=%+v cost=%+v", events, cost.Observed()) + } +} + func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) { coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil }) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/observation/cost.go b/internal/observation/cost.go index 60ee947..d233bd0 100644 --- a/internal/observation/cost.go +++ b/internal/observation/cost.go @@ -1,11 +1,25 @@ package observation import ( + "context" "math" "go.maximumdirect.net/promptkit/internal/domain" ) +type costContextKey struct{} + +// WithCostAccumulator carries one invocation's cost state. +func WithCostAccumulator(ctx context.Context, accumulator *CostAccumulator) context.Context { + return context.WithValue(ctx, costContextKey{}, accumulator) +} + +// CostFromContext returns the current invocation's cost state, if installed. +func CostFromContext(ctx context.Context) *CostAccumulator { + value, _ := ctx.Value(costContextKey{}).(*CostAccumulator) + return value +} + // ResolveCost applies reported-cost precedence over a configured-zero fallback. func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost { if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) { diff --git a/internal/observation/group.go b/internal/observation/group.go index 4f3150a..b01fb3f 100644 --- a/internal/observation/group.go +++ b/internal/observation/group.go @@ -14,6 +14,7 @@ type Group struct { terminals int attempts, retries, repairs uint64 usage domain.ObservedTokenUsage + cost CostAccumulator } // Observe enrolls a run and incorporates its first terminal event only. @@ -39,6 +40,7 @@ func (g *Group) Observe(event domain.ExecutionEvent) { g.usage = domain.ObservedTokenUsage{PromptTokens: complete, CompletionTokens: complete, ReasoningTokens: complete, TotalTokens: complete, CachedTokens: complete, CacheWriteTokens: complete} } mergeGroupUsage(&g.usage, event.Usage) + g.cost.Add(event.CostUSD) g.terminals++ } @@ -47,6 +49,9 @@ func (g *Group) Snapshot() domain.ExecutionGroupSummary { g.mu.Lock() defer g.mu.Unlock() result := domain.ExecutionGroupSummary{Members: make([]domain.ExecutionGroupMember, 0, len(g.members)), TerminalCoverageComplete: len(g.members) == g.terminals, Attempts: g.attempts, TransientRetries: g.retries, RepairCalls: g.repairs, Usage: g.usage} + if g.terminals > 0 || len(g.members) == 0 { + result.CostUSD = g.cost.Observed() + } for id, terminal := range g.members { result.Members = append(result.Members, domain.ExecutionGroupMember{RunID: id, TerminalObserved: terminal}) } diff --git a/internal/observation/group_test.go b/internal/observation/group_test.go index e5e58d7..2f9ae6e 100644 --- a/internal/observation/group_test.go +++ b/internal/observation/group_test.go @@ -41,6 +41,25 @@ func TestGroupTerminalAccounting(t *testing.T) { } } +func TestGroupCostCoverageIsIndependentOfTerminalCoverage(t *testing.T) { + var group Group + if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{Known: true, Complete: true}) { + t.Fatalf("empty=%+v", got) + } + group.Observe(domain.ExecutionEvent{Kind: domain.ExecutionStarted, RunID: "pending"}) + if got := group.Snapshot().CostUSD; got != (domain.ObservedCost{}) { + t.Fatalf("pending=%+v", got) + } + charge := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider} + terminal := domain.ExecutionEvent{Kind: domain.ExecutionFinished, RunID: "done", CostUSD: charge} + group.Observe(terminal) + group.Observe(terminal) + got := group.Snapshot() + if got.TerminalCoverageComplete || got.CostUSD != charge { + t.Fatalf("snapshot=%+v", got) + } +} + func TestGroupConcurrentSnapshots(t *testing.T) { var group Group var wg sync.WaitGroup diff --git a/internal/observation/recorder.go b/internal/observation/recorder.go index 58b09f8..d4e10ad 100644 --- a/internal/observation/recorder.go +++ b/internal/observation/recorder.go @@ -30,6 +30,7 @@ type Recorder struct { generationDuration time.Duration retryDelay time.Duration usage UsageAccumulator + cost *CostAccumulator } // Selection contains resolved identities only. Unresolved fields stay empty. @@ -53,6 +54,7 @@ type AttemptData struct { StatusCode int Usage domain.TokenUsage UsagePresence *domain.TokenUsagePresence + CostUSD domain.ObservedCost } // DelayData describes one completed or interrupted retry wait. @@ -99,6 +101,9 @@ func (r *Recorder) RunID() string { return r.runID } // SetParentRunID records the immediate source identity before start delivery. func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id } +// SetCostAccumulator selects the invocation-owned cost state for terminal reporting. +func (r *Recorder) SetCostAccumulator(cost *CostAccumulator) { r.cost = cost } + // Start delivers the opening event with only identities already resolved. func (r *Recorder) Start() { event := r.event(domain.ExecutionStarted) @@ -138,6 +143,7 @@ func (r *Recorder) Attempt(data AttemptData) { event.StatusCode = data.StatusCode if data.Dispatched { event.Usage = observedUsage(data.Usage, data.UsagePresence) + event.CostUSD = data.CostUSD } r.deliver(event) } @@ -176,6 +182,9 @@ func (r *Recorder) Finish(data FinishData) { event.Duration = 0 } event.Usage = r.usage.Observed() + if r.cost != nil { + event.CostUSD = r.cost.Observed() + } r.deliver(event) } @@ -222,7 +231,7 @@ type executionBoundary struct{ context.Context } func (c executionBoundary) Value(key any) any { switch key.(type) { - case contextKey, claimSinkKey: + case contextKey, claimSinkKey, costContextKey: return nil default: return c.Context.Value(key) diff --git a/internal/observation/recorder_test.go b/internal/observation/recorder_test.go index 96f23d6..e74f506 100644 --- a/internal/observation/recorder_test.go +++ b/internal/observation/recorder_test.go @@ -19,10 +19,15 @@ func TestExecutionBoundaryPreservesCallerContext(t *testing.T) { recorder := newRecorder("outer", time.Now, nil) sink := recorder parent = WithClaimSink(WithRecorder(parent, recorder), &sink) + var cost CostAccumulator + parent = WithCostAccumulator(parent, &cost) child := WithoutExecutionState(parent) if FromContext(child) != nil || FromContext(parent) != recorder { t.Fatal("execution boundary did not isolate recorder state") } + if CostFromContext(child) != nil || CostFromContext(parent) != &cost { + t.Fatal("execution boundary did not isolate cost state") + } RegisterClaim(child, nil) if sink != recorder { t.Fatal("execution boundary retained the parent claim sink")