Collect costs for dispatched model attempts

This commit is contained in:
2026-11-22 18:27:01 +00:00
parent cd0e1f5d39
commit 81a2a7591d
9 changed files with 129 additions and 6 deletions

View File

@@ -130,7 +130,7 @@ func (c *Coordinator) generate(
if result.Dispatched {
attempts++
}
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal)
if result.Outcome.Err == nil {
if err := ctx.Err(); err != nil {
if captureEvidence {
@@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
generation = 1
}
result := c.executeInjected(ctx, req)
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1)
if captureEvidence {
evidence.Usage = usage.Observed()
}
@@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe
requested := req.StructuredOutput != nil
result.Outcome.StructuredOutputRequested = &requested
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
if result.Outcome.Response != nil {
result.Outcome.CostUSD = result.Outcome.Response.CostUSD
}
result.RequestDuration = elapsed(start, c.now())
if result.Outcome.Err == nil && result.Outcome.Response == nil {
result.Outcome.Err = llm.ErrMalformedResponse
@@ -240,8 +243,15 @@ type attemptResult struct {
// recordAttempt runs after the attempt has released its permit, including when
// the recorder calls application code. Injected clients expose no HTTP facts.
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) {
outcome := result.Outcome
var resolved domain.ObservedCost
if result.Dispatched {
resolved = observation.ResolveCost(outcome.CostUSD, zeroCost)
if cost != nil {
cost.Add(resolved)
}
}
failure := ClassifyFailure(outcome.Err)
if capture {
attempt := AttemptEvidence{
@@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved})
}
}

View File

@@ -58,6 +58,60 @@ func TestObservedRetryAndExhaustion(t *testing.T) {
}
}
func TestDispatchedAttemptCostsAreCollectedWithoutObserver(t *testing.T) {
calls := 0
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return response(503, `{"error":{}}`), nil
}
return response(200, `{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`), nil
})
var cost observation.CostAccumulator
ctx := observation.WithCostAccumulator(context.Background(), &cost)
req := testRequest(1)
req.Target.Provider = "openrouter"
if _, err := coordinator.Generate(ctx, req); err != nil {
t.Fatal(err)
}
got := cost.Observed()
if got.Value != .02 || !got.Known || got.Complete || got.Source != domain.CostSourceProvider {
t.Fatalf("cost=%+v", got)
}
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { return nil, errors.New("transport failed") })
var zero observation.CostAccumulator
req.Target.ZeroCost = true
if _, err := configured.Generate(observation.WithCostAccumulator(context.Background(), &zero), req); err == nil {
t.Fatal("expected error")
}
if got := zero.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
t.Fatalf("fallback=%+v", got)
}
}
func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) {
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
charge := .25
coordinator := NewInjected(configured.manager, fixedClient{response: &domain.GenerateResponse{CostUSD: &charge}, err: errors.New("failed")})
var cost observation.CostAccumulator
var events []domain.ExecutionEvent
recorder, err := observation.New(func(event domain.ExecutionEvent) { events = append(events, event) })
if err != nil {
t.Fatal(err)
}
recorder.SetCostAccumulator(&cost)
ctx := observation.WithRecorder(observation.WithCostAccumulator(context.Background(), &cost), recorder)
if _, err := coordinator.Generate(ctx, testRequest(0)); err == nil {
t.Fatal("expected error")
}
recorder.Finish(observation.FinishData{})
want := domain.ObservedCost{Value: .25, Known: true, Complete: true, Source: domain.CostSourceProvider}
if len(events) != 2 || events[0].CostUSD != want || events[1].CostUSD != want || cost.Observed() != want {
t.Fatalf("events=%+v cost=%+v", events, cost.Observed())
}
}
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
ctx, cancel := context.WithCancel(context.Background())