Collect costs for dispatched model attempts
This commit is contained in:
@@ -130,7 +130,7 @@ func (c *Coordinator) generate(
|
||||
if result.Dispatched {
|
||||
attempts++
|
||||
}
|
||||
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, generation, ordinal)
|
||||
recordAttempt(result, false, captureEvidence, &evidence, &usage, recorder, observation.CostFromContext(ctx), req.Target.ZeroCost, generation, ordinal)
|
||||
if result.Outcome.Err == nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if captureEvidence {
|
||||
@@ -185,7 +185,7 @@ func (c *Coordinator) generateInjected(ctx context.Context, req domain.GenerateR
|
||||
generation = 1
|
||||
}
|
||||
result := c.executeInjected(ctx, req)
|
||||
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), generation, 1)
|
||||
recordAttempt(result, true, captureEvidence, &evidence, &usage, observation.FromContext(ctx), observation.CostFromContext(ctx), req.Target.ZeroCost, generation, 1)
|
||||
if captureEvidence {
|
||||
evidence.Usage = usage.Observed()
|
||||
}
|
||||
@@ -219,6 +219,9 @@ func (c *Coordinator) executeInjected(ctx context.Context, req domain.GenerateRe
|
||||
requested := req.StructuredOutput != nil
|
||||
result.Outcome.StructuredOutputRequested = &requested
|
||||
result.Outcome.Response, result.Outcome.Err = c.injected.Generate(ctx, req)
|
||||
if result.Outcome.Response != nil {
|
||||
result.Outcome.CostUSD = result.Outcome.Response.CostUSD
|
||||
}
|
||||
result.RequestDuration = elapsed(start, c.now())
|
||||
if result.Outcome.Err == nil && result.Outcome.Response == nil {
|
||||
result.Outcome.Err = llm.ErrMalformedResponse
|
||||
@@ -240,8 +243,15 @@ type attemptResult struct {
|
||||
|
||||
// recordAttempt runs after the attempt has released its permit, including when
|
||||
// the recorder calls application code. Injected clients expose no HTTP facts.
|
||||
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, generation, ordinal uint64) {
|
||||
func recordAttempt(result attemptResult, injected, capture bool, evidence *GenerationEvidence, usage *observation.UsageAccumulator, recorder *observation.Recorder, cost *observation.CostAccumulator, zeroCost bool, generation, ordinal uint64) {
|
||||
outcome := result.Outcome
|
||||
var resolved domain.ObservedCost
|
||||
if result.Dispatched {
|
||||
resolved = observation.ResolveCost(outcome.CostUSD, zeroCost)
|
||||
if cost != nil {
|
||||
cost.Add(resolved)
|
||||
}
|
||||
}
|
||||
failure := ClassifyFailure(outcome.Err)
|
||||
if capture {
|
||||
attempt := AttemptEvidence{
|
||||
@@ -277,7 +287,7 @@ func recordAttempt(result attemptResult, injected, capture bool, evidence *Gener
|
||||
recorder.Attempt(observation.AttemptData{Generation: generation, Attempt: ordinal,
|
||||
Dispatched: result.Dispatched, CapacityWait: result.CapacityWait, GenerationDuration: result.RequestDuration,
|
||||
Outcome: attemptOutcome(outcome.Err), Failure: failure, StatusCode: outcome.StatusCode,
|
||||
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence})
|
||||
Usage: outcome.Usage, UsagePresence: outcome.UsagePresence, CostUSD: resolved})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user