Report costs for ordinary runs

This commit is contained in:
2026-11-22 18:28:39 +00:00
parent 81a2a7591d
commit d77e3e2f83
5 changed files with 142 additions and 5 deletions

124
cost_run_contract_test.go Normal file
View File

@@ -0,0 +1,124 @@
package promptkit_test
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type costRoundTrip func(*http.Request) (*http.Response, error)
func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
options = append(options, promptkit.WithBundledOpenRouter(false))
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...)
if err != nil {
t.Fatal(err)
}
return engine
}
func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
for _, observed := range []bool{false, true} {
t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) {
calls := 0
transport := costRoundTrip(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil
})
engine := costEngine(t, false, &http.Client{Transport: transport})
var events []promptkit.ExecutionEvent
req := promptkit.RunRequest{PromptID: "prompt"}
if observed {
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
}
result, err := engine.Run(context.Background(), req)
want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider}
if err != nil || result == nil || result.CostUSD != want || calls != 2 {
t.Fatalf("result=%+v calls=%d err=%v", result, calls, err)
}
if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) {
t.Fatalf("events=%+v", events)
}
})
}
}
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
charge := .125
for _, tc := range []struct {
name string
response *promptkit.GenerateResponse
err error
want promptkit.ObservedCost
}{
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
} {
t.Run(tc.name, func(t *testing.T) {
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return tc.response, tc.err
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
})
}
}
func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) {
charge := .1
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "invalid"}, nil
}
return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil
})))
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed}
if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want {
t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err)
}
}
func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) {
engine := costEngine(t, true, nil)
var terminal promptkit.ExecutionEvent
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) {
if e.Kind == promptkit.ExecutionFinished {
terminal = e
}
}})
if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
}
}

View File

@@ -56,10 +56,11 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its one dispatch. A terminal event is an aggregate and must not be added to its
attempt events. attempt events.
Ordinary runs install invocation-local cost accounting even without an observer.
The generation coordinator resolves cost once per dispatched attempt and adds The generation coordinator resolves cost once per dispatched attempt and adds
it to invocation-local accounting independently of event delivery. A recorder it to that state independently of event delivery. The result and recorder read
can read that same accumulator at termination; groups merge first terminal the same accumulator; groups merge first terminal costs independently of
costs independently of terminal coverage. terminal coverage.
Reasoning counts use the same per-field rules and are a subset of completion Reasoning counts use the same per-field rules and are a subset of completion
tokens. The public cache-hit helper derives a ratio from complete, consistent tokens. The public cache-hit helper derives a ratio from complete, consistent

View File

@@ -353,6 +353,8 @@ Validation: generation, observation recorder/group, and diagnostic-boundary test
## Stage 12 — Ordinary Run Cost Lifetime And Result Accounting ## Stage 12 — Ordinary Run Cost Lifetime And Result Accounting
**Completed.**
- Create/install a new private accumulator at the public ordinary Run boundary, - Create/install a new private accumulator at the public ordinary Run boundary,
before optional recorder creation. Pass that same instance to the recorder. before optional recorder creation. Pass that same instance to the recorder.
Ensure direct internal runner execution creates a local accumulator if one Ensure direct internal runner execution creates a local accumulator if one

View File

@@ -602,6 +602,8 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
// result on error. // result on error.
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
ctx = observation.WithoutExecutionState(ctx) ctx = observation.WithoutExecutionState(ctx)
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := usecase.NewOperationScope(ctx, req.OperationTimeout) scope := usecase.NewOperationScope(ctx, req.OperationTimeout)
defer scope.Close() defer scope.Close()
ctx = scope.Context ctx = scope.Context
@@ -617,6 +619,7 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
return result, err return result, err
} }
recorder, idErr := observation.New(observer) recorder, idErr := observation.New(observer)
recorder.SetCostAccumulator(cost)
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil { if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
recorder.SetParentRunID(req.ContinueFrom.internal.RunID()) recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
} }

View File

@@ -137,6 +137,9 @@ func NewRunnerWithRepairer(
} }
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) { func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
if observation.CostFromContext(ctx) == nil {
ctx = observation.WithCostAccumulator(ctx, &observation.CostAccumulator{})
}
recorder := observation.FromContext(ctx) recorder := observation.FromContext(ctx)
runID := "" runID := ""
if recorder != nil { if recorder != nil {
@@ -274,7 +277,7 @@ func (r *Runner) executePreparedRun(
exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content) exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content)
} }
return &domain.RunResult{ result := &domain.RunResult{
Exchange: exchange, Exchange: exchange,
ParentRunID: prepared.ParentRunID, ParentRunID: prepared.ParentRunID,
RunID: runID, RunID: runID,
@@ -295,7 +298,11 @@ func (r *Runner) executePreparedRun(
StartTime: start, StartTime: start,
EndTime: end, EndTime: end,
Duration: end.Sub(start), Duration: end.Sub(start),
}, nil }
if cost := observation.CostFromContext(ctx); cost != nil {
result.CostUSD = cost.Observed()
}
return result, nil
} }
func wrapGenerationError(err error) error { func wrapGenerationError(err error) error {