Report costs for ordinary runs
This commit is contained in:
124
cost_run_contract_test.go
Normal file
124
cost_run_contract_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type costRoundTrip func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||
|
||||
func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}},
|
||||
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}
|
||||
options = append(options, promptkit.WithBundledOpenRouter(false))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
|
||||
for _, observed := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) {
|
||||
calls := 0
|
||||
transport := costRoundTrip(func(*http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil
|
||||
})
|
||||
engine := costEngine(t, false, &http.Client{Transport: transport})
|
||||
var events []promptkit.ExecutionEvent
|
||||
req := promptkit.RunRequest{PromptID: "prompt"}
|
||||
if observed {
|
||||
req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) }
|
||||
}
|
||||
result, err := engine.Run(context.Background(), req)
|
||||
want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider}
|
||||
if err != nil || result == nil || result.CostUSD != want || calls != 2 {
|
||||
t.Fatalf("result=%+v calls=%d err=%v", result, calls, err)
|
||||
}
|
||||
if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
|
||||
charge := .125
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
response *promptkit.GenerateResponse
|
||||
err error
|
||||
want promptkit.ObservedCost
|
||||
}{
|
||||
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
|
||||
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return tc.response, tc.err
|
||||
})))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) {
|
||||
if e.Kind == promptkit.ExecutionFinished {
|
||||
terminal = e
|
||||
}
|
||||
}})
|
||||
if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) {
|
||||
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) {
|
||||
charge := .1
|
||||
calls := 0
|
||||
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return &promptkit.GenerateResponse{Content: "invalid"}, nil
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil
|
||||
})))
|
||||
var terminal promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) {
|
||||
if e.Kind == promptkit.ExecutionFinished {
|
||||
terminal = e
|
||||
}
|
||||
}})
|
||||
want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed}
|
||||
if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want {
|
||||
t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) {
|
||||
engine := costEngine(t, true, nil)
|
||||
var terminal promptkit.ExecutionEvent
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) {
|
||||
if e.Kind == promptkit.ExecutionFinished {
|
||||
terminal = e
|
||||
}
|
||||
}})
|
||||
if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user