Cover concurrent multi-provider invocation accounting
This commit is contained in:
127
invocation_workflow_contract_test.go
Normal file
127
invocation_workflow_contract_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) {
|
||||
pendingStarted, release := make(chan struct{}), make(chan struct{})
|
||||
releasePending := sync.OnceFunc(func() { close(release) })
|
||||
defer releasePending()
|
||||
var retryCalls atomic.Int32
|
||||
transport := costRoundTrip(func(req *http.Request) (*http.Response, error) {
|
||||
var body struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status, payload := 200, ""
|
||||
switch body.Model {
|
||||
case "router":
|
||||
if retryCalls.Add(1) == 1 {
|
||||
status, payload = 503, `{"error":{}}`
|
||||
} else {
|
||||
payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`
|
||||
}
|
||||
case "failure":
|
||||
payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}`
|
||||
case "vllm":
|
||||
payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}`
|
||||
case "pending":
|
||||
close(pendingStarted)
|
||||
<-release
|
||||
payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}`
|
||||
default:
|
||||
t.Errorf("unexpected model %q", body.Model)
|
||||
}
|
||||
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil
|
||||
})
|
||||
engine := func(provider, model string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}},
|
||||
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}
|
||||
result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
router, vllm := engine("openrouter", "router"), engine("vllm", "vllm")
|
||||
var group promptkit.ExecutionGroup
|
||||
if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("empty=%+v", empty)
|
||||
}
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||||
results := make(chan error, 4)
|
||||
var completed sync.WaitGroup
|
||||
var pendingDone sync.WaitGroup
|
||||
start := func(e *promptkit.Engine, model string, fail bool) {
|
||||
completed.Add(1)
|
||||
go func() {
|
||||
defer completed.Done()
|
||||
req := promptkit.RunRequest{PromptID: "prompt"}
|
||||
if model != "" {
|
||||
req.Execution = &promptkit.ExecutionTargetOverride{Model: model}
|
||||
}
|
||||
_, err := e.Run(ctx, req)
|
||||
if fail && err != nil {
|
||||
err = nil
|
||||
} else if fail {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
pendingDone.Add(1)
|
||||
go func() {
|
||||
defer pendingDone.Done()
|
||||
_, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}})
|
||||
results <- err
|
||||
}()
|
||||
<-pendingStarted
|
||||
if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete {
|
||||
t.Fatalf("pending=%+v", pending)
|
||||
}
|
||||
start(router, "", false)
|
||||
start(router, "failure", true)
|
||||
start(vllm, "", false)
|
||||
completed.Wait()
|
||||
for range 3 {
|
||||
if err := <-results; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
snapshot := group.Snapshot()
|
||||
if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 {
|
||||
t.Fatalf("snapshot=%+v", snapshot)
|
||||
}
|
||||
if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete {
|
||||
t.Fatalf("usage=%+v", snapshot.Usage)
|
||||
}
|
||||
if _, ok := snapshot.Usage.CacheHitRatio(); ok {
|
||||
t.Fatal("ratio available despite an unreported retry")
|
||||
}
|
||||
releasePending()
|
||||
if err := <-results; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pendingDone.Wait()
|
||||
if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 {
|
||||
t.Fatalf("final=%+v", final)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user