Files
promptkit/invocation_workflow_contract_test.go

128 lines
5.3 KiB
Go

package promptkit_test
import (
"context"
"encoding/json"
"io"
"math"
"net/http"
"strings"
"sync"
"sync/atomic"
"testing"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestConcurrentInvocationObservationAcrossProviders(t *testing.T) {
pendingStarted, release := make(chan struct{}), make(chan struct{})
releasePending := sync.OnceFunc(func() { close(release) })
defer releasePending()
var retryCalls atomic.Int32
transport := costRoundTrip(func(req *http.Request) (*http.Response, error) {
var body struct {
Model string `json:"model"`
}
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
return nil, err
}
status, payload := 200, ""
switch body.Model {
case "router":
if retryCalls.Add(1) == 1 {
status, payload = 503, `{"error":{}}`
} else {
payload = `{"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":5},"completion_tokens_details":{"reasoning_tokens":3},"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`
}
case "failure":
payload = `{"usage":{"prompt_tokens":30,"completion_tokens":6,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":15,"cache_write_tokens":1},"completion_tokens_details":{"reasoning_tokens":1},"cost":0.01},"choices":[]}`
case "vllm":
payload = `{"usage":{"prompt_tokens":20,"completion_tokens":8,"total_tokens":28,"prompt_tokens_details":{"cached_tokens":10,"created_cache_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}},"choices":[{"message":{"content":"ok"}}]}`
case "pending":
close(pendingStarted)
<-release
payload = `{"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12,"prompt_tokens_details":{"cached_tokens":0,"created_cache_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}},"choices":[{"message":{"content":"ok"}}]}`
default:
t.Errorf("unexpected model %q", body.Model)
}
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(payload))}, nil
})
engine := func(provider, model string) *promptkit.Engine {
t.Helper()
input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("workflow", map[string]any{
"backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": provider, "zero_cost": true}},
"profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": model}},
"prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
})}
result, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: &http.Client{Transport: transport}}, promptkit.WithBundledOpenRouter(false))
if err != nil {
t.Fatal(err)
}
return result
}
router, vllm := engine("openrouter", "router"), engine("vllm", "vllm")
var group promptkit.ExecutionGroup
if empty := group.Snapshot(); len(empty.Members) != 0 || empty.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) {
t.Fatalf("empty=%+v", empty)
}
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
results := make(chan error, 4)
var completed sync.WaitGroup
var pendingDone sync.WaitGroup
start := func(e *promptkit.Engine, model string, fail bool) {
completed.Add(1)
go func() {
defer completed.Done()
req := promptkit.RunRequest{PromptID: "prompt"}
if model != "" {
req.Execution = &promptkit.ExecutionTargetOverride{Model: model}
}
_, err := e.Run(ctx, req)
if fail && err != nil {
err = nil
} else if fail {
err = io.ErrUnexpectedEOF
}
results <- err
}()
}
pendingDone.Add(1)
go func() {
defer pendingDone.Done()
_, err := vllm.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Execution: &promptkit.ExecutionTargetOverride{Model: "pending"}})
results <- err
}()
<-pendingStarted
if pending := group.Snapshot(); len(pending.Members) != 1 || pending.CostUSD != (promptkit.ObservedCost{}) || pending.TerminalCoverageComplete {
t.Fatalf("pending=%+v", pending)
}
start(router, "", false)
start(router, "failure", true)
start(vllm, "", false)
completed.Wait()
for range 3 {
if err := <-results; err != nil {
t.Fatal(err)
}
}
snapshot := group.Snapshot()
if len(snapshot.Members) != 4 || snapshot.TerminalCoverageComplete || snapshot.Attempts != 4 || !snapshot.CostUSD.Known || !snapshot.CostUSD.Complete || snapshot.CostUSD.Source != promptkit.CostSourceMixed || math.Abs(snapshot.CostUSD.Value-.03) > 1e-12 {
t.Fatalf("snapshot=%+v", snapshot)
}
if snapshot.Usage.PromptTokens.Value != 150 || snapshot.Usage.ReasoningTokens.Value != 6 || snapshot.Usage.CachedTokens.Value != 65 || snapshot.Usage.CacheWriteTokens.Value != 10 || snapshot.Usage.PromptTokens.Complete {
t.Fatalf("usage=%+v", snapshot.Usage)
}
if _, ok := snapshot.Usage.CacheHitRatio(); ok {
t.Fatal("ratio available despite an unreported retry")
}
releasePending()
if err := <-results; err != nil {
t.Fatal(err)
}
pendingDone.Wait()
if final := group.Snapshot(); !final.TerminalCoverageComplete || len(final.Members) != 4 || final.Attempts != 5 || math.Abs(final.CostUSD.Value-.03) > 1e-12 {
t.Fatalf("final=%+v", final)
}
}