Files
promptkit/cost_prepared_contract_test.go

132 lines
5.8 KiB
Go
Raw Normal View History

package promptkit_test
import (
"context"
"errors"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestPreparedCostAndRejectedClaims(t *testing.T) {
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "bad"}, nil
}
return nil, errors.New("repair failed")
})))
ctx := context.Background()
var events []promptkit.ExecutionEvent
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
if err != nil || len(events) != 0 {
t.Fatalf("prepare error=%v events=%+v", err, events)
}
result, err := engine.RunPrepared(ctx, handle)
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
}
events = nil
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
t.Fatalf("reused err=%v events=%+v", err, events)
}
}
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
childCalls := 0
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
childCalls++
charge := .2
if childCalls == 2 {
charge = .3
}
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
var childRun, childPrepared *promptkit.RunResult
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
var err error
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
return nil, err
}
childPrepared, err = child.RunPrepared(ctx, handle)
if err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
}
summary := group.Snapshot()
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 2.0
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
})))
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
summary := group.Snapshot()
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
}
}
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := .5
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var childResult *promptkit.RunResult
var childErr error
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
}})
if err != nil {
t.Fatal(err)
}
var group promptkit.ExecutionGroup
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
}
summary := group.Snapshot()
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}