132 lines
5.8 KiB
Go
132 lines
5.8 KiB
Go
|
|
package promptkit_test
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"math"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestPreparedCostAndRejectedClaims(t *testing.T) {
|
||
|
|
calls := 0
|
||
|
|
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
calls++
|
||
|
|
if calls == 1 {
|
||
|
|
return &promptkit.GenerateResponse{Content: "bad"}, nil
|
||
|
|
}
|
||
|
|
return nil, errors.New("repair failed")
|
||
|
|
})))
|
||
|
|
ctx := context.Background()
|
||
|
|
var events []promptkit.ExecutionEvent
|
||
|
|
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
|
||
|
|
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
|
||
|
|
if err != nil || len(events) != 0 {
|
||
|
|
t.Fatalf("prepare error=%v events=%+v", err, events)
|
||
|
|
}
|
||
|
|
result, err := engine.RunPrepared(ctx, handle)
|
||
|
|
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
|
||
|
|
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
|
||
|
|
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
|
||
|
|
}
|
||
|
|
events = nil
|
||
|
|
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
|
||
|
|
t.Fatalf("reused err=%v events=%+v", err, events)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
|
||
|
|
childCalls := 0
|
||
|
|
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
childCalls++
|
||
|
|
charge := .2
|
||
|
|
if childCalls == 2 {
|
||
|
|
charge = .3
|
||
|
|
}
|
||
|
|
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||
|
|
})))
|
||
|
|
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var childRun, childPrepared *promptkit.RunResult
|
||
|
|
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
var err error
|
||
|
|
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
childPrepared, err = child.RunPrepared(ctx, handle)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
charge := 1.0
|
||
|
|
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||
|
|
})))
|
||
|
|
var group promptkit.ExecutionGroup
|
||
|
|
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||
|
|
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
|
||
|
|
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
|
||
|
|
}
|
||
|
|
summary := group.Snapshot()
|
||
|
|
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||
|
|
t.Fatalf("summary=%+v", summary)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
|
||
|
|
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
charge := 2.0
|
||
|
|
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
|
||
|
|
})))
|
||
|
|
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
charge := 1.0
|
||
|
|
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||
|
|
})))
|
||
|
|
var group promptkit.ExecutionGroup
|
||
|
|
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
summary := group.Snapshot()
|
||
|
|
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
|
||
|
|
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
|
||
|
|
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
charge := .5
|
||
|
|
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||
|
|
})))
|
||
|
|
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
charge := 1.0
|
||
|
|
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||
|
|
})))
|
||
|
|
var childResult *promptkit.RunResult
|
||
|
|
var childErr error
|
||
|
|
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
|
||
|
|
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||
|
|
}})
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var group promptkit.ExecutionGroup
|
||
|
|
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
|
||
|
|
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
|
||
|
|
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
|
||
|
|
}
|
||
|
|
summary := group.Snapshot()
|
||
|
|
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||
|
|
t.Fatalf("summary=%+v", summary)
|
||
|
|
}
|
||
|
|
}
|