Isolate cost accounting for prepared executions
This commit is contained in:
131
cost_prepared_contract_test.go
Normal file
131
cost_prepared_contract_test.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestPreparedCostAndRejectedClaims(t *testing.T) {
|
||||
calls := 0
|
||||
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return &promptkit.GenerateResponse{Content: "bad"}, nil
|
||||
}
|
||||
return nil, errors.New("repair failed")
|
||||
})))
|
||||
ctx := context.Background()
|
||||
var events []promptkit.ExecutionEvent
|
||||
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
|
||||
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
|
||||
if err != nil || len(events) != 0 {
|
||||
t.Fatalf("prepare error=%v events=%+v", err, events)
|
||||
}
|
||||
result, err := engine.RunPrepared(ctx, handle)
|
||||
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
|
||||
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
|
||||
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
|
||||
}
|
||||
events = nil
|
||||
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
|
||||
t.Fatalf("reused err=%v events=%+v", err, events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
|
||||
childCalls := 0
|
||||
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
childCalls++
|
||||
charge := .2
|
||||
if childCalls == 2 {
|
||||
charge = .3
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||||
})))
|
||||
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var childRun, childPrepared *promptkit.RunResult
|
||||
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
var err error
|
||||
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
childPrepared, err = child.RunPrepared(ctx, handle)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
charge := 1.0
|
||||
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||
})))
|
||||
var group promptkit.ExecutionGroup
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||||
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
|
||||
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||||
t.Fatalf("summary=%+v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
|
||||
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
charge := 2.0
|
||||
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
|
||||
})))
|
||||
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
charge := 1.0
|
||||
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||
})))
|
||||
var group promptkit.ExecutionGroup
|
||||
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
|
||||
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
|
||||
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
charge := .5
|
||||
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||||
})))
|
||||
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
charge := 1.0
|
||||
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||
})))
|
||||
var childResult *promptkit.RunResult
|
||||
var childErr error
|
||||
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
|
||||
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var group promptkit.ExecutionGroup
|
||||
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
|
||||
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
|
||||
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
|
||||
}
|
||||
summary := group.Snapshot()
|
||||
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||||
t.Fatalf("summary=%+v", summary)
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,8 @@ attempts above the first in a generation, and repair generations with at least
|
||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||
attempt events.
|
||||
|
||||
Ordinary runs install invocation-local cost accounting even without an observer.
|
||||
Ordinary runs and claimed prepared executions install invocation-local cost
|
||||
accounting even without an observer.
|
||||
The generation coordinator resolves cost once per dispatched attempt and adds
|
||||
it to that state independently of event delivery. The result and recorder read
|
||||
the same accumulator; groups merge first terminal costs independently of
|
||||
|
||||
@@ -373,6 +373,8 @@ Validation: assembled ordinary run/reliable execution contract tests.
|
||||
|
||||
## Stage 13 — Prepared Cost Lifetime And Nested Isolation
|
||||
|
||||
**Completed.**
|
||||
|
||||
- In internal RunPrepared, after successful claim create/install its own cost
|
||||
accumulator before recorder creation and generation. Use the same result and
|
||||
terminal accounting path as ordinary Run. Do not store an accumulator or
|
||||
|
||||
@@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
||||
return nil, err
|
||||
}
|
||||
defer payload.clear()
|
||||
cost := &observation.CostAccumulator{}
|
||||
ctx = observation.WithCostAccumulator(ctx, cost)
|
||||
scope := NewOperationScope(ctx, payload.operationTimeout)
|
||||
defer scope.Close()
|
||||
defer func() { err = scope.Finalize(err) }()
|
||||
@@ -121,6 +123,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
||||
if observer != nil {
|
||||
var idErr error
|
||||
recorder, idErr = observation.New(observer)
|
||||
recorder.SetCostAccumulator(cost)
|
||||
recorder.SetParentRunID(payload.parent.RunID())
|
||||
observation.RegisterClaim(ctx, recorder)
|
||||
ctx = observation.WithRecorder(ctx, recorder)
|
||||
|
||||
Reference in New Issue
Block a user