Isolate cost accounting for prepared executions

This commit is contained in:
2026-11-22 18:30:02 +00:00
parent d77e3e2f83
commit ba4de9d11e
4 changed files with 138 additions and 1 deletions

View File

@@ -0,0 +1,131 @@
package promptkit_test
import (
"context"
"errors"
"math"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestPreparedCostAndRejectedClaims(t *testing.T) {
calls := 0
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return &promptkit.GenerateResponse{Content: "bad"}, nil
}
return nil, errors.New("repair failed")
})))
ctx := context.Background()
var events []promptkit.ExecutionEvent
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
if err != nil || len(events) != 0 {
t.Fatalf("prepare error=%v events=%+v", err, events)
}
result, err := engine.RunPrepared(ctx, handle)
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
}
events = nil
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
t.Fatalf("reused err=%v events=%+v", err, events)
}
}
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
childCalls := 0
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
childCalls++
charge := .2
if childCalls == 2 {
charge = .3
}
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
var childRun, childPrepared *promptkit.RunResult
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
var err error
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
return nil, err
}
childPrepared, err = child.RunPrepared(ctx, handle)
if err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
}
summary := group.Snapshot()
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 2.0
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
})))
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
return nil, err
}
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var group promptkit.ExecutionGroup
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
summary := group.Snapshot()
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
}
}
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := .5
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
})))
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
charge := 1.0
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
})))
var childResult *promptkit.RunResult
var childErr error
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
}})
if err != nil {
t.Fatal(err)
}
var group promptkit.ExecutionGroup
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
}
summary := group.Snapshot()
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
t.Fatalf("summary=%+v", summary)
}
}

View File

@@ -56,7 +56,8 @@ attempts above the first in a generation, and repair generations with at least
one dispatch. A terminal event is an aggregate and must not be added to its one dispatch. A terminal event is an aggregate and must not be added to its
attempt events. attempt events.
Ordinary runs install invocation-local cost accounting even without an observer. Ordinary runs and claimed prepared executions install invocation-local cost
accounting even without an observer.
The generation coordinator resolves cost once per dispatched attempt and adds The generation coordinator resolves cost once per dispatched attempt and adds
it to that state independently of event delivery. The result and recorder read it to that state independently of event delivery. The result and recorder read
the same accumulator; groups merge first terminal costs independently of the same accumulator; groups merge first terminal costs independently of

View File

@@ -373,6 +373,8 @@ Validation: assembled ordinary run/reliable execution contract tests.
## Stage 13 — Prepared Cost Lifetime And Nested Isolation ## Stage 13 — Prepared Cost Lifetime And Nested Isolation
**Completed.**
- In internal RunPrepared, after successful claim create/install its own cost - In internal RunPrepared, after successful claim create/install its own cost
accumulator before recorder creation and generation. Use the same result and accumulator before recorder creation and generation. Use the same result and
terminal accounting path as ordinary Run. Do not store an accumulator or terminal accounting path as ordinary Run. Do not store an accumulator or

View File

@@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
return nil, err return nil, err
} }
defer payload.clear() defer payload.clear()
cost := &observation.CostAccumulator{}
ctx = observation.WithCostAccumulator(ctx, cost)
scope := NewOperationScope(ctx, payload.operationTimeout) scope := NewOperationScope(ctx, payload.operationTimeout)
defer scope.Close() defer scope.Close()
defer func() { err = scope.Finalize(err) }() defer func() { err = scope.Finalize(err) }()
@@ -121,6 +123,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
if observer != nil { if observer != nil {
var idErr error var idErr error
recorder, idErr = observation.New(observer) recorder, idErr = observation.New(observer)
recorder.SetCostAccumulator(cost)
recorder.SetParentRunID(payload.parent.RunID()) recorder.SetParentRunID(payload.parent.RunID())
observation.RegisterClaim(ctx, recorder) observation.RegisterClaim(ctx, recorder)
ctx = observation.WithRecorder(ctx, recorder) ctx = observation.WithRecorder(ctx, recorder)