Isolate cost accounting for prepared executions
This commit is contained in:
131
cost_prepared_contract_test.go
Normal file
131
cost_prepared_contract_test.go
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreparedCostAndRejectedClaims(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
return &promptkit.GenerateResponse{Content: "bad"}, nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("repair failed")
|
||||||
|
})))
|
||||||
|
ctx := context.Background()
|
||||||
|
var events []promptkit.ExecutionEvent
|
||||||
|
ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) })
|
||||||
|
handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}})
|
||||||
|
if err != nil || len(events) != 0 {
|
||||||
|
t.Fatalf("prepare error=%v events=%+v", err, events)
|
||||||
|
}
|
||||||
|
result, err := engine.RunPrepared(ctx, handle)
|
||||||
|
want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}
|
||||||
|
if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want {
|
||||||
|
t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err)
|
||||||
|
}
|
||||||
|
events = nil
|
||||||
|
if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 {
|
||||||
|
t.Fatalf("reused err=%v events=%+v", err, events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) {
|
||||||
|
childCalls := 0
|
||||||
|
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
childCalls++
|
||||||
|
charge := .2
|
||||||
|
if childCalls == 2 {
|
||||||
|
charge = .3
|
||||||
|
}
|
||||||
|
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||||||
|
})))
|
||||||
|
handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var childRun, childPrepared *promptkit.RunResult
|
||||||
|
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
var err error
|
||||||
|
childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
childPrepared, err = child.RunPrepared(ctx, handle)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
charge := 1.0
|
||||||
|
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||||
|
})))
|
||||||
|
var group promptkit.ExecutionGroup
|
||||||
|
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||||||
|
result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 {
|
||||||
|
t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD)
|
||||||
|
}
|
||||||
|
summary := group.Snapshot()
|
||||||
|
if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||||||
|
t.Fatalf("summary=%+v", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) {
|
||||||
|
diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
charge := 2.0
|
||||||
|
return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil
|
||||||
|
})))
|
||||||
|
outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
charge := 1.0
|
||||||
|
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||||
|
})))
|
||||||
|
var group promptkit.ExecutionGroup
|
||||||
|
result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
summary := group.Snapshot()
|
||||||
|
if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 {
|
||||||
|
t.Fatalf("result=%+v group=%+v", result.CostUSD, summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedCallbackChildHasIndependentCost(t *testing.T) {
|
||||||
|
child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
charge := .5
|
||||||
|
return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil
|
||||||
|
})))
|
||||||
|
parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
charge := 1.0
|
||||||
|
return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil
|
||||||
|
})))
|
||||||
|
var childResult *promptkit.RunResult
|
||||||
|
var childErr error
|
||||||
|
handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) {
|
||||||
|
childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var group promptkit.ExecutionGroup
|
||||||
|
result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle)
|
||||||
|
if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 {
|
||||||
|
t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr)
|
||||||
|
}
|
||||||
|
summary := group.Snapshot()
|
||||||
|
if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 {
|
||||||
|
t.Fatalf("summary=%+v", summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -56,7 +56,8 @@ attempts above the first in a generation, and repair generations with at least
|
|||||||
one dispatch. A terminal event is an aggregate and must not be added to its
|
one dispatch. A terminal event is an aggregate and must not be added to its
|
||||||
attempt events.
|
attempt events.
|
||||||
|
|
||||||
Ordinary runs install invocation-local cost accounting even without an observer.
|
Ordinary runs and claimed prepared executions install invocation-local cost
|
||||||
|
accounting even without an observer.
|
||||||
The generation coordinator resolves cost once per dispatched attempt and adds
|
The generation coordinator resolves cost once per dispatched attempt and adds
|
||||||
it to that state independently of event delivery. The result and recorder read
|
it to that state independently of event delivery. The result and recorder read
|
||||||
the same accumulator; groups merge first terminal costs independently of
|
the same accumulator; groups merge first terminal costs independently of
|
||||||
|
|||||||
@@ -373,6 +373,8 @@ Validation: assembled ordinary run/reliable execution contract tests.
|
|||||||
|
|
||||||
## Stage 13 — Prepared Cost Lifetime And Nested Isolation
|
## Stage 13 — Prepared Cost Lifetime And Nested Isolation
|
||||||
|
|
||||||
|
**Completed.**
|
||||||
|
|
||||||
- In internal RunPrepared, after successful claim create/install its own cost
|
- In internal RunPrepared, after successful claim create/install its own cost
|
||||||
accumulator before recorder creation and generation. Use the same result and
|
accumulator before recorder creation and generation. Use the same result and
|
||||||
terminal accounting path as ordinary Run. Do not store an accumulator or
|
terminal accounting path as ordinary Run. Do not store an accumulator or
|
||||||
|
|||||||
@@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer payload.clear()
|
defer payload.clear()
|
||||||
|
cost := &observation.CostAccumulator{}
|
||||||
|
ctx = observation.WithCostAccumulator(ctx, cost)
|
||||||
scope := NewOperationScope(ctx, payload.operationTimeout)
|
scope := NewOperationScope(ctx, payload.operationTimeout)
|
||||||
defer scope.Close()
|
defer scope.Close()
|
||||||
defer func() { err = scope.Finalize(err) }()
|
defer func() { err = scope.Finalize(err) }()
|
||||||
@@ -121,6 +123,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
if observer != nil {
|
if observer != nil {
|
||||||
var idErr error
|
var idErr error
|
||||||
recorder, idErr = observation.New(observer)
|
recorder, idErr = observation.New(observer)
|
||||||
|
recorder.SetCostAccumulator(cost)
|
||||||
recorder.SetParentRunID(payload.parent.RunID())
|
recorder.SetParentRunID(payload.parent.RunID())
|
||||||
observation.RegisterClaim(ctx, recorder)
|
observation.RegisterClaim(ctx, recorder)
|
||||||
ctx = observation.WithRecorder(ctx, recorder)
|
ctx = observation.WithRecorder(ctx, recorder)
|
||||||
|
|||||||
Reference in New Issue
Block a user