diff --git a/cost_prepared_contract_test.go b/cost_prepared_contract_test.go new file mode 100644 index 0000000..accdc0b --- /dev/null +++ b/cost_prepared_contract_test.go @@ -0,0 +1,131 @@ +package promptkit_test + +import ( + "context" + "errors" + "math" + "testing" + + "go.maximumdirect.net/promptkit" +) + +func TestPreparedCostAndRejectedClaims(t *testing.T) { + calls := 0 + engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if calls == 1 { + return &promptkit.GenerateResponse{Content: "bad"}, nil + } + return nil, errors.New("repair failed") + }))) + ctx := context.Background() + var events []promptkit.ExecutionEvent + ctx = promptkit.WithExecutionObserver(ctx, func(e promptkit.ExecutionEvent) { events = append(events, e) }) + handle, err := engine.PrepareExecution(ctx, promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}}) + if err != nil || len(events) != 0 { + t.Fatalf("prepare error=%v events=%+v", err, events) + } + result, err := engine.RunPrepared(ctx, handle) + want := promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero} + if result != nil || err == nil || calls != 2 || len(events) != 4 || events[len(events)-1].CostUSD != want { + t.Fatalf("result=%+v events=%+v calls=%d err=%v", result, events, calls, err) + } + events = nil + if _, err := engine.RunPrepared(ctx, handle); !errors.Is(err, promptkit.ErrInvalidRequest) || len(events) != 0 { + t.Fatalf("reused err=%v events=%+v", err, events) + } +} + +func TestNestedOrdinaryAndPreparedRunsIsolateCosts(t *testing.T) { + childCalls := 0 + child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + childCalls++ + charge := .2 + if childCalls == 2 { + charge = .3 + } + return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil + }))) + handle, err := child.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatal(err) + } + var childRun, childPrepared *promptkit.RunResult + outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + var err error + childRun, err = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + return nil, err + } + childPrepared, err = child.RunPrepared(ctx, handle) + if err != nil { + return nil, err + } + charge := 1.0 + return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil + }))) + var group promptkit.ExecutionGroup + ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe) + result, err := outer.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatal(err) + } + if result.CostUSD.Value != 1 || childRun.CostUSD.Value != .2 || childPrepared.CostUSD.Value != .3 { + t.Fatalf("parent=%+v child=%+v prepared=%+v", result.CostUSD, childRun.CostUSD, childPrepared.CostUSD) + } + summary := group.Snapshot() + if len(summary.Members) != 3 || !summary.TerminalCoverageComplete || !summary.CostUSD.Complete || summary.CostUSD.Source != promptkit.CostSourceProvider || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 { + t.Fatalf("summary=%+v", summary) + } +} + +func TestNestedDiagnosticDoesNotContributeCostOrEvents(t *testing.T) { + diagnostic := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + charge := 2.0 + return &promptkit.GenerateResponse{Content: "OK", CostUSD: &charge}, nil + }))) + outer := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(ctx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if _, err := diagnostic.SmokeTest(ctx, promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile"}); err != nil { + return nil, err + } + charge := 1.0 + return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil + }))) + var group promptkit.ExecutionGroup + result, err := outer.Run(promptkit.WithExecutionObserver(context.Background(), group.Observe), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatal(err) + } + summary := group.Snapshot() + if result.CostUSD.Value != 1 || summary.CostUSD.Value != 1 || len(summary.Members) != 1 { + t.Fatalf("result=%+v group=%+v", result.CostUSD, summary) + } +} + +func TestPreparedCallbackChildHasIndependentCost(t *testing.T) { + child := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + charge := .5 + return &promptkit.GenerateResponse{Content: "child", CostUSD: &charge}, nil + }))) + parent := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + charge := 1.0 + return &promptkit.GenerateResponse{Content: "parent", CostUSD: &charge}, nil + }))) + var childResult *promptkit.RunResult + var childErr error + handle, err := parent.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt", OnPrepared: func(ctx context.Context, _ promptkit.PreparedRun) { + childResult, childErr = child.Run(ctx, promptkit.RunRequest{PromptID: "prompt"}) + }}) + if err != nil { + t.Fatal(err) + } + var group promptkit.ExecutionGroup + result, err := parent.RunPrepared(promptkit.WithExecutionObserver(context.Background(), group.Observe), handle) + if err != nil || childErr != nil || result.CostUSD.Value != 1 || childResult == nil || childResult.CostUSD.Value != .5 { + t.Fatalf("parent=%+v child=%+v errors=%v,%v", result, childResult, err, childErr) + } + summary := group.Snapshot() + if len(summary.Members) != 2 || math.Abs(summary.CostUSD.Value-1.5) > 1e-12 { + t.Fatalf("summary=%+v", summary) + } +} diff --git a/docs/internal/observation.md b/docs/internal/observation.md index 9bca0cf..1980f2c 100644 --- a/docs/internal/observation.md +++ b/docs/internal/observation.md @@ -56,7 +56,8 @@ attempts above the first in a generation, and repair generations with at least one dispatch. A terminal event is an aggregate and must not be added to its attempt events. -Ordinary runs install invocation-local cost accounting even without an observer. +Ordinary runs and claimed prepared executions install invocation-local cost +accounting even without an observer. The generation coordinator resolves cost once per dispatched attempt and adds it to that state independently of event delivery. The result and recorder read the same accumulator; groups merge first terminal costs independently of diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 92b7e5f..f7815d6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -373,6 +373,8 @@ Validation: assembled ordinary run/reliable execution contract tests. ## Stage 13 — Prepared Cost Lifetime And Nested Isolation +**Completed.** + - In internal RunPrepared, after successful claim create/install its own cost accumulator before recorder creation and generation. Use the same result and terminal accounting path as ordinary Run. Do not store an accumulator or diff --git a/internal/usecase/prepared_execution.go b/internal/usecase/prepared_execution.go index 86317f7..929abab 100644 --- a/internal/usecase/prepared_execution.go +++ b/internal/usecase/prepared_execution.go @@ -110,6 +110,8 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) ( return nil, err } defer payload.clear() + cost := &observation.CostAccumulator{} + ctx = observation.WithCostAccumulator(ctx, cost) scope := NewOperationScope(ctx, payload.operationTimeout) defer scope.Close() defer func() { err = scope.Finalize(err) }() @@ -121,6 +123,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) ( if observer != nil { var idErr error recorder, idErr = observation.New(observer) + recorder.SetCostAccumulator(cost) recorder.SetParentRunID(payload.parent.RunID()) observation.RegisterClaim(ctx, recorder) ctx = observation.WithRecorder(ctx, recorder)