diff --git a/cost_run_contract_test.go b/cost_run_contract_test.go new file mode 100644 index 0000000..e348da0 --- /dev/null +++ b/cost_run_contract_test.go @@ -0,0 +1,124 @@ +package promptkit_test + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +type costRoundTrip func(*http.Request) (*http.Response, error) + +func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine { + t.Helper() + input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{ + "backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}}, + "profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}}, + "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, + })} + options = append(options, promptkit.WithBundledOpenRouter(false)) + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...) + if err != nil { + t.Fatal(err) + } + return engine +} + +func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) { + for _, observed := range []bool{false, true} { + t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) { + calls := 0 + transport := costRoundTrip(func(*http.Request) (*http.Response, error) { + calls++ + if calls == 1 { + return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil + }) + engine := costEngine(t, false, &http.Client{Transport: transport}) + var events []promptkit.ExecutionEvent + req := promptkit.RunRequest{PromptID: "prompt"} + if observed { + req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) } + } + result, err := engine.Run(context.Background(), req) + want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider} + if err != nil || result == nil || result.CostUSD != want || calls != 2 { + t.Fatalf("result=%+v calls=%d err=%v", result, calls, err) + } + if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) { + t.Fatalf("events=%+v", events) + } + }) + } +} + +func TestOrdinaryRunCostFailureAndFallback(t *testing.T) { + charge := .125 + for _, tc := range []struct { + name string + response *promptkit.GenerateResponse + err error + want promptkit.ObservedCost + }{ + {"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}}, + {"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}}, + } { + t.Run(tc.name, func(t *testing.T) { + engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return tc.response, tc.err + }))) + var terminal promptkit.ExecutionEvent + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) { + if e.Kind == promptkit.ExecutionFinished { + terminal = e + } + }}) + if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) { + t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err) + } + }) + } +} + +func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) { + charge := .1 + calls := 0 + engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + if calls == 1 { + return &promptkit.GenerateResponse{Content: "invalid"}, nil + } + return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil + }))) + var terminal promptkit.ExecutionEvent + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) { + if e.Kind == promptkit.ExecutionFinished { + terminal = e + } + }}) + want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed} + if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want { + t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err) + } +} + +func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) { + engine := costEngine(t, true, nil) + var terminal promptkit.ExecutionEvent + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) { + if e.Kind == promptkit.ExecutionFinished { + terminal = e + } + }}) + if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) { + t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err) + } +} diff --git a/docs/internal/observation.md b/docs/internal/observation.md index aecb444..9bca0cf 100644 --- a/docs/internal/observation.md +++ b/docs/internal/observation.md @@ -56,10 +56,11 @@ attempts above the first in a generation, and repair generations with at least one dispatch. A terminal event is an aggregate and must not be added to its attempt events. +Ordinary runs install invocation-local cost accounting even without an observer. The generation coordinator resolves cost once per dispatched attempt and adds -it to invocation-local accounting independently of event delivery. A recorder -can read that same accumulator at termination; groups merge first terminal -costs independently of terminal coverage. +it to that state independently of event delivery. The result and recorder read +the same accumulator; groups merge first terminal costs independently of +terminal coverage. Reasoning counts use the same per-field rules and are a subset of completion tokens. The public cache-hit helper derives a ratio from complete, consistent diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 922510a..92b7e5f 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -353,6 +353,8 @@ Validation: generation, observation recorder/group, and diagnostic-boundary test ## Stage 12 — Ordinary Run Cost Lifetime And Result Accounting +**Completed.** + - Create/install a new private accumulator at the public ordinary Run boundary, before optional recorder creation. Pass that same instance to the recorder. Ensure direct internal runner execution creates a local accumulator if one diff --git a/engine.go b/engine.go index 1769246..872283e 100644 --- a/engine.go +++ b/engine.go @@ -602,6 +602,8 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare // result on error. func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { ctx = observation.WithoutExecutionState(ctx) + cost := &observation.CostAccumulator{} + ctx = observation.WithCostAccumulator(ctx, cost) scope := usecase.NewOperationScope(ctx, req.OperationTimeout) defer scope.Close() ctx = scope.Context @@ -617,6 +619,7 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) { return result, err } recorder, idErr := observation.New(observer) + recorder.SetCostAccumulator(cost) if req.ContinueFrom != nil && req.ContinueFrom.internal != nil { recorder.SetParentRunID(req.ContinueFrom.internal.RunID()) } diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 5853aef..3b383cb 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -137,6 +137,9 @@ func NewRunnerWithRepairer( } func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) { + if observation.CostFromContext(ctx) == nil { + ctx = observation.WithCostAccumulator(ctx, &observation.CostAccumulator{}) + } recorder := observation.FromContext(ctx) runID := "" if recorder != nil { @@ -274,7 +277,7 @@ func (r *Runner) executePreparedRun( exchange = domain.CaptureExchange(parent, runID, prepared.Messages[parent.MessageCount():], genResp.Content) } - return &domain.RunResult{ + result := &domain.RunResult{ Exchange: exchange, ParentRunID: prepared.ParentRunID, RunID: runID, @@ -295,7 +298,11 @@ func (r *Runner) executePreparedRun( StartTime: start, EndTime: end, Duration: end.Sub(start), - }, nil + } + if cost := observation.CostFromContext(ctx); cost != nil { + result.CostUSD = cost.Observed() + } + return result, nil } func wrapGenerationError(err error) error {