package promptkit_test import ( "context" "errors" "io" "net/http" "strings" "testing" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) type costRoundTrip func(*http.Request) (*http.Response, error) func (f costRoundTrip) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } func costEngine(t *testing.T, zero bool, client *http.Client, options ...promptkit.Option) *promptkit.Engine { t.Helper() input := configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("cost", map[string]any{ "backends": []any{map[string]any{"id": "backend", "endpoint": "https://example.invalid/v1", "provider": "openrouter", "zero_cost": zero}}, "profiles": []any{map[string]any{"id": "profile", "backend": "backend", "model": "model"}}, "prompts": []any{map[string]any{"id": "prompt", "default_profile": "profile", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, })} options = append(options, promptkit.WithBundledOpenRouter(false)) engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}, HTTPClient: client}, options...) if err != nil { t.Fatal(err) } return engine } func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) { for _, observed := range []bool{false, true} { t.Run(map[bool]string{false: "without observer", true: "with observer"}[observed], func(t *testing.T) { calls := 0 transport := costRoundTrip(func(*http.Request) (*http.Response, error) { calls++ if calls == 1 { return &http.Response{StatusCode: 503, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{}}`))}, nil } return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"usage":{"cost":0.02},"choices":[{"message":{"content":"ok"}}]}`))}, nil }) engine := costEngine(t, false, &http.Client{Transport: transport}) var events []promptkit.ExecutionEvent req := promptkit.RunRequest{PromptID: "prompt"} if observed { req.Observer = func(e promptkit.ExecutionEvent) { events = append(events, e) } } result, err := engine.Run(context.Background(), req) want := promptkit.ObservedCost{Value: .02, Known: true, Complete: false, Source: promptkit.CostSourceProvider} if err != nil || result == nil || result.CostUSD != want || calls != 2 { t.Fatalf("result=%+v calls=%d err=%v", result, calls, err) } if observed && (len(events) != 5 || events[len(events)-1].CostUSD != want) { t.Fatalf("events=%+v", events) } }) } } func TestOrdinaryRunCostFailureAndFallback(t *testing.T) { charge := .125 for _, tc := range []struct { name string response *promptkit.GenerateResponse err error want promptkit.ObservedCost }{ {"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}}, {"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}}, } { t.Run(tc.name, func(t *testing.T) { engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return tc.response, tc.err }))) var terminal promptkit.ExecutionEvent result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Observer: func(e promptkit.ExecutionEvent) { if e.Kind == promptkit.ExecutionFinished { terminal = e } }}) if (tc.err != nil) != (err != nil) || terminal.CostUSD != tc.want || (result != nil && result.CostUSD != tc.want) { t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err) } }) } } func TestOrdinaryRunCombinesRepairCostSources(t *testing.T) { charge := .1 calls := 0 engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ if calls == 1 { return &promptkit.GenerateResponse{Content: "invalid"}, nil } return &promptkit.GenerateResponse{Content: `{"valid":true}`, CostUSD: &charge}, nil }))) var terminal promptkit.ExecutionEvent result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Validation: &promptkit.OutputContractInput{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: new(1)}, Observer: func(e promptkit.ExecutionEvent) { if e.Kind == promptkit.ExecutionFinished { terminal = e } }}) want := promptkit.ObservedCost{Value: .1, Known: true, Complete: true, Source: promptkit.CostSourceMixed} if err != nil || result == nil || calls != 2 || result.CostUSD != want || terminal.CostUSD != want { t.Fatalf("result=%+v terminal=%+v calls=%d err=%v", result, terminal, calls, err) } } func TestOrdinaryRunBeforeDispatchHasEmptyCost(t *testing.T) { engine := costEngine(t, true, nil) var terminal promptkit.ExecutionEvent result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "missing", Observer: func(e promptkit.ExecutionEvent) { if e.Kind == promptkit.ExecutionFinished { terminal = e } }}) if result != nil || err == nil || terminal.CostUSD != (promptkit.ObservedCost{Known: true, Complete: true}) { t.Fatalf("result=%+v terminal=%+v err=%v", result, terminal, err) } }