From 43f55edb983f709dbec675262877469f897c365c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 3 Nov 2026 15:36:26 +0000 Subject: [PATCH] Test bundled OpenRouter smoke diagnostics --- bundled_smoke_contract_test.go | 127 +++++++++++++++++++++++++++++++++ docs/roadmap/implementation.md | 4 ++ 2 files changed, 131 insertions(+) create mode 100644 bundled_smoke_contract_test.go diff --git a/bundled_smoke_contract_test.go b/bundled_smoke_contract_test.go new file mode 100644 index 0000000..2d9c14f --- /dev/null +++ b/bundled_smoke_contract_test.go @@ -0,0 +1,127 @@ +package promptkit_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "reflect" + "strings" + "testing" + + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func TestBundledSmokeHTTPDefaults(t *testing.T) { + t.Setenv("OPENROUTER_API_KEY", "synthetic-key") + var urls []string + var models []string + transport := roundTripFunc(func(request *http.Request) (*http.Response, error) { + urls = append(urls, request.URL.String()) + body := `{"data":{"label":"member","is_free_tier":false,"usage":0}}` + if request.Method == http.MethodPost { + var payload map[string]any + if err := json.NewDecoder(request.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + models = append(models, payload["model"].(string)) + content := "PROMPTKIT_OK" + if _, structured := payload["response_format"]; structured { + content = `{"status":"ok"}` + } + encoded, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}}) + if err != nil { + t.Fatal(err) + } + body = string(encoded) + } else if strings.HasSuffix(request.URL.Path, "/key") && request.Header.Get("Authorization") != "Bearer synthetic-key" { + t.Errorf("authentication header missing") + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil + }) + engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}}) + if err != nil { + t.Fatal(err) + } + before, _ := engine.ListProfiles(context.Background()) + inspectionBefore, err := engine.InspectConfiguration() + if err != nil { + t.Fatal(err) + } + for _, kind := range []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput} { + report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: kind, BackendID: "openrouter"}) + if err != nil || report == nil || !report.Completed || report.Failure != "" { + t.Fatalf("%s report = %+v, %v", kind, report, err) + } + if kind == promptkit.SmokeTestAuthentication && (report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted) { + t.Fatalf("authentication facts = %+v", report.Facts) + } + if kind == promptkit.SmokeTestTextInference || kind == promptkit.SmokeTestStructuredOutput { + if report.Target.ProfileID != "gpt-luna-latest" || report.Target.EffectiveSettings.Model != "~openai/gpt-luna-latest" { + t.Fatalf("target = %+v", report.Target) + } + } + } + wantURLs := []string{"https://openrouter.ai/api/v1", "https://openrouter.ai/api/v1/key", "https://openrouter.ai/api/v1/chat/completions", "https://openrouter.ai/api/v1/chat/completions"} + if !reflect.DeepEqual(urls, wantURLs) || !reflect.DeepEqual(models, []string{"~openai/gpt-luna-latest", "~openai/gpt-luna-latest"}) { + t.Fatalf("urls %v, models %v", urls, models) + } + after, _ := engine.ListProfiles(context.Background()) + if !reflect.DeepEqual(before, after) { + t.Fatalf("inventory changed: %v %v", before, after) + } + inspectionAfter, err := engine.InspectConfiguration() + if err != nil { + t.Fatal(err) + } + diff, err := inspectionBefore.Diff(inspectionAfter) + if err != nil || len(diff) != 0 { + t.Fatalf("inspection changed: %+v, %v", diff, err) + } +} + +func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) { + replacement := configkit.Input{Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})} + replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}} + replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient)) + if err != nil { + t.Fatal(err) + } + replacementReport, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"}) + if err != nil || replacementReport.Target.ProfileID != "gpt-luna-latest" || replacementReport.Target.EffectiveSettings.Model != "replacement" { + t.Fatalf("replaced default = %+v, %v", replacementReport, err) + } + replacementClient.response.Content = "unexpected" + mismatch, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"}) + if err != nil || mismatch == nil || !mismatch.Completed { + t.Fatalf("concluded mismatch = %+v, %v", mismatch, err) + } + input := configkit.Input{Source: configkit.Mapping("application", map[string]any{ + "backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}}, + "profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}}, + "prompts": []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}}, + })} + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + request promptkit.SmokeTestRequest + profile, model string + }{ + {promptkit.SmokeTestRequest{BackendID: "openrouter"}, "custom", "custom-model"}, + {promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt"}, "free", "openrouter/free"}, + {promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt", ProfileID: "gpt-luna-latest"}, "gpt-luna-latest", "replacement"}, + } { + test.request.Kind = promptkit.SmokeTestTextInference + report, err := engine.SmokeTest(context.Background(), test.request) + if err != nil || report.Target.ProfileID != test.profile || report.Target.EffectiveSettings.Model != test.model { + t.Fatalf("report = %+v, %v", report, err) + } + if strings.Contains(client.requests[len(client.requests)-1].Prompt.Messages[0].Content, "private application marker") { + t.Fatal("application content used by diagnostic") + } + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 435b08c..783a056 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -242,6 +242,10 @@ Validation: targeted bundled smoke tests and existing smoke contract/integration tests. Reuse existing endpoint/auth fixtures instead of adding an adapter or duplicating the full diagnostic suite. +Completion: offline smoke tests verify reachability, authentication, both +inference cases, fallback and override selection, replacement behavior, and +unchanged inspection. Targeted smoke tests pass on Go 1.27.0. + ## Stage 6: Update Documentation And Verify The Sprint Scope: public GoDoc, current guidance/examples, and end-to-end verification.