Test bundled OpenRouter smoke diagnostics

bundled_smoke_contract_test.go

@@ -0,0 +1,127 @@
+package promptkit_test
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+
+	"go.maximumdirect.net/configkit"
+	"go.maximumdirect.net/promptkit"
+)
+
+func TestBundledSmokeHTTPDefaults(t *testing.T) {
+	t.Setenv("OPENROUTER_API_KEY", "synthetic-key")
+	var urls []string
+	var models []string
+	transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		urls = append(urls, request.URL.String())
+		body := `{"data":{"label":"member","is_free_tier":false,"usage":0}}`
+		if request.Method == http.MethodPost {
+			var payload map[string]any
+			if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
+				t.Fatal(err)
+			}
+			models = append(models, payload["model"].(string))
+			content := "PROMPTKIT_OK"
+			if _, structured := payload["response_format"]; structured {
+				content = `{"status":"ok"}`
+			}
+			encoded, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			body = string(encoded)
+		} else if strings.HasSuffix(request.URL.Path, "/key") && request.Header.Get("Authorization") != "Bearer synthetic-key" {
+			t.Errorf("authentication header missing")
+		}
+		return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}, nil
+	})
+	engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	before, _ := engine.ListProfiles(context.Background())
+	inspectionBefore, err := engine.InspectConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, kind := range []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication, promptkit.SmokeTestTextInference, promptkit.SmokeTestStructuredOutput} {
+		report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: kind, BackendID: "openrouter"})
+		if err != nil || report == nil || !report.Completed || report.Failure != "" {
+			t.Fatalf("%s report = %+v, %v", kind, report, err)
+		}
+		if kind == promptkit.SmokeTestAuthentication && (report.Facts.CredentialsAccepted == nil || !*report.Facts.CredentialsAccepted) {
+			t.Fatalf("authentication facts = %+v", report.Facts)
+		}
+		if kind == promptkit.SmokeTestTextInference || kind == promptkit.SmokeTestStructuredOutput {
+			if report.Target.ProfileID != "gpt-luna-latest" || report.Target.EffectiveSettings.Model != "~openai/gpt-luna-latest" {
+				t.Fatalf("target = %+v", report.Target)
+			}
+		}
+	}
+	wantURLs := []string{"https://openrouter.ai/api/v1", "https://openrouter.ai/api/v1/key", "https://openrouter.ai/api/v1/chat/completions", "https://openrouter.ai/api/v1/chat/completions"}
+	if !reflect.DeepEqual(urls, wantURLs) || !reflect.DeepEqual(models, []string{"~openai/gpt-luna-latest", "~openai/gpt-luna-latest"}) {
+		t.Fatalf("urls %v, models %v", urls, models)
+	}
+	after, _ := engine.ListProfiles(context.Background())
+	if !reflect.DeepEqual(before, after) {
+		t.Fatalf("inventory changed: %v %v", before, after)
+	}
+	inspectionAfter, err := engine.InspectConfiguration()
+	if err != nil {
+		t.Fatal(err)
+	}
+	diff, err := inspectionBefore.Diff(inspectionAfter)
+	if err != nil || len(diff) != 0 {
+		t.Fatalf("inspection changed: %+v, %v", diff, err)
+	}
+}
+
+func TestBundledSmokeSelectionUsesEffectiveConfiguration(t *testing.T) {
+	replacement := configkit.Input{Source: configkit.Mapping("replacement", map[string]any{"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}}})}
+	replacementClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
+	replacementEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &replacement}}, promptkit.WithLLMClient(replacementClient))
+	if err != nil {
+		t.Fatal(err)
+	}
+	replacementReport, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
+	if err != nil || replacementReport.Target.ProfileID != "gpt-luna-latest" || replacementReport.Target.EffectiveSettings.Model != "replacement" {
+		t.Fatalf("replaced default = %+v, %v", replacementReport, err)
+	}
+	replacementClient.response.Content = "unexpected"
+	mismatch, err := replacementEngine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, BackendID: "openrouter"})
+	if err != nil || mismatch == nil || !mismatch.Completed {
+		t.Fatalf("concluded mismatch = %+v, %v", mismatch, err)
+	}
+	input := configkit.Input{Source: configkit.Mapping("application", map[string]any{
+		"backends": []any{map[string]any{"id": "openrouter", "endpoint": "https://example.invalid/v1", "smoke_tests": map[string]any{"default_profile": "custom"}}},
+		"profiles": []any{map[string]any{"id": "gpt-luna-latest", "backend": "openrouter", "model": "replacement"}, map[string]any{"id": "custom", "backend": "openrouter", "model": "custom-model"}},
+		"prompts":  []any{map[string]any{"id": "prompt", "default_profile": "free", "messages": []any{map[string]any{"role": "user", "content": "private application marker"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
+	})}
+	client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "PROMPTKIT_OK"}}
+	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, promptkit.WithLLMClient(client))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, test := range []struct {
+		request        promptkit.SmokeTestRequest
+		profile, model string
+	}{
+		{promptkit.SmokeTestRequest{BackendID: "openrouter"}, "custom", "custom-model"},
+		{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt"}, "free", "openrouter/free"},
+		{promptkit.SmokeTestRequest{BackendID: "openrouter", PromptID: "prompt", ProfileID: "gpt-luna-latest"}, "gpt-luna-latest", "replacement"},
+	} {
+		test.request.Kind = promptkit.SmokeTestTextInference
+		report, err := engine.SmokeTest(context.Background(), test.request)
+		if err != nil || report.Target.ProfileID != test.profile || report.Target.EffectiveSettings.Model != test.model {
+			t.Fatalf("report = %+v, %v", report, err)
+		}
+		if strings.Contains(client.requests[len(client.requests)-1].Prompt.Messages[0].Content, "private application marker") {
+			t.Fatal("application content used by diagnostic")
+		}
+	}
+}

docs/roadmap/implementation.md

@@ -242,6 +242,10 @@ Validation: targeted bundled smoke tests and existing smoke contract/integration
 tests. Reuse existing endpoint/auth fixtures instead of adding an adapter or
 duplicating the full diagnostic suite.
 
+Completion: offline smoke tests verify reachability, authentication, both
+inference cases, fallback and override selection, replacement behavior, and
+unchanged inspection. Targeted smoke tests pass on Go 1.27.0.
+
 ## Stage 6: Update Documentation And Verify The Sprint
 
 Scope: public GoDoc, current guidance/examples, and end-to-end verification.