Add vLLM models authentication diagnostics

This commit is contained in:
2026-11-22 18:17:30 +00:00
parent ca8cf1b827
commit c3257b63f0
9 changed files with 116 additions and 9 deletions

View File

@@ -263,14 +263,15 @@ host; unlike the backend base endpoint, its path and query are allowed. Null
means fallback, while an explicit blank value is invalid. means fallback, while an explicit blank value is invalid.
Omitted/null `authentication` leaves the provider's diagnostic default in Omitted/null `authentication` leaves the provider's diagnostic default in
effect; `openrouter` defaults to `openrouter_key` and the generic provider has effect; `openrouter` defaults to `openrouter_key`, `vllm` defaults to
no default. A non-null object's optional `kind` is trimmed and case-normalized; accepted `vllm_models`, and the generic provider has no default. A non-null object's
optional `kind` is trimmed and case-normalized; accepted
values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint values are `none`, `openrouter_key`, and `vllm_models`. Its omitted/null endpoint
uses the adapter's backend-relative destination (`openrouter_key` appends uses the adapter's backend-relative destination (`openrouter_key` appends
`/key` to the effective backend base); an explicit endpoint follows the same `/key` to the effective backend base); an explicit endpoint follows the same
full-URL rules as reachability. `none` declares authentication inapplicable. full-URL rules as reachability. `none` declares authentication inapplicable.
Without a selected adapter, or for `vllm_models`, authentication diagnostics Without a selected adapter, authentication diagnostics report an unsupported
currently report an unsupported selection. An endpoint-only authentication selection. An endpoint-only authentication
block overrides the destination of a provider default; `kind: none` suppresses block overrides the destination of a provider default; `kind: none` suppresses
that default. that default.
These values are configuration only: engine construction does not contact a These values are configuration only: engine construction does not contact a

View File

@@ -73,6 +73,15 @@ not validation of the full account document. Other statuses and unrecognized
recognition allowance reports `authentication_response_limit_exceeded`, leaving recognition allowance reports `authentication_response_limit_exceeded`, leaving
credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go). credential acceptance unknown; see the [public reason](../../smoke_diagnostics.go).
The `vllm_models` adapter issues one `GET` to `/models` relative to the
configured base unless an explicit diagnostic endpoint is supplied. It sends
an available credential but does not require one. A 200 response is recognized
when it is a JSON object with `object: "list"` and an array-valued `data`;
empty arrays are accepted. A 401 means rejection; other responses do not
establish acceptance. Recognition uses the same bounded response handling as
the key probe. Acceptance does not establish that authentication is enforced or
that inference is authorized.
Diagnostic HTTP evidence describes the last request observed at Promptkit's Diagnostic HTTP evidence describes the last request observed at Promptkit's
RoundTripper boundary and any response associated with it. It includes the RoundTripper boundary and any response associated with it. It includes the
selected destination without URL user information, response status, and selected destination without URL user information, response status, and

View File

@@ -202,6 +202,8 @@ Validation: smoke selection/authentication tests and existing OpenRouter contrac
## Stage 6 — vLLM Models Authentication Adapter ## Stage 6 — vLLM Models Authentication Adapter
**Completed.**
- Implement `vllm_models` beside the OpenRouter adapter, sharing generic HTTP - Implement `vllm_models` beside the OpenRouter adapter, sharing generic HTTP
execution and bounded recognition helpers rather than duplicating transport. execution and bounded recognition helpers rather than duplicating transport.
Select it by default for `vllm` and allow explicit use on other providers. Select it by default for `vllm` and allow explicit use on other providers.

View File

@@ -73,7 +73,14 @@ func (c *OpenAICompatibleClient) ProbeAuthentication(ctx context.Context, adapte
if err != nil { if err != nil {
return result, err return result, err
} }
if recognizedOpenRouterKeyResponse(contents) { recognized := false
switch adapter {
case "openrouter_key":
recognized = recognizedOpenRouterKeyResponse(contents)
case "vllm_models":
recognized = recognizedVLLMModelsResponse(contents)
}
if recognized {
accepted := true accepted := true
result.Accepted = &accepted result.Accepted = &accepted
} else { } else {

View File

@@ -1,6 +1,10 @@
package llm package llm
import "go.maximumdirect.net/promptkit/internal/domain" import (
"net/url"
"go.maximumdirect.net/promptkit/internal/domain"
)
// EffectiveProvider resolves an omitted provider to the generic conventions. // EffectiveProvider resolves an omitted provider to the generic conventions.
func EffectiveProvider(provider string) string { func EffectiveProvider(provider string) string {
@@ -26,8 +30,11 @@ func AuthenticationAdapter(provider string, smoke *domain.BackendSmokeTestConfig
if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Kind != nil { if smoke != nil && smoke.Authentication != nil && smoke.Authentication.Kind != nil {
return *smoke.Authentication.Kind return *smoke.Authentication.Kind
} }
if provider == "openrouter" { switch provider {
case "openrouter":
return "openrouter_key" return "openrouter_key"
case "vllm":
return "vllm_models"
} }
return "" return ""
} }
@@ -40,8 +47,13 @@ func AuthenticationEndpoint(adapter, base string, smoke *domain.BackendSmokeTest
if adapter == "openrouter_key" { if adapter == "openrouter_key" {
return OpenRouterKeyEndpoint(base) return OpenRouterKeyEndpoint(base)
} }
if adapter == "vllm_models" {
return url.JoinPath(base, "models")
}
return base, nil return base, nil
} }
// SupportedAuthenticationAdapter identifies implemented authentication probes. // SupportedAuthenticationAdapter identifies implemented authentication probes.
func SupportedAuthenticationAdapter(adapter string) bool { return adapter == "openrouter_key" } func SupportedAuthenticationAdapter(adapter string) bool {
return adapter == "openrouter_key" || adapter == "vllm_models"
}

View File

@@ -7,7 +7,7 @@ import (
) )
func TestAuthenticationSelection(t *testing.T) { func TestAuthenticationSelection(t *testing.T) {
none, explicit, endpoint := "none", "openrouter_key", "https://diagnostics.example/probe" none, explicit, models, endpoint := "none", "openrouter_key", "vllm_models", "https://diagnostics.example/probe"
for _, tc := range []struct { for _, tc := range []struct {
name, provider, adapter, destination string name, provider, adapter, destination string
metadata *domain.BackendSmokeTestConfig metadata *domain.BackendSmokeTestConfig
@@ -18,6 +18,10 @@ func TestAuthenticationSelection(t *testing.T) {
{name: "explicit adapter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &explicit}}, adapter: "openrouter_key", destination: "https://example.test/v1/key"}, {name: "explicit adapter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &explicit}}, adapter: "openrouter_key", destination: "https://example.test/v1/key"},
{name: "disabled", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"}, {name: "disabled", provider: "openrouter", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
{name: "generic unsupported", destination: "https://example.test/v1"}, {name: "generic unsupported", destination: "https://example.test/v1"},
{name: "vllm default", provider: "vllm", adapter: "vllm_models", destination: "https://example.test/v1/models"},
{name: "vllm override", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Endpoint: &endpoint}}, adapter: "vllm_models", destination: endpoint},
{name: "explicit vllm on generic", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &models}}, adapter: "vllm_models", destination: "https://example.test/v1/models"},
{name: "disabled vllm", provider: "vllm", metadata: &domain.BackendSmokeTestConfig{Authentication: &domain.AuthenticationProbeConfig{Kind: &none}}, adapter: "none", destination: "https://example.test/v1"},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
kind := AuthenticationAdapter(tc.provider, tc.metadata) kind := AuthenticationAdapter(tc.provider, tc.metadata)

View File

@@ -0,0 +1,16 @@
package llm
import "encoding/json"
func recognizedVLLMModelsResponse(contents []byte) bool {
var envelope map[string]json.RawMessage
if json.Unmarshal(contents, &envelope) != nil || envelope == nil {
return false
}
var object string
if json.Unmarshal(envelope["object"], &object) != nil || object != "list" {
return false
}
var data []json.RawMessage
return json.Unmarshal(envelope["data"], &data) == nil && data != nil
}

View File

@@ -0,0 +1,51 @@
package llm
import (
"context"
"io"
"net/http"
"strings"
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
)
func TestVLLMModelsAuthentication(t *testing.T) {
for _, tc := range []struct {
name, body, key string
status int
accepted *bool
unrecognized, overflow bool
}{
{name: "empty list", status: 200, body: `{"object":"list","data":[]}`, accepted: boolPtr(true)},
{name: "nonempty list", status: 200, body: `{"object":"list","data":[{"id":"model"}]}`, key: "key", accepted: boolPtr(true)},
{name: "wrong object", status: 200, body: `{"object":"model","data":[]}`, unrecognized: true},
{name: "null data", status: 200, body: `{"object":"list","data":null}`, unrecognized: true},
{name: "object data", status: 200, body: `{"object":"list","data":{}}`, unrecognized: true},
{name: "invalid framing", status: 200, body: `{"object":"list","data":[]} {}`, unrecognized: true},
{name: "overflow", status: 200, body: strings.Repeat(" ", maxAuthenticationResponseBytes+1), overflow: true},
{name: "unauthorized", status: 401, accepted: boolPtr(false)},
{name: "other status", status: 503},
} {
t.Run(tc.name, func(t *testing.T) {
var credential string
client := NewOpenAICompatibleClient(OpenAICompatibleConfig{HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
credential = req.Header.Get("Authorization")
return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil
})}})
got, err := client.ProbeAuthentication(context.Background(), "vllm_models", "https://example.invalid/v1/models", 0, domain.ExecutionTarget{APIKey: tc.key})
if err != nil || got.Unrecognized != tc.unrecognized || got.Overflow != tc.overflow || (got.Accepted == nil) != (tc.accepted == nil) || (got.Accepted != nil && *got.Accepted != *tc.accepted) {
t.Fatalf("result=%+v err=%v", got, err)
}
want := ""
if tc.key != "" {
want = "Bearer " + tc.key
}
if credential != want {
t.Fatalf("authorization=%q", credential)
}
})
}
}
func boolPtr(value bool) *bool { return &value }

View File

@@ -95,6 +95,11 @@ func TestAuthenticationProbeApplicabilityAndHTTPAvailability(t *testing.T) {
backend: backendWithAuthenticationKind("openrouter_key"), backend: backendWithAuthenticationKind("openrouter_key"),
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable, reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
}, },
{
name: "vllm HTTP adapter unavailable",
backend: domain.Backend{Provider: "vllm"},
reason: domain.SmokeTestReasonHTTPDiagnosticsUnavailable,
},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
probe := authenticationProbe{} probe := authenticationProbe{}