Capture provider policy in backend execution targets
This commit is contained in:
@@ -265,7 +265,7 @@ func fromProfileResource(v *configuration.ProfileResource) *ProfileInspection {
|
||||
}
|
||||
|
||||
func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
||||
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||
out := &BackendInspection{BackendID: v.ID, Endpoint: v.Endpoint, Provider: copyStringPtr(v.Provider), ZeroCost: copyBoolPtr(v.ZeroCost), APIKeyEnv: copyStringPtr(v.APIKeyEnv), ConcurrencyLimit: copyIntPtr(v.ConcurrencyLimit), QueueCapacity: copyIntPtr(v.QueueCapacity)}
|
||||
if v.ExtraParams != nil {
|
||||
data := copyAnyMap(v.ExtraParams)
|
||||
out.ExtraParams = &data
|
||||
|
||||
@@ -151,6 +151,8 @@ Validation: focused configuration/settings/source-helper tests.
|
||||
|
||||
## Stage 4 — Runtime Capture, Freezing, And Backend Inspection
|
||||
|
||||
**Completed.**
|
||||
|
||||
Touch `internal/configuration/targets.go`, `ownership.go`, `resources.go`, domain
|
||||
backend/target values and clones, and `bundled/openrouter.yaml` under that package.
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
backends:
|
||||
- id: openrouter
|
||||
provider: openrouter
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
smoke_tests:
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := state.GetBackend("openrouter")
|
||||
if err != nil || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost || backend.Endpoint != "https://openrouter.ai/api/v1" || backend.APIKeyEnv != "OPENROUTER_API_KEY" || backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil || *backend.SmokeTests.DefaultProfile != "gpt-luna-latest" || backend.SmokeTests.Authentication == nil || backend.SmokeTests.Authentication.Kind == nil || *backend.SmokeTests.Authentication.Kind != "openrouter_key" {
|
||||
t.Fatalf("backend = %+v, %v", backend, err)
|
||||
}
|
||||
if !reflect.DeepEqual(state.backendIDs, []string{"openrouter"}) || len(state.promptIDs) != 0 || len(state.collections) != 0 {
|
||||
@@ -40,6 +40,7 @@ func TestBundledOpenRouterUnderlay(t *testing.T) {
|
||||
t.Fatalf("profile %s = %+v, %v", id, profile, err)
|
||||
}
|
||||
want := defaults.ExecutionTargetDefault()
|
||||
want.Provider = "openrouter"
|
||||
want.Model, want.BackendID, want.Endpoint, want.APIKeyEnv = model, "openrouter", "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"
|
||||
if tier, ok := strings.CutPrefix(id, "auto-"); ok {
|
||||
want.ExtraParams = map[string]any{"plugins": []any{map[string]any{"id": "auto-router", "cost_tier": tier}}}
|
||||
|
||||
@@ -59,6 +59,35 @@ func TestTargetResolutionNormalizesNestedValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderPolicyCaptureAndInspection(t *testing.T) {
|
||||
state, err := load(mapping("provider policy", map[string]any{
|
||||
"collections": map[string]any{"team": map[string]any{
|
||||
"backends": []any{map[string]any{"id": "renamed", "endpoint": "https://example.invalid/v1", "provider": "OPENROUTER", "zero_cost": true}, map[string]any{"id": "child", "extends": map[string]any{"id": "renamed"}, "zero_cost": false}},
|
||||
"profiles": []any{map[string]any{"id": "selected", "backend": "team/child", "endpoint": "https://other.invalid/v1", "model": "m"}},
|
||||
}},
|
||||
"profiles": []any{map[string]any{"id": "generic", "endpoint": "https://example.invalid/v1", "model": "m"}},
|
||||
}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backend, err := state.GetBackend("team/child")
|
||||
if err != nil || backend.Provider != "openrouter" || backend.ZeroCost {
|
||||
t.Fatalf("backend: %+v, %v", backend, err)
|
||||
}
|
||||
profile, err := state.GetProfile(context.Background(), "team/selected")
|
||||
if err != nil || profile.ResolvedTarget.Provider != "openrouter" || profile.ResolvedTarget.ZeroCost || profile.ResolvedTarget.Endpoint != "https://other.invalid/v1" {
|
||||
t.Fatalf("profile: %+v, %v", profile, err)
|
||||
}
|
||||
profile, err = state.GetProfile(context.Background(), "generic")
|
||||
if err != nil || profile.ResolvedTarget.Provider != "openai-compatible" || profile.ResolvedTarget.ZeroCost {
|
||||
t.Fatalf("generic: %+v, %v", profile, err)
|
||||
}
|
||||
resource, err := state.InspectBackend(context.Background(), "team/child")
|
||||
if err != nil || resource.Provider == nil || *resource.Provider != "openrouter" || resource.ZeroCost == nil || *resource.ZeroCost {
|
||||
t.Fatalf("inspection: %+v, %v", resource, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptResolutionNormalizesOutputAndRejectsDuplicateNames(t *testing.T) {
|
||||
definition := map[string]any{
|
||||
"id": "example", "inputs": []any{map[string]any{"name": " document "}},
|
||||
|
||||
@@ -21,6 +21,8 @@ func copyProviderMap(value map[string]any) map[string]any {
|
||||
}
|
||||
|
||||
func ownBackendSettings(in settings.BackendSettings) settings.BackendSettings {
|
||||
in.Provider = copyPointer(in.Provider)
|
||||
in.ZeroCost = copyPointer(in.ZeroCost)
|
||||
in.Endpoint = copyPointer(in.Endpoint)
|
||||
in.APIKeyEnv = copyPointer(in.APIKeyEnv)
|
||||
in.ConcurrencyLimit = copyPointer(in.ConcurrencyLimit)
|
||||
|
||||
@@ -26,6 +26,8 @@ type ProfileResource struct {
|
||||
// Its fields borrow immutable configuration captured by the adapter.
|
||||
type BackendResource struct {
|
||||
ID, Endpoint string
|
||||
Provider *string
|
||||
ZeroCost *bool
|
||||
APIKeyEnv *string
|
||||
ConcurrencyLimit, QueueCapacity *int
|
||||
ExtraParams map[string]any
|
||||
@@ -131,7 +133,7 @@ func (state *loaded) InspectBackend(ctx context.Context, id string) (*BackendRes
|
||||
return nil, fmt.Errorf("%w: %q", backend.ErrBackendNotFound, id)
|
||||
}
|
||||
return &BackendResource{
|
||||
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv,
|
||||
ID: id, Endpoint: valueOrZero(config.Endpoint), APIKeyEnv: config.APIKeyEnv, Provider: config.Provider, ZeroCost: config.ZeroCost,
|
||||
ConcurrencyLimit: config.ConcurrencyLimit, QueueCapacity: config.QueueCapacity,
|
||||
ExtraParams: config.ExtraParams, SmokeTests: captureBackend(id, config).SmokeTests,
|
||||
}, nil
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||
"go.maximumdirect.net/promptkit/internal/llm"
|
||||
profilepkg "go.maximumdirect.net/promptkit/internal/profile"
|
||||
"go.maximumdirect.net/promptkit/internal/settings"
|
||||
)
|
||||
@@ -36,7 +37,7 @@ func (state *loaded) buildExecutionProfiles(config *settings.Settings) error {
|
||||
}
|
||||
|
||||
func captureBackend(id string, config backendConfig) domain.Backend {
|
||||
definition := domain.Backend{ID: id, ExtraParams: config.ExtraParams}
|
||||
definition := domain.Backend{ID: id, Provider: llm.EffectiveProvider(valueOrZero(config.Provider)), ZeroCost: valueOrZero(config.ZeroCost), ExtraParams: config.ExtraParams}
|
||||
if config.SmokeTests != nil {
|
||||
smokeTests := &domain.BackendSmokeTestConfig{DefaultProfile: config.SmokeTests.DefaultProfile}
|
||||
if config.SmokeTests.Reachability != nil {
|
||||
@@ -67,6 +68,7 @@ func captureBackend(id string, config backendConfig) domain.Backend {
|
||||
|
||||
func captureExecutionProfile(id string, config profileConfig, backends map[string]domain.Backend) (domain.ExecutionProfile, error) {
|
||||
target := defaults.ExecutionTargetDefault()
|
||||
target.Provider = llm.EffectiveProvider("")
|
||||
if config.Backend != nil && *config.Backend != "" {
|
||||
// These defaults are capture-owned and immutable. Public and runtime
|
||||
// lookups still detach mutable values; profile overlays copy before merging.
|
||||
@@ -75,6 +77,8 @@ func captureExecutionProfile(id string, config profileConfig, backends map[strin
|
||||
return domain.ExecutionProfile{}, fmt.Errorf("profile %q backend: %w: %q", id, backend.ErrBackendNotFound, *config.Backend)
|
||||
}
|
||||
target.BackendID = selected.ID
|
||||
target.Provider = selected.Provider
|
||||
target.ZeroCost = selected.ZeroCost
|
||||
target.Endpoint = selected.Endpoint
|
||||
target.APIKeyEnv = selected.APIKeyEnv
|
||||
target.ExtraParams = selected.ExtraParams
|
||||
|
||||
@@ -188,6 +188,8 @@ type PromptMessageTemplate struct {
|
||||
// diagnostic metadata.
|
||||
type Backend struct {
|
||||
ID string
|
||||
Provider string
|
||||
ZeroCost bool
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
@@ -236,6 +238,8 @@ type ExecutionTargetPresence struct {
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings for a run.
|
||||
type ExecutionTarget struct {
|
||||
Provider string `yaml:"-" json:"-"`
|
||||
ZeroCost bool `yaml:"-" json:"-"`
|
||||
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
package llm
|
||||
|
||||
// EffectiveProvider resolves an omitted provider to the generic conventions.
|
||||
func EffectiveProvider(provider string) string {
|
||||
if provider == "" {
|
||||
return "openai-compatible"
|
||||
}
|
||||
return provider
|
||||
}
|
||||
|
||||
// SupportedProvider reports whether a normalized provider selection is supported.
|
||||
// An empty selection uses the generic OpenAI-compatible conventions.
|
||||
func SupportedProvider(provider string) bool {
|
||||
|
||||
@@ -95,6 +95,8 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
||||
}
|
||||
admitter := &fakeRunAdmitter{}
|
||||
profile := defaultExecutionProfile()
|
||||
profile.ResolvedTarget.Provider = "vllm"
|
||||
profile.ResolvedTarget.ZeroCost = true
|
||||
profile.ResolvedTarget.ExtraParams = map[string]any{
|
||||
"metadata": map[string]any{"source": "original"},
|
||||
}
|
||||
@@ -144,9 +146,11 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
|
||||
}
|
||||
|
||||
renderer.rendered.Messages[0].Content = "source mutation"
|
||||
profile.ResolvedTarget.Provider = "openrouter"
|
||||
profile.ResolvedTarget.ZeroCost = false
|
||||
|
||||
second := prepared.Details()
|
||||
if second != first || second.Messages[0].Content != "original message" ||
|
||||
if second != first || second.Messages[0].Content != "original message" || second.EffectiveModelParams.Provider != "vllm" || !second.EffectiveModelParams.ZeroCost ||
|
||||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||
t.Fatalf("borrowed details did not preserve the snapshot: %#v", second)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T) {
|
||||
profile := &domain.ExecutionProfile{ID: "partial", ResolvedTarget: &domain.ExecutionTarget{
|
||||
Provider: "openrouter", ZeroCost: true,
|
||||
TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds,
|
||||
Temperature: 0, ExtraParams: map[string]any{"configured": true},
|
||||
}, TargetPresence: domain.ExecutionTargetPresence{Temperature: true}}
|
||||
@@ -35,7 +36,7 @@ func TestPreparationCompletesResolvedTargetThroughRequestOverrides(t *testing.T)
|
||||
t.Fatalf("request did not complete partial target: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Endpoint != "https://runtime.example/v1" ||
|
||||
prepared.EffectiveModelParams.Model != "runtime-model" || !prepared.TargetPresence.Temperature ||
|
||||
prepared.EffectiveModelParams.Model != "runtime-model" || prepared.EffectiveModelParams.Provider != "openrouter" || !prepared.EffectiveModelParams.ZeroCost || !prepared.TargetPresence.Temperature ||
|
||||
prepared.EffectiveModelParams.ExtraParams["request"] != true {
|
||||
t.Fatalf("request overrides were not applied: %+v", prepared)
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ type ProfileInspection struct {
|
||||
// explicit zero values and empty collections in JSON and YAML.
|
||||
type BackendInspection struct {
|
||||
BackendID string `json:"id" yaml:"id"`
|
||||
Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"`
|
||||
ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"`
|
||||
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
||||
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
|
||||
ConcurrencyLimit *int `json:"concurrency_limit,omitempty" yaml:"concurrency_limit,omitempty"`
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Destination: configkit.Keys("promptkit"), Source: configkit.Mapping("resources", map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
|
||||
"backends": []any{map[string]any{"id": "local", "provider": " VLLM ", "zero_cost": false, "endpoint": "https://example.test/v1", "api_key_env": "BACKEND_KEY", "concurrency_limit": 2, "extra_params": map[string]any{"null": nil, "array": []any{nil}}, "smoke_tests": map[string]any{"reachability": map[string]any{}, "authentication": map[string]any{"kind": "none"}}}, map[string]any{"id": "plain", "endpoint": "https://plain.test/v1"}},
|
||||
"profiles": []any{map[string]any{"id": "parent", "backend": "local", "model": "model"}, map[string]any{"id": "profile", "extends": map[string]any{"id": "parent"}, "temperature": 0, "top_p": 0, "max_tokens": 0, "timeout_seconds": 0, "service_tier": "", "reasoning_effort": "", "api_key_env": "", "api_key_required": false, "endpoint": "", "extra_params": map[string]any{}}, map[string]any{"id": "unset", "extra_params": nil}},
|
||||
"prompts": []any{map[string]any{"id": "prompt", "inputs": []any{map[string]any{"name": "a", "required": false, "description": ""}, map[string]any{"name": "b"}}, "messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}}, map[string]any{"id": "empty-inputs", "inputs": []any{}, "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}, map[string]any{"id": "omitted-inputs", "messages": []any{map[string]any{"role": "user", "content": "x"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}}},
|
||||
})}}})
|
||||
@@ -77,7 +77,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
b := decode(backend)
|
||||
if b["id"] != "local" || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
|
||||
if b["id"] != "local" || b["provider"] != "vllm" || b["zero_cost"] != false || b["api_key_env"] != "BACKEND_KEY" || fmt.Sprint(b["concurrency_limit"]) != "2" || fmt.Sprint(b["queue_capacity"]) != "1024" {
|
||||
t.Errorf("backend = %#v", b)
|
||||
}
|
||||
params := b["extra_params"].(map[string]any)
|
||||
@@ -92,7 +92,7 @@ func TestResourceInspectionGenericEncoding(t *testing.T) {
|
||||
t.Errorf("reachability = %#v", reach)
|
||||
}
|
||||
absent(b, "extends")
|
||||
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit")
|
||||
absent(decode(plain), "smoke_tests", "queue_capacity", "concurrency_limit", "provider", "zero_cost")
|
||||
p := decode(profile)
|
||||
if p["id"] != "profile" || p["backend"] != "local" || p["model"] != "model" || p["endpoint"] != "" || p["api_key_env"] != "" || p["service_tier"] != "" || p["reasoning_effort"] != "" || p["api_key_required"] != false || fmt.Sprint(p["temperature"]) != "0" || fmt.Sprint(p["top_p"]) != "0" || fmt.Sprint(p["max_tokens"]) != "0" || fmt.Sprint(p["timeout_seconds"]) != "0" || len(p["extra_params"].(map[string]any)) != 0 {
|
||||
t.Errorf("profile = %#v", p)
|
||||
|
||||
Reference in New Issue
Block a user