Files
notarius/internal/framework/llm/promptkit_sources_integration_test.go

271 lines
10 KiB
Go

package llm
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPromptKitOperatorDocumentComposesForInspectionAndExecution(t *testing.T) {
for _, name := range []string{
"NOTARIUS_TEST_BACKEND_KEY",
"NOTARIUS_TEST_PROFILE_KEY",
"NOTARIUS_TEST_CATALOG_KEY",
} {
t.Setenv(name, "")
}
operatorDir := t.TempDir()
profilePath := filepath.Join(operatorDir, "operator.yaml")
if err := os.WriteFile(filepath.Join(operatorDir, "inherited.yaml"), []byte(`profiles:
inherited-base:
backend: custom-backend
model: imported-model
temperature: 0.7
service_tier: flex
extra_params:
inherited: retained
nested:
inherited: true
`), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(profilePath, []byte(`configkit:
imports:
- path: inherited.yaml
backends:
custom-backend:
endpoint: http://127.0.0.1:1/v1
api_key_env: NOTARIUS_TEST_BACKEND_KEY
concurrency_limit: 1
queue_capacity: 0
openrouter:
api_key_env: NOTARIUS_TEST_CATALOG_KEY
profiles:
custom-profile:
extends: inherited-base
model: operator-model
temperature: 0
api_key_required: false
service_tier: null
extra_params: null
empty-map-profile:
extends: inherited-base
extra_params: {}
api_key_required: false
profile-key-profile:
backend: custom-backend
model: profile-key-model
api_key_env: NOTARIUS_TEST_PROFILE_KEY
api_key_required: false
required-profile:
extends: inherited-base
api_key_env: NOTARIUS_TEST_PROFILE_KEY
api_key_required: true
catalog-profile:
backend: openrouter
model: openai/gpt-5.6-luna
api_key_required: false
prompts:
adapter.test:
v1:
output:
repair_attempts: 1
`), 0o600); err != nil {
t.Fatal(err)
}
assets := newTestPromptKitAssets(t)
fake := &fakePromptKitLLM{}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: assets,
ProfileFile: profilePath,
EngineOptions: []promptkit.Option{
promptkit.WithLLMClient(fake),
},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v", err)
}
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
Source: PromptKitProfileSourceConfig{ProfileFile: profilePath},
Assets: assets,
})
if err != nil {
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
}
for _, test := range []struct {
id string
backend string
model string
keyEnv string
required bool
}{
{id: "custom-profile", backend: "custom-backend", model: "operator-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
{id: "inherited-base", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
{id: "empty-map-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
{id: "profile-key-profile", backend: "custom-backend", model: "profile-key-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY"},
{id: "required-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY", required: true},
{id: "catalog-profile", backend: promptkit.BackendOpenRouter, model: "openai/gpt-5.6-luna", keyEnv: "NOTARIUS_TEST_CATALOG_KEY"},
} {
inspection, err := inspector.InspectProfile(context.Background(), test.id)
if err != nil {
t.Fatalf("InspectProfile(%q) error = %v", test.id, err)
}
if inspection.ProfileID != test.id || inspection.BackendID != test.backend || inspection.Model != test.model ||
inspection.CredentialEnvironment != test.keyEnv || inspection.CredentialRequired != test.required {
t.Fatalf("InspectProfile(%q) = %#v", test.id, inspection)
}
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
ProfileID: "custom-profile",
SessionID: "source-composition-test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() with an unset optional credential error = %v", err)
}
if response.Debug == nil || response.Debug.Prompt == nil {
t.Fatalf("completion debug = %#v, want captured prompt", response.Debug)
}
debug := response.Debug.Prompt
if debug.SelectedBackendID != "custom-backend" || debug.SelectedProfileID != "custom-profile" {
t.Fatalf("runtime selection = backend %q, profile %q", debug.SelectedBackendID, debug.SelectedProfileID)
}
if got := debug.EffectiveModelParams["model"]; got != "operator-model" {
t.Fatalf("runtime model = %#v, want operator-model", got)
}
if got, ok := debug.EffectiveModelParams["temperature"].(float64); !ok || got != 0 {
t.Fatalf("runtime temperature = %#v, want explicit zero", debug.EffectiveModelParams["temperature"])
}
if got := debug.EffectiveModelParams["service_tier"]; got == "flex" {
t.Fatalf("explicit null retained inherited service tier: %#v", got)
}
if got, ok := debug.EffectiveModelParams["extra_params"]; ok && got != nil {
t.Fatalf("explicit null retained inherited provider map: %#v", got)
}
if got := debug.OutputContract["repair_attempts"]; got != float64(1) {
t.Fatalf("overlaid output repair attempts = %#v, want 1", got)
}
if len(debug.Messages) == 0 || !strings.Contains(debug.Messages[0].Content, "Transcript:") {
t.Fatalf("rendered prompt messages = %#v", debug.Messages)
}
if got := atomic.LoadInt32(&fake.calls); got != 1 {
t.Fatalf("provider calls after optional-credential execution = %d, want 1", got)
}
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
ProfileID: "required-profile",
SessionID: "source-composition-test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err == nil || !strings.Contains(err.Error(), "api_key_env points to an unset environment variable") {
t.Fatalf("CompleteStructured() with required unset credential error = %v, want native missing-environment failure", err)
}
if got := atomic.LoadInt32(&fake.calls); got != 1 {
t.Fatalf("provider calls after required-credential failure = %d, want 1", got)
}
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
ProfileID: "empty-map-profile",
SessionID: "source-composition-test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if err != nil {
t.Fatalf("CompleteStructured() with empty overlay map error = %v", err)
}
if got := fake.lastRequest().Target.ExtraParams["inherited"]; got != "retained" {
t.Fatalf("empty provider-map overlay result = %#v, want inherited entry", fake.lastRequest().Target.ExtraParams)
}
}
func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) {
t.Run("directory group conflicts", func(t *testing.T) {
directory := t.TempDir()
for name, model := range map[string]string{"first.yaml": "first-model", "second.yaml": "second-model"} {
content := "profiles:\n duplicate:\n endpoint: http://127.0.0.1:1/v1\n model: " + model + "\n"
if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
Source: PromptKitProfileSourceConfig{ProfileDir: directory},
})
if err == nil {
t.Fatal("NewPromptKitProfileInspector() error = nil, want same-group conflict")
}
})
t.Run("later group repairs an invalid lower field and overlays supplied values", func(t *testing.T) {
badEndpoint := "not an endpoint"
goodEndpoint := "http://127.0.0.1:1/v1"
lowerModel := "lower-model"
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower source", map[string]promptkit.ProfileConfig{
"layered-profile": {Endpoint: &badEndpoint, Model: &lowerModel},
})}}},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("upper source", map[string]promptkit.ProfileConfig{
"layered-profile": {Endpoint: &goodEndpoint},
})}}},
}}
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
if err != nil {
t.Fatalf("NewPromptKitProfileInspector() error = %v, want repaired final layer", err)
}
inspection, err := inspector.InspectProfile(context.Background(), "layered-profile")
if err != nil || inspection.Model != lowerModel {
t.Fatalf("InspectProfile() = (%#v, %v), want lower model retained", inspection, err)
}
})
t.Run("empty and missing selected directories", func(t *testing.T) {
if _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
Source: PromptKitProfileSourceConfig{ProfileDir: filepath.Join(t.TempDir(), "missing")},
}); err == nil {
t.Fatal("missing selected directory was accepted")
}
empty := t.TempDir()
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
Source: PromptKitProfileSourceConfig{ProfileDir: empty},
})
if err != nil {
t.Fatalf("empty selected directory error = %v, want no contribution", err)
}
if _, err := inspector.InspectProfile(context.Background(), "not-present"); err == nil {
t.Fatal("InspectProfile() in empty directory error = nil, want unknown profile")
}
})
t.Run("unused invalid final profile fails construction", func(t *testing.T) {
badEndpoint := "not an endpoint"
model := "unused-model"
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("unused invalid profile", map[string]promptkit.ProfileConfig{
"unused-profile": {Endpoint: &badEndpoint, Model: &model},
}),
}}}}}
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("NewPromptKitProfileInspector() error = %v, want invalid final configuration", err)
}
})
}