2026-10-04 12:46:23 +00:00
|
|
|
package llm
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"errors"
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync/atomic"
|
|
|
|
|
"testing"
|
2026-10-04 13:15:40 +00:00
|
|
|
"testing/fstest"
|
2026-10-04 12:46:23 +00:00
|
|
|
|
|
|
|
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func TestPromptKitOperatorDocumentComposesForInspectionAndExecution(t *testing.T) {
|
|
|
|
|
for _, name := range []string{
|
|
|
|
|
"NOTARIUS_TEST_BACKEND_KEY",
|
|
|
|
|
"NOTARIUS_TEST_PROFILE_KEY",
|
|
|
|
|
"NOTARIUS_TEST_CATALOG_KEY",
|
|
|
|
|
} {
|
|
|
|
|
t.Setenv(name, "")
|
|
|
|
|
}
|
|
|
|
|
operatorDir := t.TempDir()
|
|
|
|
|
profilePath := filepath.Join(operatorDir, "operator.yaml")
|
|
|
|
|
if err := os.WriteFile(filepath.Join(operatorDir, "inherited.yaml"), []byte(`profiles:
|
|
|
|
|
inherited-base:
|
|
|
|
|
backend: custom-backend
|
|
|
|
|
model: imported-model
|
|
|
|
|
temperature: 0.7
|
|
|
|
|
service_tier: flex
|
|
|
|
|
extra_params:
|
|
|
|
|
inherited: retained
|
|
|
|
|
nested:
|
|
|
|
|
inherited: true
|
|
|
|
|
`), 0o600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if err := os.WriteFile(profilePath, []byte(`configkit:
|
|
|
|
|
imports:
|
|
|
|
|
- path: inherited.yaml
|
|
|
|
|
backends:
|
|
|
|
|
custom-backend:
|
|
|
|
|
endpoint: http://127.0.0.1:1/v1
|
|
|
|
|
api_key_env: NOTARIUS_TEST_BACKEND_KEY
|
|
|
|
|
concurrency_limit: 1
|
|
|
|
|
queue_capacity: 0
|
|
|
|
|
openrouter:
|
|
|
|
|
api_key_env: NOTARIUS_TEST_CATALOG_KEY
|
|
|
|
|
profiles:
|
|
|
|
|
custom-profile:
|
|
|
|
|
extends: inherited-base
|
|
|
|
|
model: operator-model
|
|
|
|
|
temperature: 0
|
|
|
|
|
api_key_required: false
|
|
|
|
|
service_tier: null
|
|
|
|
|
extra_params: null
|
|
|
|
|
empty-map-profile:
|
|
|
|
|
extends: inherited-base
|
|
|
|
|
extra_params: {}
|
|
|
|
|
api_key_required: false
|
|
|
|
|
profile-key-profile:
|
|
|
|
|
backend: custom-backend
|
|
|
|
|
model: profile-key-model
|
|
|
|
|
api_key_env: NOTARIUS_TEST_PROFILE_KEY
|
|
|
|
|
api_key_required: false
|
|
|
|
|
required-profile:
|
|
|
|
|
extends: inherited-base
|
|
|
|
|
api_key_env: NOTARIUS_TEST_PROFILE_KEY
|
|
|
|
|
api_key_required: true
|
|
|
|
|
catalog-profile:
|
|
|
|
|
backend: openrouter
|
|
|
|
|
model: openai/gpt-5.6-luna
|
|
|
|
|
api_key_required: false
|
|
|
|
|
prompts:
|
|
|
|
|
adapter.test:
|
|
|
|
|
v1:
|
|
|
|
|
output:
|
|
|
|
|
repair_attempts: 1
|
|
|
|
|
`), 0o600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
assets := newTestPromptKitAssets(t)
|
|
|
|
|
fake := &fakePromptKitLLM{}
|
|
|
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
|
|
|
Assets: assets,
|
|
|
|
|
ProfileFile: profilePath,
|
|
|
|
|
EngineOptions: []promptkit.Option{
|
|
|
|
|
promptkit.WithLLMClient(fake),
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
|
|
|
|
}
|
|
|
|
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
|
|
|
|
Source: PromptKitProfileSourceConfig{ProfileFile: profilePath},
|
|
|
|
|
Assets: assets,
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, test := range []struct {
|
|
|
|
|
id string
|
|
|
|
|
backend string
|
|
|
|
|
model string
|
|
|
|
|
keyEnv string
|
|
|
|
|
required bool
|
|
|
|
|
}{
|
|
|
|
|
{id: "custom-profile", backend: "custom-backend", model: "operator-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
|
|
|
|
{id: "inherited-base", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
|
|
|
|
{id: "empty-map-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"},
|
|
|
|
|
{id: "profile-key-profile", backend: "custom-backend", model: "profile-key-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY"},
|
|
|
|
|
{id: "required-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY", required: true},
|
|
|
|
|
{id: "catalog-profile", backend: promptkit.BackendOpenRouter, model: "openai/gpt-5.6-luna", keyEnv: "NOTARIUS_TEST_CATALOG_KEY"},
|
|
|
|
|
} {
|
|
|
|
|
inspection, err := inspector.InspectProfile(context.Background(), test.id)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("InspectProfile(%q) error = %v", test.id, err)
|
|
|
|
|
}
|
|
|
|
|
if inspection.ProfileID != test.id || inspection.BackendID != test.backend || inspection.Model != test.model ||
|
|
|
|
|
inspection.CredentialEnvironment != test.keyEnv || inspection.CredentialRequired != test.required {
|
|
|
|
|
t.Fatalf("InspectProfile(%q) = %#v", test.id, inspection)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var out map[string]any
|
|
|
|
|
response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
|
|
|
PromptID: "adapter.test",
|
|
|
|
|
ProfileID: "custom-profile",
|
|
|
|
|
SessionID: "source-composition-test",
|
|
|
|
|
Inputs: contracts.LLMInputSet{
|
|
|
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
|
|
|
},
|
|
|
|
|
}, &out)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("CompleteStructured() with an unset optional credential error = %v", err)
|
|
|
|
|
}
|
|
|
|
|
if response.Debug == nil || response.Debug.Prompt == nil {
|
|
|
|
|
t.Fatalf("completion debug = %#v, want captured prompt", response.Debug)
|
|
|
|
|
}
|
|
|
|
|
debug := response.Debug.Prompt
|
|
|
|
|
if debug.SelectedBackendID != "custom-backend" || debug.SelectedProfileID != "custom-profile" {
|
|
|
|
|
t.Fatalf("runtime selection = backend %q, profile %q", debug.SelectedBackendID, debug.SelectedProfileID)
|
|
|
|
|
}
|
|
|
|
|
if got := debug.EffectiveModelParams["model"]; got != "operator-model" {
|
|
|
|
|
t.Fatalf("runtime model = %#v, want operator-model", got)
|
|
|
|
|
}
|
|
|
|
|
if got, ok := debug.EffectiveModelParams["temperature"].(float64); !ok || got != 0 {
|
|
|
|
|
t.Fatalf("runtime temperature = %#v, want explicit zero", debug.EffectiveModelParams["temperature"])
|
|
|
|
|
}
|
|
|
|
|
if got := debug.EffectiveModelParams["service_tier"]; got == "flex" {
|
|
|
|
|
t.Fatalf("explicit null retained inherited service tier: %#v", got)
|
|
|
|
|
}
|
|
|
|
|
if got, ok := debug.EffectiveModelParams["extra_params"]; ok && got != nil {
|
|
|
|
|
t.Fatalf("explicit null retained inherited provider map: %#v", got)
|
|
|
|
|
}
|
|
|
|
|
if got := debug.OutputContract["repair_attempts"]; got != float64(1) {
|
|
|
|
|
t.Fatalf("overlaid output repair attempts = %#v, want 1", got)
|
|
|
|
|
}
|
|
|
|
|
if len(debug.Messages) == 0 || !strings.Contains(debug.Messages[0].Content, "Transcript:") {
|
|
|
|
|
t.Fatalf("rendered prompt messages = %#v", debug.Messages)
|
|
|
|
|
}
|
|
|
|
|
if got := atomic.LoadInt32(&fake.calls); got != 1 {
|
|
|
|
|
t.Fatalf("provider calls after optional-credential execution = %d, want 1", got)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
|
|
|
PromptID: "adapter.test",
|
|
|
|
|
ProfileID: "required-profile",
|
|
|
|
|
SessionID: "source-composition-test",
|
|
|
|
|
Inputs: contracts.LLMInputSet{
|
|
|
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
|
|
|
},
|
|
|
|
|
}, &out)
|
|
|
|
|
if err == nil || !strings.Contains(err.Error(), "api_key_env points to an unset environment variable") {
|
|
|
|
|
t.Fatalf("CompleteStructured() with required unset credential error = %v, want native missing-environment failure", err)
|
|
|
|
|
}
|
|
|
|
|
if got := atomic.LoadInt32(&fake.calls); got != 1 {
|
|
|
|
|
t.Fatalf("provider calls after required-credential failure = %d, want 1", got)
|
|
|
|
|
}
|
|
|
|
|
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
|
|
|
|
PromptID: "adapter.test",
|
|
|
|
|
ProfileID: "empty-map-profile",
|
|
|
|
|
SessionID: "source-composition-test",
|
|
|
|
|
Inputs: contracts.LLMInputSet{
|
|
|
|
|
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
|
|
|
|
},
|
|
|
|
|
}, &out)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("CompleteStructured() with empty overlay map error = %v", err)
|
|
|
|
|
}
|
|
|
|
|
if got := fake.lastRequest().Target.ExtraParams["inherited"]; got != "retained" {
|
|
|
|
|
t.Fatalf("empty provider-map overlay result = %#v, want inherited entry", fake.lastRequest().Target.ExtraParams)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-04 13:15:40 +00:00
|
|
|
func TestPromptKitPromptContentUsesDeclaringAndReplacementSourceContexts(t *testing.T) {
|
|
|
|
|
rootDirectory := t.TempDir()
|
|
|
|
|
declaringDirectory := filepath.Join(rootDirectory, "child")
|
|
|
|
|
replacementDirectory := t.TempDir()
|
|
|
|
|
writeFile := func(path, content string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
rootConfig := filepath.Join(rootDirectory, "operator.yaml")
|
|
|
|
|
declaringConfig := filepath.Join(declaringDirectory, "imported.yaml")
|
|
|
|
|
replacementConfig := filepath.Join(replacementDirectory, "replacement.yaml")
|
|
|
|
|
writeFile(rootConfig, `configkit:
|
|
|
|
|
imports:
|
|
|
|
|
- path: child/imported.yaml
|
|
|
|
|
`)
|
|
|
|
|
writeFile(filepath.Join(rootDirectory, "messages", "shared.tmpl"), "root decoy")
|
|
|
|
|
writeFile(declaringConfig, `prompts:
|
|
|
|
|
imported.capture:
|
|
|
|
|
v1:
|
|
|
|
|
messages:
|
|
|
|
|
- role: user
|
|
|
|
|
content_file: messages/shared.tmpl
|
|
|
|
|
output:
|
|
|
|
|
format: text
|
|
|
|
|
validation_mode: none
|
|
|
|
|
`)
|
|
|
|
|
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import")
|
|
|
|
|
writeFile(replacementConfig, `prompts:
|
|
|
|
|
adapter.test:
|
|
|
|
|
v1:
|
|
|
|
|
messages:
|
|
|
|
|
- role: user
|
|
|
|
|
content_file: messages/shared.tmpl
|
|
|
|
|
`)
|
|
|
|
|
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement original")
|
|
|
|
|
|
|
|
|
|
endpoint, model := "https://example.invalid/v1", "content-context-model"
|
|
|
|
|
newClient := func() *PromptKitClient {
|
|
|
|
|
t.Helper()
|
|
|
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
|
|
|
Assets: newTestPromptKitAssets(t),
|
|
|
|
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
|
|
|
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("content context profile", map[string]promptkit.ProfileConfig{
|
|
|
|
|
"content-context-profile": {Endpoint: &endpoint, Model: &model},
|
|
|
|
|
})}}},
|
|
|
|
|
{Inputs: []configkit.Input{{Source: configkit.File(rootConfig), Imports: true}}},
|
|
|
|
|
{Inputs: []configkit.Input{{Source: configkit.File(replacementConfig), Imports: true}}},
|
|
|
|
|
}},
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
|
|
|
|
}
|
|
|
|
|
return client
|
|
|
|
|
}
|
|
|
|
|
prepareMessage := func(client *PromptKitClient, promptID string) string {
|
|
|
|
|
t.Helper()
|
|
|
|
|
request := promptkit.RunRequest{PromptID: promptID, ProfileID: "content-context-profile", SessionID: "content-context-session"}
|
|
|
|
|
if promptID == "adapter.test" {
|
|
|
|
|
request.Inputs = map[string]promptkit.ArtifactRef{
|
|
|
|
|
"transcript": promptkit.Inline(`{"source":true}`),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
prepared, err := client.engine.Prepare(context.Background(), request)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("Prepare(%q) error = %v", promptID, err)
|
|
|
|
|
}
|
|
|
|
|
if len(prepared.Messages) != 1 {
|
|
|
|
|
t.Fatalf("Prepare(%q) messages = %#v, want one", promptID, prepared.Messages)
|
|
|
|
|
}
|
|
|
|
|
return prepared.Messages[0].Content
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
first := newClient()
|
|
|
|
|
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
|
|
|
|
t.Fatalf("imported prompt content = %q, want declaring-source content", got)
|
|
|
|
|
}
|
|
|
|
|
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
|
|
|
|
t.Fatalf("partially overlaid application prompt content = %q, want replacement-source content", got)
|
|
|
|
|
}
|
|
|
|
|
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import updated")
|
|
|
|
|
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement updated")
|
|
|
|
|
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
|
|
|
|
t.Fatalf("existing engine imported prompt content after edit = %q", got)
|
|
|
|
|
}
|
|
|
|
|
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
|
|
|
|
t.Fatalf("existing engine replacement content after edit = %q", got)
|
|
|
|
|
}
|
|
|
|
|
second := newClient()
|
|
|
|
|
if got := prepareMessage(second, "imported.capture"); got != "declared import updated" {
|
|
|
|
|
t.Fatalf("new engine imported prompt content = %q", got)
|
|
|
|
|
}
|
|
|
|
|
if got := prepareMessage(second, "adapter.test"); got != "replacement updated" {
|
|
|
|
|
t.Fatalf("new engine replacement content = %q", got)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestPromptKitStaticMessageFailuresOccurAtConstruction(t *testing.T) {
|
|
|
|
|
for _, test := range []struct {
|
|
|
|
|
name string
|
|
|
|
|
prompt string
|
|
|
|
|
files fstest.MapFS
|
|
|
|
|
}{
|
|
|
|
|
{
|
|
|
|
|
name: "missing message file",
|
|
|
|
|
prompt: `prompts:
|
|
|
|
|
construction.failure:
|
|
|
|
|
v1:
|
|
|
|
|
messages:
|
|
|
|
|
- role: user
|
|
|
|
|
content_file: messages/missing.tmpl
|
|
|
|
|
output:
|
|
|
|
|
format: text
|
|
|
|
|
validation_mode: none
|
|
|
|
|
`,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
name: "invalid static template",
|
|
|
|
|
prompt: `prompts:
|
|
|
|
|
construction.failure:
|
|
|
|
|
v1:
|
|
|
|
|
messages:
|
|
|
|
|
- role: user
|
|
|
|
|
content: '{{ .request_value'
|
|
|
|
|
output:
|
|
|
|
|
format: text
|
|
|
|
|
validation_mode: none
|
|
|
|
|
`,
|
|
|
|
|
},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(test.name, func(t *testing.T) {
|
|
|
|
|
assets := NewAssetRegistry()
|
|
|
|
|
files := test.files
|
|
|
|
|
if files == nil {
|
|
|
|
|
files = fstest.MapFS{}
|
|
|
|
|
}
|
|
|
|
|
files["prompt.yaml"] = &fstest.MapFile{Data: []byte(test.prompt)}
|
|
|
|
|
if err := assets.RegisterPromptFS(files, "."); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
_, err := NewPromptKitClient(PromptKitClientConfig{
|
|
|
|
|
Assets: assets,
|
|
|
|
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"construction-profile": {
|
|
|
|
|
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
|
|
|
|
Model: promptKitTestString("construction-model"),
|
|
|
|
|
}}),
|
|
|
|
|
})
|
|
|
|
|
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
|
|
|
t.Fatalf("NewPromptKitClient() error = %v, want static construction failure", err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestPromptKitMissingRequestVariableFailsDuringPreparation(t *testing.T) {
|
|
|
|
|
assets := NewAssetRegistry()
|
|
|
|
|
if err := assets.RegisterPromptFS(fstest.MapFS{
|
|
|
|
|
"prompt.yaml": {Data: []byte(`prompts:
|
|
|
|
|
request.variable:
|
|
|
|
|
v1:
|
|
|
|
|
messages:
|
|
|
|
|
- role: user
|
|
|
|
|
content: 'Value: {{ .request_value }}'
|
|
|
|
|
output:
|
|
|
|
|
format: text
|
|
|
|
|
validation_mode: none
|
|
|
|
|
`)},
|
|
|
|
|
}, "."); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
|
|
|
|
Assets: assets,
|
|
|
|
|
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"request-profile": {
|
|
|
|
|
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
|
|
|
|
Model: promptKitTestString("request-model"),
|
|
|
|
|
}}),
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("NewPromptKitClient() error = %v, want valid static template", err)
|
|
|
|
|
}
|
|
|
|
|
if _, err := client.engine.Prepare(context.Background(), promptkit.RunRequest{
|
|
|
|
|
PromptID: "request.variable", ProfileID: "request-profile",
|
|
|
|
|
}); !errors.Is(err, promptkit.ErrPromptRender) {
|
|
|
|
|
t.Fatalf("Prepare() error = %v, want missing request variable render failure", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-04 12:46:23 +00:00
|
|
|
func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) {
|
|
|
|
|
t.Run("directory group conflicts", func(t *testing.T) {
|
|
|
|
|
directory := t.TempDir()
|
|
|
|
|
for name, model := range map[string]string{"first.yaml": "first-model", "second.yaml": "second-model"} {
|
|
|
|
|
content := "profiles:\n duplicate:\n endpoint: http://127.0.0.1:1/v1\n model: " + model + "\n"
|
|
|
|
|
if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
|
|
|
|
Source: PromptKitProfileSourceConfig{ProfileDir: directory},
|
|
|
|
|
})
|
|
|
|
|
if err == nil {
|
|
|
|
|
t.Fatal("NewPromptKitProfileInspector() error = nil, want same-group conflict")
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("later group repairs an invalid lower field and overlays supplied values", func(t *testing.T) {
|
|
|
|
|
badEndpoint := "not an endpoint"
|
|
|
|
|
goodEndpoint := "http://127.0.0.1:1/v1"
|
|
|
|
|
lowerModel := "lower-model"
|
|
|
|
|
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
|
|
|
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower source", map[string]promptkit.ProfileConfig{
|
|
|
|
|
"layered-profile": {Endpoint: &badEndpoint, Model: &lowerModel},
|
|
|
|
|
})}}},
|
|
|
|
|
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("upper source", map[string]promptkit.ProfileConfig{
|
|
|
|
|
"layered-profile": {Endpoint: &goodEndpoint},
|
|
|
|
|
})}}},
|
|
|
|
|
}}
|
|
|
|
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("NewPromptKitProfileInspector() error = %v, want repaired final layer", err)
|
|
|
|
|
}
|
|
|
|
|
inspection, err := inspector.InspectProfile(context.Background(), "layered-profile")
|
|
|
|
|
if err != nil || inspection.Model != lowerModel {
|
|
|
|
|
t.Fatalf("InspectProfile() = (%#v, %v), want lower model retained", inspection, err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("empty and missing selected directories", func(t *testing.T) {
|
|
|
|
|
if _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
|
|
|
|
Source: PromptKitProfileSourceConfig{ProfileDir: filepath.Join(t.TempDir(), "missing")},
|
|
|
|
|
}); err == nil {
|
|
|
|
|
t.Fatal("missing selected directory was accepted")
|
|
|
|
|
}
|
|
|
|
|
empty := t.TempDir()
|
|
|
|
|
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{
|
|
|
|
|
Source: PromptKitProfileSourceConfig{ProfileDir: empty},
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatalf("empty selected directory error = %v, want no contribution", err)
|
|
|
|
|
}
|
|
|
|
|
if _, err := inspector.InspectProfile(context.Background(), "not-present"); err == nil {
|
|
|
|
|
t.Fatal("InspectProfile() in empty directory error = nil, want unknown profile")
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("unused invalid final profile fails construction", func(t *testing.T) {
|
|
|
|
|
badEndpoint := "not an endpoint"
|
|
|
|
|
model := "unused-model"
|
|
|
|
|
sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
|
|
|
|
Source: promptkit.ProfileSource("unused invalid profile", map[string]promptkit.ProfileConfig{
|
|
|
|
|
"unused-profile": {Endpoint: &badEndpoint, Model: &model},
|
|
|
|
|
}),
|
|
|
|
|
}}}}}
|
|
|
|
|
_, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources})
|
|
|
|
|
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
|
|
|
|
t.Fatalf("NewPromptKitProfileInspector() error = %v, want invalid final configuration", err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|