Migrate CLI and module tests to keyed PromptKit sources
This commit is contained in:
@@ -19,6 +19,7 @@ import (
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
|
||||
@@ -59,6 +60,7 @@ import (
|
||||
scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
@@ -228,14 +230,16 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
newEngine := func(profileFile string) (*promptkit.Engine, error) {
|
||||
t.Helper()
|
||||
options, err := components.assets.PromptKitOptions()
|
||||
sources, options, err := components.assets.PromptKitConfigurationSources()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if profileFile != "" {
|
||||
options = append(options, promptkit.WithProfileFile(profileFile))
|
||||
sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{
|
||||
Source: configkit.File(profileFile), Imports: true,
|
||||
}}})
|
||||
}
|
||||
return promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
return promptkit.NewEngine(promptkit.Config{Sources: sources}, options...)
|
||||
}
|
||||
|
||||
t.Run("fallback", func(t *testing.T) {
|
||||
@@ -258,10 +262,11 @@ func TestProductionAssetsResolveDNDExtractionProfile(t *testing.T) {
|
||||
|
||||
t.Run("valid operator profile wins", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
if err := os.WriteFile(profilePath, []byte(`id: dnd-extraction
|
||||
endpoint: http://operator.example.test/v1
|
||||
model: operator-model
|
||||
timeout_seconds: 75
|
||||
if err := os.WriteFile(profilePath, []byte(`profiles:
|
||||
dnd-extraction:
|
||||
endpoint: http://operator.example.test/v1
|
||||
model: operator-model
|
||||
timeout_seconds: 75
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -274,8 +279,8 @@ timeout_seconds: 75
|
||||
t.Fatalf("InspectProfile() error = %v, want operator profile", err)
|
||||
}
|
||||
params := inspection.EffectiveModelParams
|
||||
if params.BackendID != "" || params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "" {
|
||||
t.Fatalf("operator profile parameters = %#v, want complete replacement", params)
|
||||
if params.Endpoint != "http://operator.example.test/v1" || params.Model != "operator-model" || params.TimeoutSeconds != 75 || params.ServiceTier != "flex" {
|
||||
t.Fatalf("operator profile parameters = %#v, want recursive overlay over fallback profile", params)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -320,17 +325,9 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||
if _, err := fs.ReadFile(schemaFS, filepath.Base(semanticreconcile.SchemaAssetPath)); err != nil {
|
||||
t.Fatalf("generic reconciliation schema asset: %v", err)
|
||||
}
|
||||
options, err := components.assets.PromptKitOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("production PromptKit options: %v", err)
|
||||
}
|
||||
options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "assembled-prompt-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test",
|
||||
})))
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("production prompt engine: %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, components.assets, promptkit.Config{}, map[string]promptkit.ProfileConfig{
|
||||
"assembled-prompt-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||
})
|
||||
if engine == nil {
|
||||
t.Fatal("production PromptKit engine is nil")
|
||||
}
|
||||
@@ -575,25 +572,27 @@ func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||
if err := os.WriteFile(profilePath, []byte(`id: local-profile
|
||||
backend: local
|
||||
model: local-model
|
||||
if err := os.WriteFile(profilePath, []byte(`profiles:
|
||||
local-profile:
|
||||
backend: local
|
||||
model: local-model
|
||||
`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||
"production.local.yaml": {Data: []byte(`id: production.local
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{ input "transcript" }}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json
|
||||
"production.local.yaml": {Data: []byte(`prompts:
|
||||
production.local:
|
||||
v1:
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{ input "transcript" }}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json
|
||||
`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatalf("register prompt assets: %v", err)
|
||||
|
||||
@@ -31,11 +31,12 @@ func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *
|
||||
}
|
||||
return profilePath
|
||||
}
|
||||
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
|
||||
credentialProfile := `id: credential-profile
|
||||
endpoint: ` + server.URL + `/v1
|
||||
model: credential-model
|
||||
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
localProfile := "profiles:\n local-profile:\n backend: local\n model: local-model\n"
|
||||
credentialProfile := `profiles:
|
||||
credential-profile:
|
||||
endpoint: ` + server.URL + `/v1
|
||||
model: credential-model
|
||||
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
`
|
||||
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
|
||||
|
||||
@@ -60,7 +61,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
name: "missing local backend registration",
|
||||
profilePath: writeProfile(t, "local-profile", localProfile),
|
||||
profileID: "local-profile",
|
||||
wantErr: []string{`PromptKit profile "local-profile" is invalid or unreadable`},
|
||||
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
|
||||
},
|
||||
{
|
||||
name: "absent profile",
|
||||
@@ -71,9 +72,9 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
},
|
||||
{
|
||||
name: "malformed profile",
|
||||
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nendpoint: https://provider.example/v1?credential=forbidden\nmodel: malformed-model\n"),
|
||||
profilePath: writeProfile(t, "malformed-profile", "profiles:\n malformed-profile:\n endpoint: https://provider.example/v1?credential=forbidden\n model: malformed-model\n"),
|
||||
profileID: "malformed-profile",
|
||||
wantErr: []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
|
||||
wantErr: []string{"load PromptKit profiles", "configuration is invalid or unreadable"},
|
||||
rejectErr: []string{"malformed-profile.yaml", "credential=forbidden"},
|
||||
},
|
||||
{
|
||||
@@ -149,7 +150,7 @@ api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
|
||||
func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||
assets := llm.NewAssetRegistry()
|
||||
if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
|
||||
"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
|
||||
"profiles/fallback.yaml": {Data: []byte("profiles:\n fallback-profile:\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")},
|
||||
}, "profiles"); err != nil {
|
||||
t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
|
||||
}
|
||||
@@ -160,8 +161,8 @@ func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
|
||||
|
||||
func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGeneration(t *testing.T) {
|
||||
for _, profiles := range []string{
|
||||
"id: child\nbase_profile: missing\n",
|
||||
"id: first\nbase_profile: second\n\n---\nid: second\nbase_profile: first\n",
|
||||
"profiles:\n child:\n extends: missing\n",
|
||||
"profiles:\n first:\n extends: second\n second:\n extends: first\n",
|
||||
} {
|
||||
t.Run("invalid inheritance", func(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
@@ -169,7 +170,7 @@ func TestExplicitPromptKitProfileValidationRejectsInvalidInheritanceBeforeGenera
|
||||
t.Fatal(err)
|
||||
}
|
||||
profileID := "child"
|
||||
if strings.Contains(profiles, "id: first") {
|
||||
if strings.Contains(profiles, "first:") {
|
||||
profileID = "first"
|
||||
}
|
||||
err := validateExplicitPromptKitProfiles(context.Background(), config.Config{PromptKit: config.PromptKitConfig{ProfileFile: profilePath}}, []string{profileID}, nil)
|
||||
|
||||
@@ -679,7 +679,7 @@ func writeRunContractProfiles(t *testing.T, ids ...string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for _, id := range ids {
|
||||
profile := fmt.Sprintf("id: %s\nendpoint: http://127.0.0.1:1/v1\nmodel: %s-model\n", id, id)
|
||||
profile := fmt.Sprintf("profiles:\n %s:\n endpoint: http://127.0.0.1:1/v1\n model: %s-model\n", id, id)
|
||||
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user