Verify PromptKit content origins and capture lifetime
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
@@ -197,6 +198,197 @@ prompts:
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitPromptContentUsesDeclaringAndReplacementSourceContexts(t *testing.T) {
|
||||
rootDirectory := t.TempDir()
|
||||
declaringDirectory := filepath.Join(rootDirectory, "child")
|
||||
replacementDirectory := t.TempDir()
|
||||
writeFile := func(path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rootConfig := filepath.Join(rootDirectory, "operator.yaml")
|
||||
declaringConfig := filepath.Join(declaringDirectory, "imported.yaml")
|
||||
replacementConfig := filepath.Join(replacementDirectory, "replacement.yaml")
|
||||
writeFile(rootConfig, `configkit:
|
||||
imports:
|
||||
- path: child/imported.yaml
|
||||
`)
|
||||
writeFile(filepath.Join(rootDirectory, "messages", "shared.tmpl"), "root decoy")
|
||||
writeFile(declaringConfig, `prompts:
|
||||
imported.capture:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content_file: messages/shared.tmpl
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import")
|
||||
writeFile(replacementConfig, `prompts:
|
||||
adapter.test:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content_file: messages/shared.tmpl
|
||||
`)
|
||||
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement original")
|
||||
|
||||
endpoint, model := "https://example.invalid/v1", "content-context-model"
|
||||
newClient := func() *PromptKitClient {
|
||||
t.Helper()
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("content context profile", map[string]promptkit.ProfileConfig{
|
||||
"content-context-profile": {Endpoint: &endpoint, Model: &model},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.File(rootConfig), Imports: true}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.File(replacementConfig), Imports: true}}},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
prepareMessage := func(client *PromptKitClient, promptID string) string {
|
||||
t.Helper()
|
||||
request := promptkit.RunRequest{PromptID: promptID, ProfileID: "content-context-profile", SessionID: "content-context-session"}
|
||||
if promptID == "adapter.test" {
|
||||
request.Inputs = map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"source":true}`),
|
||||
}
|
||||
}
|
||||
prepared, err := client.engine.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare(%q) error = %v", promptID, err)
|
||||
}
|
||||
if len(prepared.Messages) != 1 {
|
||||
t.Fatalf("Prepare(%q) messages = %#v, want one", promptID, prepared.Messages)
|
||||
}
|
||||
return prepared.Messages[0].Content
|
||||
}
|
||||
|
||||
first := newClient()
|
||||
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
||||
t.Fatalf("imported prompt content = %q, want declaring-source content", got)
|
||||
}
|
||||
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
||||
t.Fatalf("partially overlaid application prompt content = %q, want replacement-source content", got)
|
||||
}
|
||||
writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import updated")
|
||||
writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement updated")
|
||||
if got := prepareMessage(first, "imported.capture"); got != "declared import" {
|
||||
t.Fatalf("existing engine imported prompt content after edit = %q", got)
|
||||
}
|
||||
if got := prepareMessage(first, "adapter.test"); got != "replacement original" {
|
||||
t.Fatalf("existing engine replacement content after edit = %q", got)
|
||||
}
|
||||
second := newClient()
|
||||
if got := prepareMessage(second, "imported.capture"); got != "declared import updated" {
|
||||
t.Fatalf("new engine imported prompt content = %q", got)
|
||||
}
|
||||
if got := prepareMessage(second, "adapter.test"); got != "replacement updated" {
|
||||
t.Fatalf("new engine replacement content = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitStaticMessageFailuresOccurAtConstruction(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
prompt string
|
||||
files fstest.MapFS
|
||||
}{
|
||||
{
|
||||
name: "missing message file",
|
||||
prompt: `prompts:
|
||||
construction.failure:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content_file: messages/missing.tmpl
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "invalid static template",
|
||||
prompt: `prompts:
|
||||
construction.failure:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{ .request_value'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assets := NewAssetRegistry()
|
||||
files := test.files
|
||||
if files == nil {
|
||||
files = fstest.MapFS{}
|
||||
}
|
||||
files["prompt.yaml"] = &fstest.MapFile{Data: []byte(test.prompt)}
|
||||
if err := assets.RegisterPromptFS(files, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"construction-profile": {
|
||||
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
||||
Model: promptKitTestString("construction-model"),
|
||||
}}),
|
||||
})
|
||||
if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want static construction failure", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitMissingRequestVariableFailsDuringPreparation(t *testing.T) {
|
||||
assets := NewAssetRegistry()
|
||||
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||
"prompt.yaml": {Data: []byte(`prompts:
|
||||
request.variable:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content: 'Value: {{ .request_value }}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"request-profile": {
|
||||
Endpoint: promptKitTestString("https://example.invalid/v1"),
|
||||
Model: promptKitTestString("request-model"),
|
||||
}}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v, want valid static template", err)
|
||||
}
|
||||
if _, err := client.engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "request.variable", ProfileID: "request-profile",
|
||||
}); !errors.Is(err, promptkit.ErrPromptRender) {
|
||||
t.Fatalf("Prepare() error = %v, want missing request variable render failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) {
|
||||
t.Run("directory group conflicts", func(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
|
||||
Reference in New Issue
Block a user