Files
notarius/assets/prompt_composition_test.go

178 lines
7.3 KiB
Go

package assets_test
import (
"context"
"strings"
"testing"
"go.maximumdirect.net/promptkit"
)
func TestExtractionPromptComposition(t *testing.T) {
const (
transcriptSentinel = "shared-transcript-sentinel"
playersSentinel = "shared-players-sentinel"
partySentinel = "shared-party-sentinel"
glossarySentinel = "shared-glossary-sentinel"
npcSentinel = "npc-registry-sentinel"
catalogSentinel = "spell-catalog-sentinel"
evidenceSentinel = "Transcript units are the only evidence"
endpointOrderSentinel = "according to transcript order"
combatPolicySentinel = "substantive active combat materially organizes the"
)
_, engine := loadNativeAssets(t)
commonInputs := map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
"players": promptkit.Inline(playersSentinel),
"party": promptkit.Inline(partySentinel),
"glossary": promptkit.Inline(glossarySentinel),
}
cases := []struct {
name string
promptID string
inputs map[string]promptkit.ArtifactRef
suffixGroups [][]string
middleSentinel string
}{
{name: "npcs", promptID: "notarius/dnd.npc_registry", inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
{name: "locations", promptID: "notarius/dnd.location_registry", inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
{name: "item occurrences", promptID: "notarius/dnd.item_occurrences", inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"item_registry": promptkit.Inline(`{"items":[{"id":"item-registry-sentinel","name":"Torch"}]}`),
}), suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"item-registry-sentinel"}}},
{name: "item registry", promptID: "notarius/dnd.item_registry", inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
{name: "scene descriptions", promptID: "notarius/dnd.scene_descriptions", inputs: commonInputs, middleSentinel: combatPolicySentinel},
{
name: "combat turns",
promptID: "notarius/dnd.combat_turns",
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
},
{
name: "enemy events",
promptID: "notarius/dnd.enemy_events",
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
"combat_turns": promptkit.Inline(`{"sentinel":"combat-turns-sentinel"}`),
"npc_occurrences": promptkit.Inline(`{"sentinel":"npc-occurrences-sentinel"}`),
}),
suffixGroups: [][]string{
{evidenceSentinel, endpointOrderSentinel},
{npcSentinel},
{"combat-turns-sentinel", "npc-occurrences-sentinel"},
},
},
{
name: "npc occurrences",
promptID: "notarius/dnd.npc_occurrences",
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
}),
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
},
{
name: "location occurrences",
promptID: "notarius/dnd.location_occurrences",
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"location_registry": promptkit.Inline(`{"sentinel":"location-registry-sentinel"}`),
}),
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"location-registry-sentinel"}},
},
{
name: "spells",
promptID: "notarius/dnd.spells",
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
}),
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}, {catalogSentinel}},
},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: testCase.promptID,
APIKey: "test-only",
ProfileID: "notarius/dnd-extraction",
Inputs: testCase.inputs,
})
if err != nil {
t.Fatalf("PrepareExecution() error = %v", err)
}
defer handle.Discard()
prepared := handle.Details()
transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel)
referenceIndex := renderedInputMessageIndex(t, prepared.Messages, playersSentinel)
if referenceIndex >= transcriptIndex {
t.Fatalf("campaign references rendered at message %d, want before transcript message %d", referenceIndex, transcriptIndex)
}
assertEphemeralCache(t, prepared.Messages, referenceIndex)
assertEphemeralCache(t, prepared.Messages, transcriptIndex)
previousIndex := transcriptIndex
for _, group := range testCase.suffixGroups {
groupIndex := -1
for _, sentinel := range group {
inputIndex := renderedInputMessageIndex(t, prepared.Messages, sentinel)
if groupIndex < 0 {
groupIndex = inputIndex
} else if inputIndex != groupIndex {
t.Fatalf("input sentinel %q rendered at message %d, want grouped at message %d", sentinel, inputIndex, groupIndex)
}
}
if groupIndex <= previousIndex {
t.Fatalf("prompt suffix group rendered at message %d, want after message %d", groupIndex, previousIndex)
}
previousIndex = groupIndex
}
instructionIndex := len(prepared.Messages) - 1
if testCase.middleSentinel != "" {
policyIndex := renderedInputMessageIndex(t, prepared.Messages, testCase.middleSentinel)
if policyIndex <= transcriptIndex || policyIndex >= instructionIndex {
t.Fatalf("combat policy rendered at message %d, want after transcript message %d and before instructions message %d", policyIndex, transcriptIndex, instructionIndex)
}
}
if instructionIndex <= previousIndex {
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
}
assertEphemeralCache(t, prepared.Messages, instructionIndex)
})
}
}
func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[string]promptkit.ArtifactRef {
merged := make(map[string]promptkit.ArtifactRef, len(inputs)+len(extras))
for name, input := range inputs {
merged[name] = input
}
for name, input := range extras {
merged[name] = input
}
return merged
}
func assertEphemeralCache(t *testing.T, messages []promptkit.RenderedMessage, index int) {
t.Helper()
if cache := messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
}
}
func renderedInputMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
t.Helper()
index := -1
occurrences := 0
for messageIndex, message := range messages {
count := strings.Count(message.Content, sentinel)
if count > 0 {
index = messageIndex
occurrences += count
}
}
if occurrences != 1 {
t.Fatalf("input sentinel %q rendered %d times, want exactly once", sentinel, occurrences)
}
return index
}