Make production registration asset-free and verify direct deterministic execution
This commit is contained in:
177
assets/prompt_composition_test.go
Normal file
177
assets/prompt_composition_test.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestExtractionPromptComposition(t *testing.T) {
|
||||
const (
|
||||
transcriptSentinel = "shared-transcript-sentinel"
|
||||
playersSentinel = "shared-players-sentinel"
|
||||
partySentinel = "shared-party-sentinel"
|
||||
glossarySentinel = "shared-glossary-sentinel"
|
||||
npcSentinel = "npc-registry-sentinel"
|
||||
catalogSentinel = "spell-catalog-sentinel"
|
||||
evidenceSentinel = "Transcript units are the only evidence"
|
||||
endpointOrderSentinel = "according to transcript order"
|
||||
combatPolicySentinel = "substantive active combat materially organizes the"
|
||||
)
|
||||
_, engine := loadNativeAssets(t)
|
||||
commonInputs := map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"`+transcriptSentinel+`"}`),
|
||||
"players": promptkit.Inline(playersSentinel),
|
||||
"party": promptkit.Inline(partySentinel),
|
||||
"glossary": promptkit.Inline(glossarySentinel),
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
inputs map[string]promptkit.ArtifactRef
|
||||
suffixGroups [][]string
|
||||
middleSentinel string
|
||||
}{
|
||||
{name: "npcs", promptID: "notarius/dnd.npc_registry", inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "locations", promptID: "notarius/dnd.location_registry", inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "item occurrences", promptID: "notarius/dnd.item_occurrences", inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"item_registry": promptkit.Inline(`{"items":[{"id":"item-registry-sentinel","name":"Torch"}]}`),
|
||||
}), suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"item-registry-sentinel"}}},
|
||||
{name: "item registry", promptID: "notarius/dnd.item_registry", inputs: commonInputs, suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}}},
|
||||
{name: "scene descriptions", promptID: "notarius/dnd.scene_descriptions", inputs: commonInputs, middleSentinel: combatPolicySentinel},
|
||||
{
|
||||
name: "combat turns",
|
||||
promptID: "notarius/dnd.combat_turns",
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "enemy events",
|
||||
promptID: "notarius/dnd.enemy_events",
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"combat_turns": promptkit.Inline(`{"sentinel":"combat-turns-sentinel"}`),
|
||||
"npc_occurrences": promptkit.Inline(`{"sentinel":"npc-occurrences-sentinel"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{
|
||||
{evidenceSentinel, endpointOrderSentinel},
|
||||
{npcSentinel},
|
||||
{"combat-turns-sentinel", "npc-occurrences-sentinel"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "npc occurrences",
|
||||
promptID: "notarius/dnd.npc_occurrences",
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}},
|
||||
},
|
||||
{
|
||||
name: "location occurrences",
|
||||
promptID: "notarius/dnd.location_occurrences",
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"location_registry": promptkit.Inline(`{"sentinel":"location-registry-sentinel"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {"location-registry-sentinel"}},
|
||||
},
|
||||
{
|
||||
name: "spells",
|
||||
promptID: "notarius/dnd.spells",
|
||||
inputs: withPromptInputs(commonInputs, map[string]promptkit.ArtifactRef{
|
||||
"npc_registry": promptkit.Inline(`{"sentinel":"` + npcSentinel + `"}`),
|
||||
"spell_catalog": promptkit.Inline(`{"sentinel":"` + catalogSentinel + `"}`),
|
||||
}),
|
||||
suffixGroups: [][]string{{evidenceSentinel, endpointOrderSentinel}, {npcSentinel}, {catalogSentinel}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: testCase.promptID,
|
||||
APIKey: "test-only",
|
||||
ProfileID: "notarius/dnd-extraction",
|
||||
Inputs: testCase.inputs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PrepareExecution() error = %v", err)
|
||||
}
|
||||
defer handle.Discard()
|
||||
prepared := handle.Details()
|
||||
transcriptIndex := renderedInputMessageIndex(t, prepared.Messages, transcriptSentinel)
|
||||
referenceIndex := renderedInputMessageIndex(t, prepared.Messages, playersSentinel)
|
||||
if referenceIndex >= transcriptIndex {
|
||||
t.Fatalf("campaign references rendered at message %d, want before transcript message %d", referenceIndex, transcriptIndex)
|
||||
}
|
||||
assertEphemeralCache(t, prepared.Messages, referenceIndex)
|
||||
assertEphemeralCache(t, prepared.Messages, transcriptIndex)
|
||||
previousIndex := transcriptIndex
|
||||
for _, group := range testCase.suffixGroups {
|
||||
groupIndex := -1
|
||||
for _, sentinel := range group {
|
||||
inputIndex := renderedInputMessageIndex(t, prepared.Messages, sentinel)
|
||||
if groupIndex < 0 {
|
||||
groupIndex = inputIndex
|
||||
} else if inputIndex != groupIndex {
|
||||
t.Fatalf("input sentinel %q rendered at message %d, want grouped at message %d", sentinel, inputIndex, groupIndex)
|
||||
}
|
||||
}
|
||||
if groupIndex <= previousIndex {
|
||||
t.Fatalf("prompt suffix group rendered at message %d, want after message %d", groupIndex, previousIndex)
|
||||
}
|
||||
previousIndex = groupIndex
|
||||
}
|
||||
instructionIndex := len(prepared.Messages) - 1
|
||||
if testCase.middleSentinel != "" {
|
||||
policyIndex := renderedInputMessageIndex(t, prepared.Messages, testCase.middleSentinel)
|
||||
if policyIndex <= transcriptIndex || policyIndex >= instructionIndex {
|
||||
t.Fatalf("combat policy rendered at message %d, want after transcript message %d and before instructions message %d", policyIndex, transcriptIndex, instructionIndex)
|
||||
}
|
||||
}
|
||||
if instructionIndex <= previousIndex {
|
||||
t.Fatalf("instructions rendered at message %d, want after lane input message %d", instructionIndex, previousIndex)
|
||||
}
|
||||
assertEphemeralCache(t, prepared.Messages, instructionIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func withPromptInputs(inputs, extras map[string]promptkit.ArtifactRef) map[string]promptkit.ArtifactRef {
|
||||
merged := make(map[string]promptkit.ArtifactRef, len(inputs)+len(extras))
|
||||
for name, input := range inputs {
|
||||
merged[name] = input
|
||||
}
|
||||
for name, input := range extras {
|
||||
merged[name] = input
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func assertEphemeralCache(t *testing.T, messages []promptkit.RenderedMessage, index int) {
|
||||
t.Helper()
|
||||
if cache := messages[index].CacheControl; cache == nil || cache.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatalf("message %d cache control = %#v, want ephemeral", index, cache)
|
||||
}
|
||||
}
|
||||
|
||||
func renderedInputMessageIndex(t *testing.T, messages []promptkit.RenderedMessage, sentinel string) int {
|
||||
t.Helper()
|
||||
index := -1
|
||||
occurrences := 0
|
||||
for messageIndex, message := range messages {
|
||||
count := strings.Count(message.Content, sentinel)
|
||||
if count > 0 {
|
||||
index = messageIndex
|
||||
occurrences += count
|
||||
}
|
||||
}
|
||||
if occurrences != 1 {
|
||||
t.Fatalf("input sentinel %q rendered %d times, want exactly once", sentinel, occurrences)
|
||||
}
|
||||
return index
|
||||
}
|
||||
78
assets/schema_compatibility_test.go
Normal file
78
assets/schema_compatibility_test.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
)
|
||||
|
||||
func TestModelResponseSchemasRequireEveryObjectProperty(t *testing.T) {
|
||||
schemaFS := assets.FS()
|
||||
|
||||
count := 0
|
||||
err := fs.WalkDir(schemaFS, ".", func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() || !strings.HasSuffix(path, "_llm.v1.json") {
|
||||
return nil
|
||||
}
|
||||
content, err := fs.ReadFile(schemaFS, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var schema any
|
||||
if err := json.Unmarshal(content, &schema); err != nil {
|
||||
t.Errorf("decode response schema %q: %v", path, err)
|
||||
return nil
|
||||
}
|
||||
count++
|
||||
assertRequiredObjectProperties(t, path, "$", schema)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count == 0 {
|
||||
t.Fatal("no model response schemas found")
|
||||
}
|
||||
}
|
||||
|
||||
func assertRequiredObjectProperties(t *testing.T, schemaPath, nodePath string, value any) {
|
||||
t.Helper()
|
||||
switch node := value.(type) {
|
||||
case map[string]any:
|
||||
if properties, ok := node["properties"].(map[string]any); ok {
|
||||
required := make(map[string]struct{})
|
||||
if values, ok := node["required"].([]any); ok {
|
||||
for _, value := range values {
|
||||
if name, ok := value.(string); ok {
|
||||
required[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
propertyNames := make([]string, 0, len(properties))
|
||||
for name := range properties {
|
||||
propertyNames = append(propertyNames, name)
|
||||
}
|
||||
sort.Strings(propertyNames)
|
||||
for _, name := range propertyNames {
|
||||
if _, ok := required[name]; !ok {
|
||||
t.Errorf("response schema %q object %s declares property %q without requiring it", schemaPath, nodePath, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for name, child := range node {
|
||||
assertRequiredObjectProperties(t, schemaPath, nodePath+"."+name, child)
|
||||
}
|
||||
case []any:
|
||||
for index, child := range node {
|
||||
assertRequiredObjectProperties(t, schemaPath, nodePath+"["+strconv.Itoa(index)+"]", child)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user