Files
notarius/assets/scenes_test.go

185 lines
5.6 KiB
Go

package assets_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/fs"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"github.com/santhosh-tekuri/jsonschema/v6"
"go.maximumdirect.net/promptkit"
)
func TestScenePromptPairPreservesFullTranscriptAndContext(t *testing.T) {
_, engine := loadNativeAssets(t)
const transcript = `{"sentinel":"scene-transcript"}`
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scenes", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(transcript),
"players": promptkit.Inline("scene-players"),
"party": promptkit.Inline("scene-party"),
"glossary": promptkit.Inline("scene-glossary"),
},
})
if err != nil {
t.Fatal(err)
}
defer initial.Discard()
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scenes.repair", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Keep the encounter together.")},
})
if err != nil {
t.Fatal(err)
}
defer correction.Discard()
first, next := initial.Details(), correction.Details()
if first.SelectedProfileID != "notarius/dnd-extraction" || next.SelectedProfileID != first.SelectedProfileID ||
first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
t.Fatal("scene correction pair lost its shared native defaults/output contract")
}
var system, all string
transcriptIndex := -1
for i, message := range first.Messages {
all += message.Content
if message.Role == "system" {
system += message.Content
}
if strings.Contains(message.Content, transcript) {
transcriptIndex = i
}
if strings.Contains(message.Content, transcript) || strings.Contains(message.Content, "scene-players") {
if message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral {
t.Fatal("reference/full-transcript native cache hint lost")
}
}
}
if transcriptIndex != len(first.Messages)-1 || !strings.Contains(system, "no gaps or overlaps") || !strings.Contains(system, "coherent unit of play") {
t.Fatal("scene principles or complete final transcript lost")
}
for _, content := range []string{transcript, "scene-players", "scene-party", "scene-glossary"} {
if strings.Count(all, content) != 1 {
t.Fatalf("fixed context not rendered once: %q", content)
}
}
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Keep the encounter together.") ||
strings.Contains(next.Messages[0].Content, "scene-transcript") {
t.Fatal("correction should supply findings and a replacement assignment only")
}
}
func TestResponseSchemaValidatesMinimalSceneResponses(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/scenes/schemas/dnd_scenes_llm.v1.json")
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
valid := validSceneSchemaResponse()
validJSON, err := json.Marshal(valid)
if err != nil {
t.Fatalf("Marshal(validSceneSchemaResponse()) error = %v, want nil", err)
}
if err := validateJSONSchema(validJSON, schema); err != nil {
t.Fatalf("valid scene response rejected: %v", err)
}
tests := []struct {
name string
mutate func(map[string]any)
}{
{
name: "missing scenes",
mutate: func(response map[string]any) {
delete(response, "scenes")
},
},
{
name: "empty scenes",
mutate: func(response map[string]any) {
response["scenes"] = []any{}
},
},
{
name: "non-positive start unit",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 0
},
},
{
name: "non-positive end unit",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["end_unit_id"] = 0
},
},
{
name: "non-integer endpoint",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["start_unit_id"] = 1.5
},
},
{
name: "unknown top-level field",
mutate: func(response map[string]any) {
response["boundary_caveats"] = []any{}
},
},
{
name: "unknown scene field",
mutate: func(response map[string]any) {
response["scenes"].([]any)[0].(map[string]any)["short_title"] = "Old contract"
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
response := validSceneSchemaResponse()
tt.mutate(response)
content, err := json.Marshal(response)
if err != nil {
t.Fatalf("Marshal() error = %v, want nil", err)
}
if err := validateJSONSchema(content, schema); err == nil {
t.Fatal("validateJSONSchema() error = nil, want rejected response")
}
})
}
}
func validSceneSchemaResponse() map[string]any {
return map[string]any{
"scenes": []any{
map[string]any{
"start_unit_id": 1,
"end_unit_id": 3,
},
},
}
}
func validateJSONSchema(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return fmt.Errorf("parse instance: %w", err)
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return fmt.Errorf("parse schema: %w", err)
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
return fmt.Errorf("load schema: %w", err)
}
schema, err := compiler.Compile("schema.json")
if err != nil {
return fmt.Errorf("compile schema: %w", err)
}
return schema.Validate(instance)
}