Files
notarius/assets/combat_semantics_test.go

103 lines
4.1 KiB
Go

package assets_test
import (
"bytes"
"context"
"encoding/json"
"gitea.maximumdirect.net/eric/notarius/assets"
"io/fs"
"strings"
"testing"
"go.maximumdirect.net/promptkit"
"github.com/santhosh-tekuri/jsonschema/v6"
)
func TestCombatSemanticsPromptKeepsItsSceneLocalPolicyAndInput(t *testing.T) {
_, engine := loadNativeAssets(t)
const transcript = `{"sentinel":"combat-semantics-transcript"}`
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(transcript)},
})
if err != nil {
t.Fatal(err)
}
defer handle.Discard()
details := handle.Details()
if details.SelectedProfileID != "notarius/dnd-extraction" || details.StructuredOutput == nil {
t.Fatal("native validator defaults/output contract lost")
}
var all string
for _, message := range details.Messages {
all += message.Content
if strings.Contains(message.Content, transcript) && (message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral) {
t.Fatal("chunk transcript native cache hint lost")
}
}
for _, content := range []string{"substantive active combat materially organizes the", "Do not judge the scene title", transcript} {
if strings.Count(all, content) != 1 {
t.Fatalf("scene-local context not rendered once: %q", content)
}
}
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only"}); err == nil {
t.Fatal("native validator accepted missing transcript")
}
}
func TestCombatSemanticsResponseSchemaRetainsClassificationAndExplanationRules(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json")
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
for _, classification := range []string{"combat", "non_combat"} {
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema); err != nil {
t.Fatalf("valid %q classification rejected: %v", classification, err)
}
}
for _, test := range []struct {
name string
response map[string]any
}{
{name: "missing explanation", response: map[string]any{"classification": "combat"}},
{name: "missing classification", response: map[string]any{"explanation": "The chunk contains active combat."}},
{name: "unknown property", response: map[string]any{"classification": "combat", "explanation": "The chunk contains active combat.", "confidence": 1}},
{name: "wrong classification type", response: map[string]any{"classification": 1, "explanation": "The chunk contains active combat."}},
{name: "wrong explanation type", response: map[string]any{"classification": "combat", "explanation": 1}},
{name: "unsupported classification", response: map[string]any{"classification": "uncertain", "explanation": "The classification is uncertain."}},
{name: "blank explanation", response: map[string]any{"classification": "combat", "explanation": ""}},
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
} {
t.Run(test.name, func(t *testing.T) {
if err := validateCombatSemanticsSchema(test.response, schema); err == nil {
t.Fatal("invalid private response accepted")
}
})
}
}
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
instanceContent, err := json.Marshal(instance)
if err != nil {
return err
}
parsedInstance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return err
}
parsedSchema, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return err
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", parsedSchema); err != nil {
return err
}
compiled, err := compiler.Compile("schema.json")
if err != nil {
return err
}
return compiled.Validate(parsedInstance)
}