103 lines
4.1 KiB
Go
103 lines
4.1 KiB
Go
package assets_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
|
"io/fs"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/promptkit"
|
|
|
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
|
)
|
|
|
|
func TestCombatSemanticsPromptKeepsItsSceneLocalPolicyAndInput(t *testing.T) {
|
|
_, engine := loadNativeAssets(t)
|
|
const transcript = `{"sentinel":"combat-semantics-transcript"}`
|
|
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
|
PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only",
|
|
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(transcript)},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer handle.Discard()
|
|
details := handle.Details()
|
|
if details.SelectedProfileID != "notarius/dnd-extraction" || details.StructuredOutput == nil {
|
|
t.Fatal("native validator defaults/output contract lost")
|
|
}
|
|
var all string
|
|
for _, message := range details.Messages {
|
|
all += message.Content
|
|
if strings.Contains(message.Content, transcript) && (message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral) {
|
|
t.Fatal("chunk transcript native cache hint lost")
|
|
}
|
|
}
|
|
for _, content := range []string{"substantive active combat materially organizes the", "Do not judge the scene title", transcript} {
|
|
if strings.Count(all, content) != 1 {
|
|
t.Fatalf("scene-local context not rendered once: %q", content)
|
|
}
|
|
}
|
|
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only"}); err == nil {
|
|
t.Fatal("native validator accepted missing transcript")
|
|
}
|
|
}
|
|
|
|
func TestCombatSemanticsResponseSchemaRetainsClassificationAndExplanationRules(t *testing.T) {
|
|
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json")
|
|
if err != nil {
|
|
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
|
}
|
|
for _, classification := range []string{"combat", "non_combat"} {
|
|
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema); err != nil {
|
|
t.Fatalf("valid %q classification rejected: %v", classification, err)
|
|
}
|
|
}
|
|
for _, test := range []struct {
|
|
name string
|
|
response map[string]any
|
|
}{
|
|
{name: "missing explanation", response: map[string]any{"classification": "combat"}},
|
|
{name: "missing classification", response: map[string]any{"explanation": "The chunk contains active combat."}},
|
|
{name: "unknown property", response: map[string]any{"classification": "combat", "explanation": "The chunk contains active combat.", "confidence": 1}},
|
|
{name: "wrong classification type", response: map[string]any{"classification": 1, "explanation": "The chunk contains active combat."}},
|
|
{name: "wrong explanation type", response: map[string]any{"classification": "combat", "explanation": 1}},
|
|
{name: "unsupported classification", response: map[string]any{"classification": "uncertain", "explanation": "The classification is uncertain."}},
|
|
{name: "blank explanation", response: map[string]any{"classification": "combat", "explanation": ""}},
|
|
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
if err := validateCombatSemanticsSchema(test.response, schema); err == nil {
|
|
t.Fatal("invalid private response accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {
|
|
instanceContent, err := json.Marshal(instance)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
parsedInstance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
parsedSchema, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
compiler := jsonschema.NewCompiler()
|
|
if err := compiler.AddResource("schema.json", parsedSchema); err != nil {
|
|
return err
|
|
}
|
|
compiled, err := compiler.Compile("schema.json")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return compiled.Validate(parsedInstance)
|
|
}
|