Migrate enemy events to native correction with fixed multi-artifact grounding
This commit is contained in:
8
assets/dnd/enemy-events/prompts/correction.md
Normal file
8
assets/dnd/enemy-events/prompts/correction.md
Normal file
@@ -0,0 +1,8 @@
|
||||
Revise the previous enemy-event list using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement event list using the original combat transcript
|
||||
and fixed contextual grounding. Include only directly established engagements and
|
||||
outcomes, retaining their source evidence and chronology. Return only JSON
|
||||
matching the original response contract.
|
||||
@@ -26,7 +26,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -36,13 +36,13 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./combat-grounding.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -50,3 +50,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json
|
||||
|
||||
- id: dnd.enemy_events.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json
|
||||
|
||||
151
assets/enemy_events_test.go
Normal file
151
assets/enemy_events_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestEnemyResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
valid := validEnemyResponse()
|
||||
content, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err != nil {
|
||||
t.Fatalf("valid response rejected: %v", err)
|
||||
}
|
||||
semantic := validEnemyResponse()
|
||||
event := semantic["events"].([]any)[0].(map[string]any)
|
||||
event["name"] = ""
|
||||
ref := event["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
content, err = json.Marshal(semantic)
|
||||
if err != nil || validateJSONSchema(content, schema) != nil {
|
||||
t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnemyResponseSchemaRejectsInvalidStructure(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(event map[string]any) { delete(event, "name") },
|
||||
func(event map[string]any) { event["kind"] = 1 },
|
||||
func(event map[string]any) { event["kind"] = "unsupported" },
|
||||
func(event map[string]any) { event["unexpected"] = true },
|
||||
func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" },
|
||||
} {
|
||||
candidate := validEnemyResponse()
|
||||
mutate(candidate["events"].([]any)[0].(map[string]any))
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err == nil {
|
||||
t.Fatal("private schema accepted structurally invalid response")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestEnemyResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{"engaged", "killed", "fled", "captured", "incapacitated"} {
|
||||
candidate := validEnemyResponse()
|
||||
candidate["events"].([]any)[0].(map[string]any)["kind"] = kind
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err != nil {
|
||||
t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validEnemyResponse() map[string]any {
|
||||
return map[string]any{"events": []any{map[string]any{
|
||||
"name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
}}}
|
||||
}
|
||||
|
||||
func TestEnemyPromptPairRetainsRequiredGroundingAndNativeDefaults(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: enemyEventPromptInputs()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer initial.Discard()
|
||||
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "notarius/dnd.enemy_events.repair", APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Combine the duplicate engagement evidence.")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer correction.Discard()
|
||||
first, next := initial.Details(), correction.Details()
|
||||
if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
|
||||
t.Fatal("enemy-event prompt pair lost schema/defaults")
|
||||
}
|
||||
var system, all string
|
||||
for _, message := range first.Messages {
|
||||
all += message.Content
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
}
|
||||
for _, rule := range []string{"Emit at most one", "A hostile creature without a registry entry is allowed.", "Do not infer an engagement or outcome", "Transcript units are the only evidence"} {
|
||||
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
|
||||
t.Fatalf("missing enduring rule %q", rule)
|
||||
}
|
||||
}
|
||||
for _, marker := range []string{"enemy-transcript", "enemy-player", "enemy-party", "enemy-glossary", "enemy-npc", "enemy-turn", "enemy-opponent"} {
|
||||
if strings.Count(all, marker) != 1 {
|
||||
t.Fatalf("fixed grounding repeated: %q", marker)
|
||||
}
|
||||
}
|
||||
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Combine the duplicate engagement evidence.") || strings.Contains(next.Messages[0].Content, "enemy-transcript") {
|
||||
t.Fatal("correction repeated fixed inputs or omitted findings")
|
||||
}
|
||||
for _, name := range []string{"npc_registry", "combat_turns", "npc_occurrences"} {
|
||||
missing := enemyEventPromptInputs()
|
||||
delete(missing, name)
|
||||
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: missing})
|
||||
if handle != nil {
|
||||
handle.Discard()
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("missing %s input=%v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
|
||||
return map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"value":"enemy-transcript"}`),
|
||||
"players": promptkit.Inline("enemy-player"),
|
||||
"party": promptkit.Inline("enemy-party"),
|
||||
"glossary": promptkit.Inline("enemy-glossary"),
|
||||
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`),
|
||||
"combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`),
|
||||
"npc_occurrences": promptkit.Inline(`{"npc_occurrences":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user