Migrate NPC and location occurrences to native correction and explicit empty fallbacks
This commit is contained in:
7
assets/dnd/location-occurrences/prompts/correction.md
Normal file
7
assets/dnd/location-occurrences/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous location occurrence list using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return a complete replacement occurrence list using the original transcript and
|
||||
registry context. Keep identity-grounding context separate from current-transcript
|
||||
evidence, and return only JSON matching the original response contract.
|
||||
@@ -20,7 +20,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -30,11 +30,11 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./location-registry.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -42,3 +42,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-occurrences/schemas/dnd_location_occurrences_llm.v1.json
|
||||
|
||||
- id: dnd.location_occurrences.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-occurrences/schemas/dnd_location_occurrences_llm.v1.json
|
||||
|
||||
7
assets/dnd/npc-occurrences/prompts/correction.md
Normal file
7
assets/dnd/npc-occurrences/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous NPC occurrence list using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return a complete replacement occurrence list using the original transcript and
|
||||
registry context. Keep identity-grounding context separate from current-transcript
|
||||
evidence, and return only JSON matching the original response contract.
|
||||
@@ -20,7 +20,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -30,11 +30,11 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -42,3 +42,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-occurrences/schemas/dnd_npc_occurrences_llm.v1.json
|
||||
|
||||
- id: dnd.npc_occurrences.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-occurrences/schemas/dnd_npc_occurrences_llm.v1.json
|
||||
|
||||
140
assets/occurrence_extraction_test.go
Normal file
140
assets/occurrence_extraction_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestOccurrencePromptPairsPreserveRegistryAndTranscriptBoundaries(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
for _, test := range []struct {
|
||||
family, registry string
|
||||
rules []string
|
||||
}{
|
||||
{"npc", `{"npcs":[{"name":"occurrence-registry-name"}]}`, []string{"Use the supplied canonical NPC", "noncombat_presence", "combat_opponent"}},
|
||||
{"location", `{"locations":[{"name":"occurrence-registry-name","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry-context"}]}]}`,
|
||||
[]string{"context supports that coreference", "must not create a registry location", "provenance must never replace current-chunk evidence", "complete ordered"}},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
id := "notarius/dnd." + test.family + "_occurrences"
|
||||
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id, APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"occurrence-transcript"}]}`),
|
||||
test.family + "_registry": promptkit.Inline(test.registry),
|
||||
"players": promptkit.Inline("occurrence-players"), "party": promptkit.Inline("occurrence-party"), "glossary": promptkit.Inline("occurrence-glossary"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer initial.Discard()
|
||||
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id + ".repair", APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Correct the unsupported occurrence.")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer correction.Discard()
|
||||
first, next := initial.Details(), correction.Details()
|
||||
if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID ||
|
||||
first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
|
||||
t.Fatal("occurrence native pair lost schema/defaults")
|
||||
}
|
||||
var system, all string
|
||||
for _, message := range first.Messages {
|
||||
all += message.Content
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
}
|
||||
for _, rule := range append(test.rules, "Transcript units are the only evidence") {
|
||||
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
|
||||
t.Fatalf("missing enduring rule %q", rule)
|
||||
}
|
||||
}
|
||||
for _, content := range []string{"occurrence-registry-name", "occurrence-transcript", "occurrence-players", "occurrence-party", "occurrence-glossary"} {
|
||||
if strings.Count(all, content) != 1 {
|
||||
t.Fatalf("input not rendered once: %q", content)
|
||||
}
|
||||
}
|
||||
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Correct the unsupported occurrence.") ||
|
||||
strings.Contains(next.Messages[0].Content, "occurrence-registry-name") {
|
||||
t.Fatal("correction repeated fixed inputs")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOccurrenceSchemasRetainSupportedKindsAndPrivateSelectors(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
family string
|
||||
kinds []string
|
||||
}{
|
||||
{"npc", []string{"mentioned", "noncombat_presence", "dialogue", "combat_ally", "combat_opponent", "other"}},
|
||||
{"location", []string{"visited", "planned", "recalled", "mentioned"}},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/"+test.family+"-occurrences/schemas/dnd_"+test.family+"_occurrences_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record := func() map[string]any {
|
||||
value := map[string]any{"name": "Canonical Name", "kind": test.kinds[0], "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}}
|
||||
if test.family == "location" {
|
||||
value["registry_refs"] = []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}
|
||||
}
|
||||
return value
|
||||
}
|
||||
check := func(value map[string]any, valid bool) {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(map[string]any{"occurrences": []any{value}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); (err == nil) != valid {
|
||||
t.Fatalf("valid=%t response=%s error=%v", valid, content, err)
|
||||
}
|
||||
}
|
||||
for _, kind := range test.kinds {
|
||||
value := record()
|
||||
value["kind"] = kind
|
||||
check(value, true)
|
||||
}
|
||||
semantic := record()
|
||||
semantic["name"] = ""
|
||||
semantic["source_refs"] = []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}}
|
||||
check(semantic, true)
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(value map[string]any) { delete(value, "name") },
|
||||
func(value map[string]any) { value["kind"] = 1 },
|
||||
func(value map[string]any) { value["kind"] = "unsupported" },
|
||||
func(value map[string]any) { value[test.family+"_id"] = "assigned-later" },
|
||||
func(value map[string]any) { value["unexpected"] = true },
|
||||
func(value map[string]any) {
|
||||
value["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned-later"
|
||||
},
|
||||
} {
|
||||
value := record()
|
||||
mutate(value)
|
||||
check(value, false)
|
||||
}
|
||||
if test.family == "location" {
|
||||
value := record()
|
||||
delete(value, "registry_refs")
|
||||
check(value, false)
|
||||
value = record()
|
||||
value["registry_refs"].([]any)[0].(map[string]any)["start_unit_id"] = 0
|
||||
check(value, false)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user