Migrate registry extraction to native correction and explicit empty fallbacks
This commit is contained in:
7
assets/dnd/item-registry/extract/prompts/correction.md
Normal file
7
assets/dnd/item-registry/extract/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous item registry using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement registry, not only the changed records. Keep the
|
||||
original transcript and reference context and return only JSON matching the
|
||||
original response contract.
|
||||
@@ -17,7 +17,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../../shared/prompts/common-dnd-references.md
|
||||
@@ -27,9 +27,9 @@ prompts:
|
||||
content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -37,3 +37,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json
|
||||
|
||||
- id: dnd.item_registry.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/item-registry/extract/schemas/dnd_item_registry_llm.v1.json
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
Revise the previous location registry using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement registry, not only the changed records. Keep the
|
||||
original transcript and reference context and return only JSON matching the
|
||||
original response contract.
|
||||
@@ -17,7 +17,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../../shared/prompts/common-dnd-references.md
|
||||
@@ -27,9 +27,9 @@ prompts:
|
||||
content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -37,3 +37,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-registry/extract/schemas/dnd_location_registry_llm.v1.json
|
||||
|
||||
- id: dnd.location_registry.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-registry/extract/schemas/dnd_location_registry_llm.v1.json
|
||||
|
||||
7
assets/dnd/npc-registry/extract/prompts/correction.md
Normal file
7
assets/dnd/npc-registry/extract/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous NPC registry using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement registry, not only the changed records. Keep the
|
||||
original transcript and reference context and return only JSON matching the
|
||||
original response contract.
|
||||
@@ -17,7 +17,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../../shared/prompts/common-dnd-references.md
|
||||
@@ -27,9 +27,9 @@ prompts:
|
||||
content_file: ../../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -37,3 +37,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-registry/extract/schemas/dnd_npc_registry_llm.v1.json
|
||||
|
||||
- id: dnd.npc_registry.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-registry/extract/schemas/dnd_npc_registry_llm.v1.json
|
||||
|
||||
120
assets/registry_extraction_test.go
Normal file
120
assets/registry_extraction_test.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestRegistryExtractionPromptPairsRetainDomainRulesAndContracts(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
for _, test := range []struct {
|
||||
family, instruction string
|
||||
rules []string
|
||||
}{
|
||||
{"npc", "Revise the NPC citations.", []string{"proper name or a stable, individually distinguishing", "factual third-party mention", "Exclude human players"}},
|
||||
{"item", "Revise the item citations.", []string{"named unique items", "concrete reusable item types", "currency denomination", "generic weapons", "Do not use capitalization as an eligibility test"}},
|
||||
{"location", "Revise the location citations.", []string{"stable proper name or unique in-world designation", "the room", "Do not use capitalization as an eligibility test"}},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
id := "notarius/dnd." + test.family + "_registry"
|
||||
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id, APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"registry-transcript"}]}`),
|
||||
"players": promptkit.Inline("registry-players"), "party": promptkit.Inline("registry-party"), "glossary": promptkit.Inline("registry-glossary"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer initial.Discard()
|
||||
next, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id + ".repair", APIKey: "test-only", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(test.instruction)},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer next.Discard()
|
||||
first, correction := initial.Details(), next.Details()
|
||||
if first.SelectedProfileID != "notarius/dnd-extraction" || correction.SelectedProfileID != first.SelectedProfileID ||
|
||||
first.StructuredOutput == nil || correction.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, correction.OutputContract) {
|
||||
t.Fatal("registry prompt pair lost native schema/defaults")
|
||||
}
|
||||
var system, all string
|
||||
for _, message := range first.Messages {
|
||||
all += message.Content
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
if strings.Contains(message.Content, "registry-transcript") || strings.Contains(message.Content, "registry-players") {
|
||||
if message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral {
|
||||
t.Fatal("fixed input cache hint lost")
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, rule := range append(test.rules, "Transcript units are the only evidence", "Use the most specific supported in-world") {
|
||||
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
|
||||
t.Fatalf("missing enduring domain rule %q", rule)
|
||||
}
|
||||
}
|
||||
for _, input := range []string{"registry-transcript", "registry-players", "registry-party", "registry-glossary"} {
|
||||
if strings.Count(all, input) != 1 {
|
||||
t.Fatalf("fixed input not rendered once: %q", input)
|
||||
}
|
||||
}
|
||||
if len(correction.Messages) != 1 || correction.Messages[0].Role != "user" || !strings.Contains(correction.Messages[0].Content, test.instruction) ||
|
||||
strings.Contains(correction.Messages[0].Content, "registry-transcript") {
|
||||
t.Fatal("correction repeated fixed inputs instead of supplying findings/assignment")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryResponseSchemasSeparateStructureFromSemanticValidity(t *testing.T) {
|
||||
for _, test := range []struct{ family, field string }{
|
||||
{"npc", "npcs"}, {"item", "items"}, {"location", "locations"},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/"+test.family+"-registry/extract/schemas/dnd_"+test.family+"_registry_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, value := range []struct {
|
||||
name, record string
|
||||
valid bool
|
||||
}{
|
||||
{"grounded record", `{"name":"Canonical Name","source_refs":[{"start_unit_id":1,"end_unit_id":2}]}`, true},
|
||||
{"blank name and empty evidence", `{"name":"","source_refs":[]}`, true},
|
||||
{"nonpositive source candidate", `{"name":"Canonical Name","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}`, true},
|
||||
{"missing name", `{"source_refs":[]}`, false},
|
||||
{"opaque ID", `{"name":"Canonical Name","source_refs":[],"id":"assigned-later"}`, false},
|
||||
{"wrong name type", `{"name":7,"source_refs":[]}`, false},
|
||||
{"noninteger source identifier", `{"name":"Canonical Name","source_refs":[{"start_unit_id":1.5,"end_unit_id":2}]}`, false},
|
||||
} {
|
||||
t.Run(value.name, func(t *testing.T) {
|
||||
content := []byte(`{"` + test.field + `":[` + value.record + `]}`)
|
||||
if err := validateJSONSchema(content, schema); (err == nil) != value.valid {
|
||||
t.Fatalf("schema validity=%t, error=%v", value.valid, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, content := range []string{`{"` + test.field + `":[]}`, `{"` + test.field + `":null}`, `{}`} {
|
||||
var value map[string]any
|
||||
if err := json.Unmarshal([]byte(content), &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, isArray := value[test.field].([]any)
|
||||
if err := validateJSONSchema([]byte(content), schema); (err == nil) != isArray {
|
||||
t.Fatalf("collection shape %s: %v", content, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user