Migrate scene descriptions to native correction with invariant-preserving final fallbacks
This commit is contained in:
7
assets/dnd/scene-descriptions/prompts/correction.md
Normal file
7
assets/dnd/scene-descriptions/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous scene description using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return one complete replacement kind, title, and summary for the original
|
||||
transcript chunk. Do not change its boundary or invent details to justify a
|
||||
classification. Return only JSON matching the original response contract.
|
||||
@@ -17,7 +17,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -27,9 +27,9 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-scene-combat-policy.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -37,3 +37,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json
|
||||
|
||||
- id: dnd.scene_descriptions.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json
|
||||
|
||||
120
assets/scene_descriptions_test.go
Normal file
120
assets/scene_descriptions_test.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestSceneResponseSchemaUsesStrictPrivateContract(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."}
|
||||
if err := validateSceneResponse(t, valid, schema); err != nil {
|
||||
t.Fatalf("valid private response rejected: %v", err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response map[string]any
|
||||
valid bool
|
||||
}{
|
||||
{name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}},
|
||||
{name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}},
|
||||
{name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}},
|
||||
{name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}},
|
||||
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}},
|
||||
{name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true},
|
||||
{name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true},
|
||||
{name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}},
|
||||
{name: "wrong title type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}},
|
||||
{name: "wrong summary type", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": 7}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := validateSceneResponse(t, test.response, schema)
|
||||
if (err == nil) != test.valid {
|
||||
t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := validateJSONSchema([]byte(`{"kind":`), schema); err == nil {
|
||||
t.Fatal("validateJSONSchema() error = nil, want malformed JSON rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSceneResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{"combat", "narrative", "recap", "meta"} {
|
||||
response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"}
|
||||
if err := validateSceneResponse(t, response, schema); err != nil {
|
||||
t.Fatalf("supported scene kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateSceneResponse(t *testing.T, instance map[string]any, schemaContent []byte) error {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(instance)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return validateJSONSchema(content, schemaContent)
|
||||
}
|
||||
|
||||
func TestSceneDescriptionPromptPairRetainsPolicyAndFixedInputs(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "notarius/dnd.scene_descriptions", APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"text":"scene-description-transcript"}]}`),
|
||||
"players": promptkit.Inline("scene-description-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer initial.Discard()
|
||||
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "notarius/dnd.scene_descriptions.repair", APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Return kind: combat for this scene.")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer correction.Discard()
|
||||
first, next := initial.Details(), correction.Details()
|
||||
if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
|
||||
t.Fatal("scene pair lost schema/defaults")
|
||||
}
|
||||
var system, all string
|
||||
for _, message := range first.Messages {
|
||||
all += message.Content
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
}
|
||||
for _, rule := range []string{"complete chunk is the evidence boundary", "substantive active combat materially organizes the", "must not invent a proper noun"} {
|
||||
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
|
||||
t.Fatalf("missing enduring rule %q", rule)
|
||||
}
|
||||
}
|
||||
for _, material := range []string{"scene-description-transcript", "scene-description-player", "substantive active combat materially organizes the"} {
|
||||
if strings.Count(all, material) != 1 {
|
||||
t.Fatalf("fixed input/policy repeated: %q", material)
|
||||
}
|
||||
}
|
||||
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Return kind: combat for this scene.") || strings.Contains(next.Messages[0].Content, "scene-description-transcript") {
|
||||
t.Fatal("correction assignment lost findings or repeated source")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user