Migrate scene descriptions to native correction with invariant-preserving final fallbacks

This commit is contained in:
2026-11-22 01:26:49 +00:00
parent 3fa593362d
commit 433ed420f8
15 changed files with 604 additions and 459 deletions

View File

@@ -0,0 +1,7 @@
Revise the previous scene description using these findings:
{{ input "guidance" }}
Return one complete replacement kind, title, and summary for the original
transcript chunk. Do not change its boundary or invent details to justify a
classification. Return only JSON matching the original response contract.

View File

@@ -17,7 +17,7 @@ prompts:
messages:
- role: system
content_file: ../../shared/prompts/common-dnd-system.md
- role: user
- role: system
content_file: ../../shared/prompts/common-dnd-identity.md
- role: user
content_file: ../../shared/prompts/common-dnd-references.md
@@ -27,9 +27,9 @@ prompts:
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
cache_control:
type: ephemeral
- role: user
- role: system
content_file: ../../shared/prompts/common-dnd-scene-combat-policy.md
- role: user
- role: system
content_file: ./instructions.md
cache_control:
type: ephemeral
@@ -37,3 +37,17 @@ prompts:
format: json
validation_mode: json_schema
schema_path: dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json
- id: dnd.scene_descriptions.repair
default_profile: notarius/dnd-extraction
inputs:
- name: guidance
required: true
content_type: text/plain
messages:
- role: user
content_file: ./correction.md
output:
format: json
validation_mode: json_schema
schema_path: dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json

View File

@@ -0,0 +1,120 @@
package assets_test
import (
"context"
"encoding/json"
"io/fs"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/promptkit"
)
func TestSceneResponseSchemaUsesStrictPrivateContract(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json")
if err != nil {
t.Fatal(err)
}
valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."}
if err := validateSceneResponse(t, valid, schema); err != nil {
t.Fatalf("valid private response rejected: %v", err)
}
for _, test := range []struct {
name string
response map[string]any
valid bool
}{
{name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}},
{name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}},
{name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}},
{name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}},
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}},
{name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true},
{name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true},
{name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}},
{name: "wrong title type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}},
{name: "wrong summary type", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": 7}},
} {
t.Run(test.name, func(t *testing.T) {
err := validateSceneResponse(t, test.response, schema)
if (err == nil) != test.valid {
t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid)
}
})
}
if err := validateJSONSchema([]byte(`{"kind":`), schema); err == nil {
t.Fatal("validateJSONSchema() error = nil, want malformed JSON rejected")
}
}
func TestSceneResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json")
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{"combat", "narrative", "recap", "meta"} {
response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"}
if err := validateSceneResponse(t, response, schema); err != nil {
t.Fatalf("supported scene kind %q was rejected: %v", kind, err)
}
}
}
func validateSceneResponse(t *testing.T, instance map[string]any, schemaContent []byte) error {
t.Helper()
content, err := json.Marshal(instance)
if err != nil {
t.Fatal(err)
}
return validateJSONSchema(content, schemaContent)
}
func TestSceneDescriptionPromptPairRetainsPolicyAndFixedInputs(t *testing.T) {
_, engine := loadNativeAssets(t)
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scene_descriptions", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"units":[{"text":"scene-description-transcript"}]}`),
"players": promptkit.Inline("scene-description-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
},
})
if err != nil {
t.Fatal(err)
}
defer initial.Discard()
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scene_descriptions.repair", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Return kind: combat for this scene.")},
})
if err != nil {
t.Fatal(err)
}
defer correction.Discard()
first, next := initial.Details(), correction.Details()
if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
t.Fatal("scene pair lost schema/defaults")
}
var system, all string
for _, message := range first.Messages {
all += message.Content
if message.Role == "system" {
system += message.Content
}
}
for _, rule := range []string{"complete chunk is the evidence boundary", "substantive active combat materially organizes the", "must not invent a proper noun"} {
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
t.Fatalf("missing enduring rule %q", rule)
}
}
for _, material := range []string{"scene-description-transcript", "scene-description-player", "substantive active combat materially organizes the"} {
if strings.Count(all, material) != 1 {
t.Fatalf("fixed input/policy repeated: %q", material)
}
}
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Return kind: combat for this scene.") || strings.Contains(next.Messages[0].Content, "scene-description-transcript") {
t.Fatal("correction assignment lost findings or repeated source")
}
}

View File

@@ -1036,6 +1036,31 @@ enemy-event codec contract, using a native provider fake and current typed codec
## Stage 23: Migrate Scene-Description Extraction
**Status:** Complete. Scene descriptions now use native initial/correction
requests through the shared typed producer helper with fixed transcript/reference
materials and captured chunk identity/range. Added matching schema/default-profile
correction assets and static dependencies; removed private completion, schema/
asset registration, and manifest/fingerprint metadata. The existing one-scene
mapping, kind/title/summary contract, and trimming remain unchanged.
Permissive fallback construction uses only the final usable candidate and checks
the unchanged durable codec, exact current chunk identity/range, trimmed fields,
and current-source addressing. It does not rerun ordinary semantic validators,
invent a classification/prose, publish an empty collection, or recover an earlier
candidate after a result-less/structurally failed correction. Strict handling and
cancellation publish no fallback. Added native-provider coverage with the real
combat-semantics validator for ordered complete-chain correction, validator versus
producer profiles, shared run overrides/session, exact history, qualified final-only
semantic fallback, unusable final responses, failed continuation, and cancellation.
Retained mapping/reference/registration regressions and moved native prompt-pair/
unchanged-schema checks to assets. Independently runnable fallback tests cover
missing/empty/multiple scenes, identity/range/shape failures, and defensive ownership.
Asset, shape, source-reference, invariant, codec, and isolated fallback tests and
vet/race checks pass, as do focused pipeline tests and `git diff --check`.
Full extractor and combat-semantics package checks remain blocked at setup by
shared legacy helpers importing old ConfigKit's removed YAML `plugin/limit`;
their new native runtime tests are not claimed passing. Scheduled cleanup and
final assembled verification must execute them.
**Goal:** retain the required scene artifact while making fallback limits explicit.
**Inspect/change:** `internal/modules/dnd/extract/scenedescriptions`, its codec,

View File

@@ -2,9 +2,12 @@ package scenedescriptions
import (
"context"
"encoding/json"
"fmt"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -13,6 +16,8 @@ import (
)
const Key = "dnd/scene-descriptions"
const PromptID = "notarius/dnd.scene_descriptions"
const CorrectionPromptID = "notarius/dnd.scene_descriptions.repair"
var requiredCapabilities = []string{
"chunks",
@@ -41,102 +46,65 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.SceneDescriptionList] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
promptSHA string
responseSchemaSHA string
}
type Extractor struct{ engine *promptkit.Engine }
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{
llm: llmClient,
promptSHA: promptSHA,
responseSchemaSHA: responseSchema.SHA256,
}, nil
return &Extractor{engine: engine}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
return map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
"mapping_policy": mappingPolicy,
}
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("native engine must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("%w", err)
}
var response extractionResponse
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile,
SessionID: req.SessionID,
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction,
Inputs: shared.PromptInputs(sourceInput, req.References),
}, &response)
if err != nil {
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("complete structured output: %w", err)
chunk := source.Chunk{ID: req.Chunk.ID, Ref: req.Chunk.Ref}
index := source.NewDocumentIndex(req.Source)
materials := shared.PromptInputs(sourceInput, req.References)
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.SceneDescriptionList, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
value := mapResponse(response, &chunk)
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(_ context.Context, final *dnd.SceneDescriptionList) (dnd.SceneDescriptionList, error) {
return sceneFallback(final, chunk, index)
})
if err != nil {
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("capture model candidate: %w", err)
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("produce scene description: %w", err)
}
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: mapResponse(response, req.Chunk), ModelCandidate: candidate}, nil
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: value, Outcome: outcome}, nil
}
func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDescriptionList {
@@ -151,14 +119,14 @@ func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDesc
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.SceneDescriptionListKind,
ReferenceSlots: referenceSlots(),
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.SceneDescriptionListKind,
ReferenceSlots: referenceSlots(),
}
}
@@ -168,7 +136,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -32,12 +32,11 @@ func TestExtractMapsExactlyOneSceneToTheChunk(t *testing.T) {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
completed := client.requests[0]
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != "profile-scene-descriptions" || completed.SessionID != "session-123" {
t.Fatalf("LLM request identity = %#v", completed)
if completed.Prompt.SessionID != request.SessionID || result.Outcome.Disposition != contracts.ProducerAccepted {
t.Fatal("native session/outcome lost")
}
transcript := completed.Inputs["transcript"]
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(request.Chunk.Content) {
t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript)
if string(renderedInput(t, completed, "transcript")) != string(request.Chunk.Content) {
t.Fatal("chunk-scoped material changed")
}
}
@@ -47,12 +46,6 @@ func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) {
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
for _, slot := range []string{"players", "party", "glossary"} {
if got := string(client.requests[0].Inputs[slot].Content); got != " " {
t.Fatalf("empty %s input = %q, want explicit placeholder", slot, got)
}
}
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Mira")}}},
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Mira: ranger")}}},
@@ -61,23 +54,21 @@ func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) {
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
inputs := client.requests[1].Inputs
if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" {
t.Fatalf("reference inputs = %#v", inputs)
var text string
for _, message := range client.requests[1].Prompt.Messages {
text += message.Content
}
if strings.Contains(string(inputs["transcript"].Content), "local title") {
t.Fatal("transcript input contains reference content")
for _, reference := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: local title"} {
if strings.Count(text, reference) != 1 {
t.Fatalf("optional input not rendered once: %q", reference)
}
}
if strings.Contains(string(renderedInput(t, client.requests[1], "transcript")), "local title") {
t.Fatal("campaign context became evidence")
}
}
func TestExtractReturnsSemanticallyInvalidResponseForDeterministicValidation(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v", err)
}
if err := validateJSONSchema(t, map[string]any{"kind": "narrative", "title": " ", "summary": ""}, schema.JSONSchema); err != nil {
t.Fatalf("semantic candidate rejected by private schema: %v", err)
}
client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{
Kind: dnd.SceneKindNarrative, Title: " ", Summary: "",
}}
@@ -98,11 +89,11 @@ func TestExtractValidatesRequestsAndSurfacesProviderFailures(t *testing.T) {
for _, test := range []struct {
name string
extractor *Extractor
req contracts.TypedExtractionRequest
req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]
want string
}{
{name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"},
{name: "nil LLM client", extractor: &Extractor{}, req: request, want: "LLM client"},
{name: "nil LLM client", extractor: &Extractor{}, req: request, want: "native engine"},
{name: "wrapped preflight failure", extractor: extractor, req: mismatchedSourceInputRequest(request), want: "must match chunk"},
} {
t.Run(test.name, func(t *testing.T) {

View File

@@ -0,0 +1,35 @@
package scenedescriptions
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
)
// sceneFallback checks construction invariants, not the configured validator chain.
func sceneFallback(final *dnd.SceneDescriptionList, chunk source.Chunk, index source.DocumentIndex) (dnd.SceneDescriptionList, error) {
if final == nil {
return dnd.SceneDescriptionList{}, fmt.Errorf("no usable final scene description is available")
}
if len(final.Scenes) != 1 {
return dnd.SceneDescriptionList{}, fmt.Errorf("scene fallback requires exactly one final scene")
}
scene := final.Scenes[0]
if scene.ID != chunk.ID || scene.SourceRef != chunk.Ref {
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene does not match the current chunk identity and range")
}
if scene.ID != strings.TrimSpace(scene.ID) || scene.SourceRef.SourceID != strings.TrimSpace(scene.SourceRef.SourceID) ||
scene.Title != strings.TrimSpace(scene.Title) || scene.Summary != strings.TrimSpace(scene.Summary) {
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene fields must be trimmed")
}
if _, err := scenecodec.New().Encode(*final); err != nil {
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene cannot be published: %w", err)
}
if err := index.ValidateRef(scene.SourceRef); err != nil {
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene source range is unusable: %w", err)
}
return dnd.SceneDescriptionList{Scenes: append([]dnd.SceneDescription(nil), final.Scenes...)}, nil
}

View File

@@ -0,0 +1,61 @@
package scenedescriptions
import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestSceneFallbackRequiresOneUsableFinalSceneWithCurrentIdentity(t *testing.T) {
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1}, {ID: 2}}}
chunk := source.Chunk{ID: "chunk", Ref: source.SourceRef{SourceID: "session", StartUnitID: 1, EndUnitID: 2}}
index := source.NewDocumentIndex(doc)
valid := dnd.SceneDescription{ID: chunk.ID, SourceRef: chunk.Ref, Kind: dnd.SceneKindNarrative, Title: "Arrival", Summary: "The party arrives."}
for _, test := range []struct {
name string
change func(*dnd.SceneDescriptionList, *source.Chunk)
}{
{"empty", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes = []dnd.SceneDescription{} }},
{"many", func(value *dnd.SceneDescriptionList, _ *source.Chunk) {
value.Scenes = append(value.Scenes, value.Scenes[0])
}},
{"blank title", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Title = "" }},
{"blank summary", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Summary = "" }},
{"unsupported kind", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Kind = "unknown" }},
{"wrong identity", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].ID = "another-chunk" }},
{"partial range", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].SourceRef.EndUnitID = 1 }},
{"wrong source", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].SourceRef.SourceID = "other" }},
{"unknown endpoint", func(value *dnd.SceneDescriptionList, current *source.Chunk) {
value.Scenes[0].SourceRef.EndUnitID = 99
current.Ref = value.Scenes[0].SourceRef
}},
{"untrimmed identity", func(value *dnd.SceneDescriptionList, current *source.Chunk) {
value.Scenes[0].ID = " chunk "
current.ID = value.Scenes[0].ID
}},
{"untrimmed prose", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Summary = " Untrimmed " }},
} {
t.Run(test.name, func(t *testing.T) {
value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{valid}}
current := chunk
test.change(&value, &current)
if result, err := sceneFallback(&value, current, index); err == nil || result.Scenes != nil {
t.Fatalf("unusable fallback=%#v %v", result, err)
}
})
}
if result, err := sceneFallback(nil, chunk, index); err == nil || result.Scenes != nil {
t.Fatalf("missing final scene=%#v %v", result, err)
}
value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{valid}}
result, err := sceneFallback(&value, chunk, index)
if err != nil || !reflect.DeepEqual(result, value) {
t.Fatalf("usable final scene=%#v %v", result, err)
}
result.Scenes[0].Title = "changed"
if value.Scenes[0] != valid {
t.Fatal("fallback aliases the candidate or alters its classification/prose")
}
}

View File

@@ -0,0 +1,184 @@
package scenedescriptions
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
semantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const narrativeScene = `{"kind":"narrative","title":"The watchtower","summary":"The party faces the bandits."}`
const combatScene = `{"kind":"combat","title":"Watchtower fight","summary":"Bandits attack and the party drives them back."}`
const blankScene = `{"kind":"combat","title":"","summary":""}`
const combatJudgment = `{"classification":"combat","explanation":"Bandits attack the party during the current scene."}`
func bindSceneAssessment(t *testing.T, extractor *Extractor, request *contracts.TypedExtractionRequest[dnd.SceneDescriptionList], withSemantic bool) *[]string {
t.Helper()
validators := []contracts.TypedValidator[dnd.SceneDescriptionList]{shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{})}
if withSemantic {
validator, err := semantics.New(extractor.engine, semantics.Options{})
if err != nil {
t.Fatal(err)
}
validators = append(validators, validator)
}
var calls []string
request.Operation.Assess = func(ctx context.Context, value dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
var assessment contracts.CandidateAssessment
for _, validator := range validators {
calls = append(calls, validator.Name())
profile := request.LLMProfile
if validator.Name() == semantics.Key {
profile = "scene-validator-profile"
}
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.SceneDescriptionList]{
Stage: "extract", Source: request.Source, Chunk: request.Chunk, SourceInput: request.SourceInput,
LLMProfile: profile, SessionID: request.SessionID, Execution: request.Operation.Execution, Value: value,
})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
}
return assessment, nil
}
return &calls
}
func TestSceneCorrectionUsesRealNativeCombatJudgmentsAndPreservesSourceIdentity(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "run-wide-scene-model"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, combatScene, combatJudgment}}
extractor := newExtractor(t, client)
calls := bindSceneAssessment(t, extractor, &request, true)
result, err := extractor.Extract(context.Background(), request)
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, semantics.Key}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 4 || len(prepared) != 4 || len(result.Value.Scenes) != 1 {
t.Fatalf("scene correction=%#v %v calls=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
}
scene := result.Value.Scenes[0]
if scene.ID != request.Chunk.ID || scene.SourceRef != request.Chunk.Ref || scene.Kind != dnd.SceneKindCombat || scene.Title != "Watchtower fight" {
t.Fatalf("mapped scene=%#v", scene)
}
for i, call := range client.requests {
id, profile := PromptID, request.LLMProfile
if i == 2 {
id = CorrectionPromptID
}
if i%2 == 1 {
id, profile = semantics.PromptID, "scene-validator-profile"
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != profile || call.Target.Model != "run-wide-scene-model" || call.Prompt.SessionID != request.SessionID {
t.Fatal("producer/validator native selections lost")
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[2].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != narrativeScene {
t.Fatal("continuation includes wrong history or regenerated context")
}
if !strings.Contains(revised[len(initial)+1].Content, "Return kind: combat for this scene.") || !strings.Contains(revised[len(initial)+1].Content, "Bandits attack the party") {
t.Fatal("combat judgment guidance lost")
}
if _, err := scenecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}
func TestPermissiveSceneRetainsOnlyFinalUsableCandidateWithoutSemanticRevalidation(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
request.Operation.Controls.MaxAdditionalTurns = 1
// Both scenes remain non-combat, but the final title/summary must win.
final := `{"kind":"narrative","title":"Final watchtower description","summary":"The party is attacked by bandits."}`
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, final, combatJudgment}}
extractor := newExtractor(t, client)
calls := bindSceneAssessment(t, extractor, &request, true)
result, err := extractor.Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Outcome.FallbackReason == "" || len(client.requests) != 4 || len(*calls) != 8 || len(result.Value.Scenes) != 1 {
t.Fatalf("semantic fallback=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls)
}
scene := result.Value.Scenes[0]
if scene.Title != "Final watchtower description" || scene.Kind != dnd.SceneKindNarrative || scene.SourceRef != request.Chunk.Ref || scene.ID != request.Chunk.ID {
t.Fatalf("fallback recovered earlier scene or invented classification=%#v", scene)
}
if _, err := scenecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}
func TestSceneCannotFallbackAfterMissingOrIntrinsicallyUnusableFinalResponse(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
responses []string
failures []error
wantAssessments int
}{
{"no first response", nil, []error{failure}, 0},
{"structural failure", []string{`{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 0},
{"blank final scene", []string{blankScene, blankScene}, nil, 2},
{"failed correction", []string{blankScene}, []error{nil, failure}, 1},
// An initially usable scene cannot be recovered after a result-less correction.
{"lost usable prior scene", []string{narrativeScene}, []error{nil, failure}, 1},
{"structurally unusable revision", []string{narrativeScene, `{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 1},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
request.Operation.Controls.MaxAdditionalTurns = 1
assessments := 0
request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
assessments++
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "test/reject", Decision: contracts.AssessmentRejected,
Result: contracts.ValidationResult{ReasonCode: "test_rejection", Message: "Scene rejected.", CorrectionGuidance: "Correct the scene description."}}}}, nil
}
client := &fakeSceneDescriptionsLLMClient{responses: test.responses, errors: test.failures}
if len(test.responses) == 1 {
client.raw = test.responses[0]
}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Scenes) != 0 || assessments != test.wantAssessments {
t.Fatalf("unusable scene=%#v %v, assessments=%d", result, err, assessments)
}
})
}
}
func TestStrictSceneRejectionAndCancellationDoNotPublishFallback(t *testing.T) {
request := extractionRequest()
request.Operation.Controls.MaxAdditionalTurns = 0
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment}}
extractor := newExtractor(t, client)
bindSceneAssessment(t, extractor, &request, true)
result, err := extractor.Extract(context.Background(), request)
if err == nil || result.Outcome.Disposition != "" || len(client.requests) != 2 {
t.Fatalf("strict scene=%#v %v", result, err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request.Operation.Controls.Mode = contracts.ModePermissive
canceled := &fakeSceneDescriptionsLLMClient{raw: combatScene, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err = newExtractor(t, canceled).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(canceled.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
}

View File

@@ -1,74 +0,0 @@
package scenedescriptions
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: "dnd.scene_descriptions",
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-identity.md",
"common-dnd-references.md",
"common-dnd-transcript-chunk.md",
"common-dnd-scene-combat-policy.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions")
if err != nil {
return nil, fmt.Errorf("scope scene-description assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare scene-description prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope scene-description schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,71 +0,0 @@
package scenedescriptions
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"scene-description-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-description-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"scene-description-transcript"}`),
"players": promptkit.Inline("scene-description-player"),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
},
})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
}
const combatPolicySentinel = "substantive active combat materially organizes the"
transcriptIndex := -1
policyIndex := -1
instructionsIndex := len(prepared.Messages) - 1
policyOccurrences := 0
for index, message := range prepared.Messages {
if strings.Contains(message.Content, "scene-description-transcript") {
transcriptIndex = index
}
if count := strings.Count(message.Content, combatPolicySentinel); count > 0 {
policyIndex = index
policyOccurrences += count
}
}
if policyOccurrences != 1 || policyIndex <= transcriptIndex || policyIndex >= instructionsIndex {
t.Fatalf("combat policy appeared %d times at message %d; want once after transcript message %d and before instructions message %d", policyOccurrences, policyIndex, transcriptIndex, instructionsIndex)
}
}
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
hash, err := promptAssetMetadata()
if err != nil || !strings.HasPrefix(hash, "sha256:") {
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
}
payload, err := json.Marshal(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata())
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
}
}
}

View File

@@ -11,10 +11,10 @@ import (
)
func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v, want dependency error", err)
}
if _, err := New(&fakeSceneDescriptionsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
if _, err := New(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
t.Fatalf("New() error = %v, want reference-set error", err)
}
if got := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).Key(); got != Key {
@@ -24,7 +24,7 @@ func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
func TestModuleSpecAndReferenceSlots(t *testing.T) {
want := pipeline.ModuleSpec{
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind,
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind,
ReferenceSlots: []contracts.ReferenceSlot{
{Name: "glossary", Description: "Optional campaign glossary reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
{Name: "party", Description: "Optional party roster reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
@@ -61,32 +61,3 @@ func TestRegisterStoresTypedModuleSpecAndOptionsAreStrict(t *testing.T) {
t.Fatalf("DecodeOptions() error = %v, want strict options error", err)
}
}
func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) {
metadata := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata()
for key, want := range map[string]string{
"prompt_id": PromptID, "prompt_version": SchemaVersion,
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
"mapping_policy": mappingPolicy,
} {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key])
}
}
want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "mapping_policy": mappingPolicy}
fingerprints := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).CheckpointFingerprints()
if len(fingerprints) != len(want) {
t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want))
}
for _, fingerprint := range fingerprints {
if fingerprint.Value != want[fingerprint.Name] {
t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want)
}
}
}

View File

@@ -1,26 +0,0 @@
package scenedescriptions
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.scene_descriptions"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_descriptions_llm")
ResponseSchemaID = "notarius.dnd.scene_descriptions.llm"
ResponseSchemaName = "notarius_dnd_scene_descriptions_llm_v1"
SchemaVersion = "v1"
mappingPolicy = "dnd.scene_descriptions.mapping.v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_scene_descriptions_llm.v1.json",
})
}

View File

@@ -1,109 +0,0 @@
package scenedescriptions
import (
"bytes"
"encoding/json"
"strings"
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v, want private scene-description schema identity", schema)
}
valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."}
if err := validateJSONSchema(t, valid, schema.JSONSchema); err != nil {
t.Fatalf("valid private response rejected: %v", err)
}
for _, test := range []struct {
name string
response map[string]any
valid bool
}{
{name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}},
{name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}},
{name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}},
{name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}},
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}},
{name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true},
{name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true},
{name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}},
{name: "wrong title type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}},
{name: "wrong summary type", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": 7}},
} {
t.Run(test.name, func(t *testing.T) {
err := validateJSONSchema(t, test.response, schema.JSONSchema)
if (err == nil) != test.valid {
t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid)
}
})
}
if err := validateJSONSchemaContent([]byte(`{"kind":`), schema.JSONSchema); err == nil {
t.Fatal("validateJSONSchemaContent() error = nil, want malformed JSON rejected")
}
}
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
}
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
}
}
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{string(dnd.SceneKindCombat), string(dnd.SceneKindNarrative), string(dnd.SceneKindRecap), string(dnd.SceneKindMeta)} {
response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"}
if err := validateJSONSchema(t, response, schema.JSONSchema); err != nil {
t.Fatalf("supported scene kind %q was rejected: %v", kind, err)
}
}
}
func validateJSONSchema(t *testing.T, instance map[string]any, schemaContent []byte) error {
t.Helper()
content, err := json.Marshal(instance)
if err != nil {
t.Fatal(err)
}
return validateJSONSchemaContent(content, schemaContent)
}
func validateJSONSchemaContent(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return err
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return err
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
return err
}
compiled, err := compiler.Compile("schema.json")
if err != nil {
return err
}
return compiled.Validate(instance)
}

View File

@@ -3,14 +3,19 @@ package scenedescriptions
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func extractionRequest() contracts.TypedExtractionRequest {
func extractionRequest() contracts.TypedExtractionRequest[dnd.SceneDescriptionList] {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-alpha:chunk:0",
@@ -22,12 +27,15 @@ func extractionRequest() contracts.TypedExtractionRequest {
Units: append([]source.SourceUnit(nil), doc.Units...),
Metadata: map[string]any{"ignored": "chunk metadata"},
}
return contracts.TypedExtractionRequest{
return contracts.TypedExtractionRequest[dnd.SceneDescriptionList]{
Source: doc,
Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
SessionID: "session-123",
LLMProfile: "profile-scene-descriptions",
Operation: contracts.ProducerOperation[dnd.SceneDescriptionList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}},
}
}
@@ -45,58 +53,99 @@ func sourceDocument() *source.SourceDocument {
}
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
if err != nil {
t.Fatalf("New() error = %v, want nil", err)
}
return extractor
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]) contracts.TypedExtractionRequest[dnd.SceneDescriptionList] {
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
return req
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
profiles:
- id: profile-scene-descriptions
endpoint: https://example.invalid/v1
model: occurrence-test-model
api_key_required: false
- id: scene-validator-profile
endpoint: https://example.invalid/v1
model: validator-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
panic(err)
t.Fatal(err)
}
req.Correction = correction
if len(req.Vars) == 0 {
req.Vars = nil
return req
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
vars := make(map[string]any, len(req.Vars))
for key, value := range req.Vars {
vars[key] = value
extractor, err := New(engine, Options{}, references...)
if err != nil {
t.Fatal(err)
}
req.Vars = vars
return req
return extractor
}
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
t.Helper()
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name]
for _, message := range request.Prompt.Messages {
start := strings.Index(message.Content, "{")
if start < 0 {
continue
}
content := []byte(message.Content[start:])
var fields map[string]json.RawMessage
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
return content
}
}
t.Fatalf("missing rendered %s input", name)
return nil
}
type fakeSceneDescriptionsLLMClient struct {
response extractionResponse
err error
requests []contracts.StructuredCompletionRequest
response extractionResponse
responses []string
raw string
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeSceneDescriptionsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
func (client *fakeSceneDescriptionsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
return nil, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
content := client.raw
if index < len(client.responses) {
content = client.responses[index]
}
return contracts.StructuredCompletionResponse{Content: content}, nil
if content == "" {
encoded, err := json.Marshal(client.response)
if err != nil {
return nil, err
}
content = string(encoded)
}
return &promptkit.GenerateResponse{Content: content}, nil
}