Migrate enemy events to native correction with fixed multi-artifact grounding

This commit is contained in:
2026-11-22 01:20:46 +00:00
parent bb3153c1ad
commit 3fa593362d
14 changed files with 742 additions and 490 deletions

View File

@@ -0,0 +1,8 @@
Revise the previous enemy-event list using these findings:
{{ input "guidance" }}
Return the complete replacement event list using the original combat transcript
and fixed contextual grounding. Include only directly established engagements and
outcomes, retaining their source evidence and chronology. Return only JSON
matching the original response contract.

View File

@@ -26,7 +26,7 @@ prompts:
messages:
- role: system
content_file: ../../shared/prompts/common-dnd-system.md
- role: user
- role: system
content_file: ../../shared/prompts/common-dnd-identity.md
- role: user
content_file: ../../shared/prompts/common-dnd-references.md
@@ -36,13 +36,13 @@ prompts:
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
cache_control:
type: ephemeral
- role: user
- role: system
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
- role: user
content_file: ../../shared/prompts/common-dnd-npc-registry.md
- role: user
content_file: ./combat-grounding.md
- role: user
- role: system
content_file: ./instructions.md
cache_control:
type: ephemeral
@@ -50,3 +50,17 @@ prompts:
format: json
validation_mode: json_schema
schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json
- id: dnd.enemy_events.repair
default_profile: notarius/dnd-extraction
inputs:
- name: guidance
required: true
content_type: text/plain
messages:
- role: user
content_file: ./correction.md
output:
format: json
validation_mode: json_schema
schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json

151
assets/enemy_events_test.go Normal file
View File

@@ -0,0 +1,151 @@
package assets_test
import (
"context"
"encoding/json"
"io/fs"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/promptkit"
)
func TestEnemyResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
if err != nil {
t.Fatal(err)
}
valid := validEnemyResponse()
content, err := json.Marshal(valid)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema); err != nil {
t.Fatalf("valid response rejected: %v", err)
}
semantic := validEnemyResponse()
event := semantic["events"].([]any)[0].(map[string]any)
event["name"] = ""
ref := event["source_refs"].([]any)[0].(map[string]any)
ref["start_unit_id"] = 0
ref["end_unit_id"] = -1
content, err = json.Marshal(semantic)
if err != nil || validateJSONSchema(content, schema) != nil {
t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err)
}
}
func TestEnemyResponseSchemaRejectsInvalidStructure(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
if err != nil {
t.Fatal(err)
}
for _, mutate := range []func(map[string]any){
func(event map[string]any) { delete(event, "name") },
func(event map[string]any) { event["kind"] = 1 },
func(event map[string]any) { event["kind"] = "unsupported" },
func(event map[string]any) { event["unexpected"] = true },
func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" },
} {
candidate := validEnemyResponse()
mutate(candidate["events"].([]any)[0].(map[string]any))
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema); err == nil {
t.Fatal("private schema accepted structurally invalid response")
}
}
}
func TestEnemyResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{"engaged", "killed", "fled", "captured", "incapacitated"} {
candidate := validEnemyResponse()
candidate["events"].([]any)[0].(map[string]any)["kind"] = kind
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema); err != nil {
t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err)
}
}
}
func validEnemyResponse() map[string]any {
return map[string]any{"events": []any{map[string]any{
"name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
}}}
}
func TestEnemyPromptPairRetainsRequiredGroundingAndNativeDefaults(t *testing.T) {
_, engine := loadNativeAssets(t)
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: enemyEventPromptInputs()})
if err != nil {
t.Fatal(err)
}
defer initial.Discard()
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.enemy_events.repair", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Combine the duplicate engagement evidence.")},
})
if err != nil {
t.Fatal(err)
}
defer correction.Discard()
first, next := initial.Details(), correction.Details()
if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
t.Fatal("enemy-event prompt pair lost schema/defaults")
}
var system, all string
for _, message := range first.Messages {
all += message.Content
if message.Role == "system" {
system += message.Content
}
}
for _, rule := range []string{"Emit at most one", "A hostile creature without a registry entry is allowed.", "Do not infer an engagement or outcome", "Transcript units are the only evidence"} {
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
t.Fatalf("missing enduring rule %q", rule)
}
}
for _, marker := range []string{"enemy-transcript", "enemy-player", "enemy-party", "enemy-glossary", "enemy-npc", "enemy-turn", "enemy-opponent"} {
if strings.Count(all, marker) != 1 {
t.Fatalf("fixed grounding repeated: %q", marker)
}
}
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Combine the duplicate engagement evidence.") || strings.Contains(next.Messages[0].Content, "enemy-transcript") {
t.Fatal("correction repeated fixed inputs or omitted findings")
}
for _, name := range []string{"npc_registry", "combat_turns", "npc_occurrences"} {
missing := enemyEventPromptInputs()
delete(missing, name)
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: missing})
if handle != nil {
handle.Discard()
}
if err == nil || !strings.Contains(err.Error(), name) {
t.Fatalf("missing %s input=%v", name, err)
}
}
}
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
return map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"value":"enemy-transcript"}`),
"players": promptkit.Inline("enemy-player"),
"party": promptkit.Inline("enemy-party"),
"glossary": promptkit.Inline("enemy-glossary"),
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`),
"combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`),
"npc_occurrences": promptkit.Inline(`{"npc_occurrences":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`),
}
}

View File

@@ -992,6 +992,29 @@ invocations, not by obsolete completion-interface mocks.
## Stage 22: Migrate Enemy-Event Extraction
**Status:** Complete. Enemy extraction now uses fixed source/reference projections,
native initial/correction requests, and the shared typed producer helper. Added
same-schema/default-profile correction assets and static dependencies; removed
private completion, schema/asset registration, and manifest/fingerprint metadata.
NPC, scene, combat-turn, and combat-opponent reference compatibility, source-free
grounding, contextual subject names, engagement checks, and source-order mapping
remain unchanged. Ineligible scenes produce directly assessed empty results
without native requests, preserving existing unavailable-classification warnings.
Permissive terminal output is an explicit codec-valid empty event list; accepted
work consuming fallback references retains incoming reuse ineligibility.
Retained grounding/domain regressions with native provider fakes and added
full-chain engagement/source correction, fixed multi-artifact context/history,
run selections, empty predecessor handoffs, strict/permissive exhaustion,
failed continuation, structural failure, and cancellation cases. Added a
leaf-assembled native pipeline test retaining the intrinsic codec boundary even
with explicit empty validator chains, and migrated the existing CLI codec
contract's typed request. Native prompt-pair, required-input, and unchanged-schema
checks run in assets. Asset tests/vet/race checks, focused pipeline tests, and
`git diff --check` pass. Extractor, validator, codec, and assembled CLI package
checks remain blocked at setup by shared legacy helpers importing old ConfigKit's
removed YAML `plugin/limit`; their new runtime tests are not claimed passing.
Scheduled dependency cleanup and final verification must execute those tests.
**Goal:** preserve multi-artifact grounding while changing execution ownership.
**Inspect/change:** `internal/modules/dnd/extract/enemyevents`, grounding/mapping

View File

@@ -66,7 +66,7 @@ func (invalidEnemyEventExtractor) Key() string { return invalidEnemyEventExtract
func (invalidEnemyEventExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (invalidEnemyEventExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
func (invalidEnemyEventExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
if err := ctx.Err(); err != nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, err
}

View File

@@ -2,7 +2,11 @@ package enemyevents
import (
"context"
"encoding/json"
"fmt"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -11,11 +15,9 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const (
Key = "dnd/enemy-events"
mappingPolicy = "dnd.enemy_events.extract_mapping.v2"
sceneGatePolicy = "dnd.enemy_events.scene_gate.v1"
)
const Key = "dnd/enemy-events"
const PromptID = "notarius/dnd.enemy_events"
const CorrectionPromptID = "notarius/dnd.enemy_events.repair"
var requiredCapabilities = []string{
"chunks",
@@ -27,22 +29,17 @@ var providedCapabilities = []string{
}
var _ contracts.Extractor[dnd.EnemyEventList] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
engine *promptkit.Engine
grounding *groundingResolver
promptSHA string
responseSchemaSHA string
}
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
@@ -55,62 +52,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare grounding: %w", err)
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{
llm: llmClient,
grounding: grounding,
promptSHA: promptSHA,
responseSchemaSHA: responseSchema.SHA256,
}, nil
return &Extractor{engine: engine, grounding: grounding}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
return map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"mapping_policy": mappingPolicy,
"scene_gate_policy": sceneGatePolicy,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
}
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "scene_gate_policy", Value: sceneGatePolicy},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("native engine must not be nil")
}
if e.grounding == nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("grounding resolver must not be nil")
@@ -126,10 +80,10 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
switch match.State {
case sceneregistry.MatchExact:
if match.Kind != dnd.SceneKindCombat {
return emptyResult(), nil
return e.withoutModel(ctx, req, emptyResult())
}
case sceneregistry.MatchMissing, sceneregistry.MatchMismatched:
return unavailableSceneResult(), nil
return e.withoutModel(ctx, req, unavailableSceneResult())
default:
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("unsupported scene eligibility match state %q", match.State)
}
@@ -137,32 +91,38 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
if err != nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("resolve enemy-event grounding: %w", err)
}
inputs := shared.PromptInputs(sourceInput, req.References)
for name, input := range grounding.PromptInputs() {
inputs[name] = input
order := shared.NewSourceRefOrder(req.Source)
sourceID := req.Source.ID
materials := shared.PromptInputs(sourceInput, req.References)
for name, material := range grounding.PromptInputs() {
materials[name] = material
}
var response extractionResponse
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile,
SessionID: req.SessionID,
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction,
Inputs: inputs,
}, &response)
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.EnemyEventList, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
value := canonicalEnemyEventList(response, order, sourceID)
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(context.Context, *dnd.EnemyEventList) (dnd.EnemyEventList, error) {
return emptyResult().Value, nil
})
if err != nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("complete structured output: %w", err)
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("produce enemy events: %w", err)
}
candidate, err := shared.ModelCandidateFromResponse(completion)
if err != nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("capture model candidate: %w", err)
}
return contracts.TypedExtractionResult[dnd.EnemyEventList]{
Value: canonicalEnemyEventList(response, shared.NewSourceRefOrder(req.Source), req.Source.ID),
ModelCandidate: candidate,
}, nil
return contracts.TypedExtractionResult[dnd.EnemyEventList]{Value: value, Outcome: outcome}, nil
}
func emptyResult() contracts.TypedExtractionResult[dnd.EnemyEventList] {
@@ -181,16 +141,26 @@ func unavailableSceneResult() contracts.TypedExtractionResult[dnd.EnemyEventList
return result
}
// withoutModel assesses a deterministic skip without a native request or cache key.
func (e *Extractor) withoutModel(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList], result contracts.TypedExtractionResult[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, result.Value, func(context.Context) (dnd.EnemyEventList, error) { return emptyResult().Value, nil })
if err != nil {
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("assess skipped enemy extraction: %w", err)
}
result.Value, result.Outcome = value, outcome
return result, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.EnemyEventListKind,
ReferenceSlots: referenceSlots(),
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.EnemyEventListKind,
ReferenceSlots: referenceSlots(),
}
}
@@ -200,7 +170,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -2,12 +2,13 @@ package enemyevents
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -46,14 +47,14 @@ func TestExtractMapsEnemyEventsInSourceOrder(t *testing.T) {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
request := client.requests[0]
if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "enemy-profile" || request.SessionID != "session-123" {
t.Fatalf("LLM request identity = %#v", request)
if request.Prompt.SessionID != "session-123" {
t.Fatal("session selection lost")
}
if got := string(request.Inputs[CombatTurnReferenceSlot].Content); !strings.Contains(got, `"actor":"Ashfang"`) || strings.Contains(got, "source_ref") {
t.Fatalf("combat grounding = %s", got)
if got := string(renderedInput(t, request, CombatTurnReferenceSlot)); !strings.Contains(got, `"actor":"Ashfang"`) || strings.Contains(got, "source_ref") {
t.Fatalf("combat grounding=%s", got)
}
if got := string(request.Inputs[NPCOccurrenceReferenceSlot].Content); !strings.Contains(got, `"kind":"combat_opponent"`) || strings.Contains(got, "Aria") {
t.Fatalf("occurrence grounding = %s", got)
if got := string(renderedInput(t, request, NPCOccurrenceReferenceSlot)); !strings.Contains(got, `"kind":"combat_opponent"`) || strings.Contains(got, "Aria") {
t.Fatalf("opponent grounding=%s", got)
}
}
@@ -93,6 +94,13 @@ func TestExtractSkipsModelForIneligibleScenes(t *testing.T) {
if test.change != nil {
test.change(request.Chunk)
}
request.Operation.ReuseEligible = false
prepared, assessed := 0, 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
request.Operation.Assess = func(context.Context, dnd.EnemyEventList) (contracts.CandidateAssessment, error) {
assessed++
return contracts.CandidateAssessment{}, nil
}
result, err := newEnemyExtractor(t, client).Extract(context.Background(), request)
if err != nil {
t.Fatal(err)
@@ -100,6 +108,9 @@ func TestExtractSkipsModelForIneligibleScenes(t *testing.T) {
if len(client.requests) != 0 || result.Value.Events == nil || len(result.Value.Events) != 0 {
t.Fatalf("result = %#v, calls = %d", result, len(client.requests))
}
if result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || prepared != 0 || assessed != 1 {
t.Fatalf("deterministic skip=%#v, prepared/assessed=%d/%d", result.Outcome, prepared, assessed)
}
if test.wantWarning {
if len(result.Diagnostics) != 1 || result.Diagnostics[0].ReasonCode != "scene_classification_unavailable" || result.Diagnostics[0].Disposition != contracts.DiagnosticDispositionWarning || result.Diagnostics[0].Category != contracts.DiagnosticCategoryDegradation || result.Diagnostics[0].Samples[0].Scope != SceneDescriptionReferenceSlot {
t.Fatalf("diagnostics = %#v", result.Diagnostics)
@@ -115,21 +126,21 @@ func TestExtractReportsRequiredGroundingAndProviderFailures(t *testing.T) {
client := &fakeEnemyEventsLLMClient{}
request := enemyExtractionRequest(t)
request.References = withoutSlot(request.References, NPCRegistryReferenceSlot)
if _, err := (&Extractor{llm: client, grounding: mustGroundingResolver(t, request.References)}).Extract(context.Background(), request); err == nil || !strings.Contains(err.Error(), "NPC registry") {
if _, err := (&Extractor{engine: newEnemyExtractor(t, client).engine, grounding: mustGroundingResolver(t, request.References)}).Extract(context.Background(), request); err == nil || !strings.Contains(err.Error(), "NPC registry") {
t.Fatalf("Extract() error = %v, want required grounding context", err)
}
provider := &fakeEnemyEventsLLMClient{err: errors.New("provider unavailable")}
if _, err := newEnemyExtractor(t, provider).Extract(context.Background(), enemyExtractionRequest(t)); err == nil || !strings.Contains(err.Error(), "complete structured output") || !strings.Contains(err.Error(), "provider unavailable") {
if _, err := newEnemyExtractor(t, provider).Extract(context.Background(), enemyExtractionRequest(t)); err == nil || !strings.Contains(err.Error(), "produce enemy events") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("Extract() error = %v, want provider context", err)
}
}
func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
func TestConstructorSpecOptionsAndNativeDependencies(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v", err)
}
if _, err := New(&fakeEnemyEventsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
if _, err := New(newEnemyExtractor(t, &fakeEnemyEventsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
t.Fatalf("New() error = %v", err)
}
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
@@ -140,7 +151,7 @@ func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) {
first.Requires[0] = "changed"
first.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
second := ModuleSpec()
if second.Requires[0] != "chunks" || second.ArtifactKind != dnd.EnemyEventListKind || second.ExecutionClass != contracts.ExecutionClassLLMBacked || second.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || second.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
if second.Requires[0] != "chunks" || second.ArtifactKind != dnd.EnemyEventListKind || second.ExecutionClass != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(second.Prompts, []string{PromptID, CorrectionPromptID}) || second.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
t.Fatalf("ModuleSpec() reused mutable state: %#v", second)
}
registry := pipeline.NewExtractorRegistry()
@@ -151,82 +162,4 @@ func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) {
t.Fatalf("registered spec = %#v, %t", spec, ok)
}
extractor := newEnemyExtractor(t, &fakeEnemyEventsLLMClient{}, groundingReferences(t, "Ashfang", dnd.SceneKindCombat))
metadata := extractor.ManifestMetadata()
encoded, err := json.Marshal(metadata)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), "Ashfang") || metadata["mapping_policy"] != mappingPolicy || metadata["scene_gate_policy"] != sceneGatePolicy {
t.Fatalf("unsafe or incomplete metadata = %s", encoded)
}
fingerprints := extractor.CheckpointFingerprints()
if len(fingerprints) != 4 || fingerprints[0].Value != metadata["prompt_sha256"] || fingerprints[1].Value != metadata["response_schema_sha256"] || fingerprints[2].Value != mappingPolicy || fingerprints[3].Value != sceneGatePolicy {
t.Fatalf("fingerprints = %#v", fingerprints)
}
}
func newEnemyExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
if len(references) == 0 {
references = []contracts.ReferenceSet{groundingReferences(t, "Ashfang", dnd.SceneKindCombat)}
}
extractor, err := New(client, Options{}, references...)
if err != nil {
t.Fatal(err)
}
return extractor
}
func mustGroundingResolver(t *testing.T, references contracts.ReferenceSet) *groundingResolver {
t.Helper()
resolver, err := newGroundingResolver(references)
if err != nil {
t.Fatal(err)
}
return resolver
}
func enemyExtractionRequest(t *testing.T) contracts.TypedExtractionRequest {
t.Helper()
chunk := combatChunk()
chunk.Units = []source.SourceUnit{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}}
chunk.MediaType = "application/json"
chunk.Content = []byte(`{"id":"combat-scene","units":[1,2,3,4]}`)
return contracts.TypedExtractionRequest{
Source: &source.SourceDocument{ID: "combat-session", Units: append([]source.SourceUnit(nil), chunk.Units...)},
Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", "application/json", chunk.Content, digest(chunk.Content), "file:///combat-session.json"),
References: groundingReferences(t, "Ashfang", dnd.SceneKindCombat),
LLMProfile: "enemy-profile",
SessionID: "session-123",
}
}
type fakeEnemyEventsLLMClient struct {
response extractionResponse
err error
requests []contracts.StructuredCompletionRequest
}
func (client *fakeEnemyEventsLLMClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneEnemyRequest(request))
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func cloneEnemyRequest(request contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
request.Inputs = request.Inputs.Clone()
return request
}

View File

@@ -0,0 +1,199 @@
package enemyevents
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/engagements"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const duplicateEngagements = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":1,"end_unit_id":1}]},{"name":"ASHFANG","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
const invalidEventCitation = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
const knownEvents = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":1}]},{"name":"Orcs","kind":"fled","source_refs":[{"start_unit_id":4,"end_unit_id":4}]}]}`
func bindEnemyAssessment(request *contracts.TypedExtractionRequest[dnd.EnemyEventList]) *[]string {
validators := []contracts.TypedValidator[dnd.EnemyEventList]{
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
relatedness.New(relatedness.Options{}), engagements.New(engagements.Options{}),
}
var calls []string
request.Operation.Assess = func(ctx context.Context, value dnd.EnemyEventList) (contracts.CandidateAssessment, error) {
var assessment contracts.CandidateAssessment
for _, validator := range validators {
calls = append(calls, validator.Name())
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.EnemyEventList]{
Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value, Execution: request.Operation.Execution,
})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
}
return assessment, nil
}
return &calls
}
func TestEnemyNativeCorrectionKeepsAllGroundingFixedAndSourceFree(t *testing.T) {
for _, first := range []string{duplicateEngagements, invalidEventCitation} {
request := enemyExtractionRequest(t)
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "enemy-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
calls := bindEnemyAssessment(&request)
client := &fakeEnemyEventsLLMClient{responses: []string{first, knownEvents}}
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
if len(client.requests) == 1 {
for _, name := range []string{NPCRegistryReferenceSlot, SceneDescriptionReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} {
request.References.Slots[name].Items[0].Content[0] = '!'
}
}
}
result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request)
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, engagements.Key}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Events) != 2 || len(client.requests) != 2 || len(prepared) != 2 {
t.Fatalf("enemy correction=%#v %v, validators=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
}
if result.Value.Events[0].SourceRefs[0].StartUnitID != 1 || result.Value.Events[0].SourceRefs[0].EndUnitID != 2 || result.Value.Events[1].Name != "Orcs" || result.Value.Events[1].Kind != dnd.EnemyEventKindFled {
t.Fatalf("contextual names/evidence=%#v", result.Value)
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "enemy-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native selection lost")
}
for _, name := range []string{NPCRegistryReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} {
material := string(renderedInput(t, call, name))
if !strings.Contains(material, "Ashfang") {
t.Fatalf("missing %s grounding", name)
}
for _, forbidden := range []string{"source_ref", "source_id", "start_unit_id", "npc:sha256:", "combat-session", "summary"} {
if strings.Contains(material, forbidden) {
t.Fatalf("%s exposed %q", name, forbidden)
}
}
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
t.Fatal("native history changed fixed grounding")
}
guidance := revised[len(initial)+1].Content
if first == duplicateEngagements && !strings.Contains(guidance, "at most one") {
t.Fatal("engagement guidance lost")
}
if first == invalidEventCitation && !strings.Contains(guidance, "source range endpoints") {
t.Fatal("source guidance lost")
}
content, err := eventcodec.New().Encode(result.Value)
if err != nil {
t.Fatal(err)
}
decoded, err := eventcodec.New().Decode(content)
if err != nil || !reflect.DeepEqual(result.Value, decoded) {
t.Fatalf("event wire round trip=%#v %v", decoded, err)
}
}
}
func TestEnemyTerminalModesAndCancellation(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
failures []error
assessments, runs int
wantError bool
}{
{"strict engagement rejection", contracts.ModeStrict, duplicateEngagements, nil, 2, 2, true},
{"permissive engagement rejection", contracts.ModePermissive, duplicateEngagements, nil, 2, 2, false},
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed continuation", contracts.ModePermissive, invalidEventCitation, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"events":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := enemyExtractionRequest(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
calls := bindEnemyAssessment(&request)
runs := 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
client := &fakeEnemyEventsLLMClient{raw: test.response, errors: test.failures}
result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request)
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
t.Fatalf("terminal=%#v %v, validators=%#v runs=%d", result, err, *calls, runs)
}
if test.wantError {
if result.Outcome.Disposition != "" {
t.Fatal("strict failure published output")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 {
t.Fatalf("fallback invented events=%#v", result)
}
if _, err := eventcodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
})
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request := enemyExtractionRequest(t)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeEnemyEventsLLMClient{raw: knownEvents, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newEnemyExtractor(t, client).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" {
t.Fatalf("canceled outcome=%#v %v", result, err)
}
}
func TestEnemyAcceptsEmptyFallbackReferencesWithoutInventingEvents(t *testing.T) {
request := enemyExtractionRequest(t)
npc, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}})
if err != nil {
t.Fatal(err)
}
turns, err := combatcodec.New().Encode(dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}})
if err != nil {
t.Fatal(err)
}
occurrences, err := occurrencecodec.New().Encode(dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{}})
if err != nil {
t.Fatal(err)
}
for name, content := range map[string][]byte{NPCRegistryReferenceSlot: npc, CombatTurnReferenceSlot: turns, NPCOccurrenceReferenceSlot: occurrences} {
request.References = replaceItem(request.References, name, newReferenceItem(name, content))
}
request.Operation.ReuseEligible = false
bindEnemyAssessment(&request)
client := &fakeEnemyEventsLLMClient{raw: `{"events":[]}`}
result, err := newEnemyExtractor(t, client, contracts.ReferenceSet{}).Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 || len(client.requests) != 1 {
t.Fatalf("empty reference handoff=%#v %v", result, err)
}
}

View File

@@ -0,0 +1,120 @@
package enemyevents
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
)
type enemyPipelineOutput struct{}
func (enemyPipelineOutput) Key() string { return "test/enemy-output" }
func (enemyPipelineOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
func TestNativeEnemyPipelineRetainsIntrinsicCodecBoundaryWithEmptyValidators(t *testing.T) {
for _, name := range []string{"Ashfang", " "} {
t.Run(name, func(t *testing.T) {
client := &fakeEnemyEventsLLMClient{response: extractionResponse{Events: []enemyEventResponse{
{Name: name, Kind: "engaged", SourceRefs: []enemySourceRefResponse{{StartUnitID: 1, EndUnitID: 1}}},
}}}
engine := newEnemyExtractor(t, client).engine
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
check(transcript.Register(registries.Inputs))
check(units.Register(registries.Chunkers))
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, eventcodec.New()))
check(Register(registries.Extractors))
check(appendorder.RegisterTyped(registries.Mergers, dnd.EnemyEventListKind, func(values []dnd.EnemyEventList) (dnd.EnemyEventList, error) {
result := dnd.EnemyEventList{Events: []dnd.EnemyEvent{}}
for _, value := range values {
result.Events = append(result.Events, value.Events...)
}
return result, nil
}))
check(noop.RegisterTyped[dnd.EnemyEventList](registries.Normalizers, dnd.EnemyEventListKind))
check(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/enemy-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic},
func() (contracts.OutputEncoder, error) { return enemyPipelineOutput{}, nil }))
references := groundingReferences(t, "Ashfang", dnd.SceneKindCombat)
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{
ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "combat-session", StartUnitID: 1, EndUnitID: 1},
Kind: dnd.SceneKindCombat, Title: "Combat", Summary: "Ashfang opposes the party.",
}}})
check(err)
references = replaceItem(references, SceneDescriptionReferenceSlot, newReferenceItem(SceneDescriptionReferenceSlot, content))
referenceSources := make(map[string]pipeline.ReferenceSource)
root := t.TempDir()
for name, slot := range references.Slots {
path := filepath.Join(root, name+".json")
check(os.WriteFile(path, slot.Items[0].Content, 0600))
referenceSources[name] = pipeline.ExternalReference(path)
}
profile := pipeline.PipelineProfile{
ID: "native-enemies", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding("test/enemy-output"),
LLMProfile: "enemy-profile",
Artifacts: map[string]pipeline.ArtifactLaneProfile{"enemies": {
Extract: pipeline.ModuleBinding{Module: Key, References: referenceSources, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{}}},
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
}},
}
catalog := pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
check(err)
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
check(err)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
check(err)
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1,
RawInput: []byte(`{"segments":[{"id":1,"speaker":"GM","start":0,"end":1,"text":"Ashfang attacks the party."}]}`),
})
if len(client.requests) != 1 {
t.Fatalf("native calls=%d", len(client.requests))
}
if strings.TrimSpace(name) == "" {
if err == nil || !strings.Contains(err.Error(), "serialize accepted extract output") || !strings.Contains(err.Error(), "name must not be empty") {
t.Fatalf("intrinsic codec error=%v", err)
}
return
}
check(err)
if result.FallbackCount != 0 || len(result.NormalizeOutputs) != 1 {
t.Fatalf("assembled outcome=%#v", result)
}
value, err := eventcodec.New().Decode(result.NormalizeOutputs[0].Artifact.Content)
check(err)
if len(value.Events) != 1 || value.Events[0].Name != name || value.Events[0].SourceRefs[0].SourceID != "combat-session" {
t.Fatalf("durable events=%#v", value)
}
})
}
}

View File

@@ -1,76 +0,0 @@
package enemyevents
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: "dnd.enemy_events",
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "combat-grounding.md", Path: "prompts/combat-grounding.md"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-transcript-chunk.md",
"common-dnd-references.md",
"common-dnd-npc-registry.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/enemy-events")
if err != nil {
return nil, fmt.Errorf("scope enemy-event assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare enemy-event prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope enemy-event schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,82 +0,0 @@
package enemyevents
import (
"context"
"io/fs"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareEnemyEventPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
schemaFS, err := registry.SchemaFS()
if err != nil {
t.Fatal(err)
}
if _, err := fs.ReadFile(schemaFS, "dnd_enemy_events_llm.v1.json"); err != nil {
t.Fatalf("response schema asset: %v", err)
}
prepared := prepareEnemyEventPrompt(t)
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_enemy_events_llm.v1.json" || prepared.SelectedProfileID != "dnd-extraction" {
t.Fatalf("prepared prompt = %#v", prepared)
}
}
func TestEnemyEventPromptRequiresGroundingInputs(t *testing.T) {
engine := newEnemyEventPromptEngine(t)
for _, inputName := range []string{"npc_registry", "combat_turns", "npc_occurrences"} {
t.Run(inputName, func(t *testing.T) {
inputs := enemyEventPromptInputs()
delete(inputs, inputName)
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: inputs,
})
if err == nil || !strings.Contains(err.Error(), inputName) {
t.Fatalf("Prepare() error = %v, want required %q input", err, inputName)
}
})
}
}
func prepareEnemyEventPrompt(t *testing.T) *promptkit.PreparedRun {
t.Helper()
prepared, err := newEnemyEventPromptEngine(t).Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: enemyEventPromptInputs(),
})
if err != nil {
t.Fatal(err)
}
return prepared
}
func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine {
t.Helper()
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"dnd-extraction": promptkitfixture.Profile("http://127.0.0.1:1/v1", "enemy-test-model"),
})
}
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
return map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"value":"enemy-transcript"}`),
"players": promptkit.Inline("enemy-player"),
"party": promptkit.Inline("enemy-party"),
"glossary": promptkit.Inline("enemy-glossary"),
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`),
"combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`),
"npc_occurrences": promptkit.Inline(`{"npc_occurrences":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`),
}
}

View File

@@ -1,25 +0,0 @@
package enemyevents
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.enemy_events"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_enemy_events_llm")
ResponseSchemaID = "notarius.dnd.enemy_events.llm"
ResponseSchemaName = "notarius_dnd_enemy_events_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_enemy_events_llm.v1.json",
})
}

View File

@@ -1,117 +0,0 @@
package enemyevents
import (
"bytes"
"encoding/json"
"strings"
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v", schema)
}
valid := validEnemyResponse()
content, err := json.Marshal(valid)
if err != nil {
t.Fatal(err)
}
if err := validateEnemySchema(content, schema.JSONSchema); err != nil {
t.Fatalf("valid response rejected: %v", err)
}
semantic := validEnemyResponse()
event := semantic["events"].([]any)[0].(map[string]any)
event["name"] = ""
ref := event["source_refs"].([]any)[0].(map[string]any)
ref["start_unit_id"] = 0
ref["end_unit_id"] = -1
content, err = json.Marshal(semantic)
if err != nil || validateEnemySchema(content, schema.JSONSchema) != nil {
t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err)
}
}
func TestResponseSchemaRejectsInvalidStructure(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, mutate := range []func(map[string]any){
func(event map[string]any) { delete(event, "name") },
func(event map[string]any) { event["kind"] = 1 },
func(event map[string]any) { event["kind"] = "unsupported" },
func(event map[string]any) { event["unexpected"] = true },
func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" },
} {
candidate := validEnemyResponse()
mutate(candidate["events"].([]any)[0].(map[string]any))
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateEnemySchema(content, schema.JSONSchema); err == nil {
t.Fatal("private schema accepted structurally invalid response")
}
}
first, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
t.Fatalf("schema defensive copy = %s, %v", second.JSONSchema, err)
}
}
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{string(dnd.EnemyEventKindEngaged), string(dnd.EnemyEventKindKilled), string(dnd.EnemyEventKindFled), string(dnd.EnemyEventKindCaptured), string(dnd.EnemyEventKindIncapacitated)} {
candidate := validEnemyResponse()
candidate["events"].([]any)[0].(map[string]any)["kind"] = kind
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateEnemySchema(content, schema.JSONSchema); err != nil {
t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err)
}
}
}
func validEnemyResponse() map[string]any {
return map[string]any{"events": []any{map[string]any{
"name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
}}}
}
func validateEnemySchema(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return err
}
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return err
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", document); err != nil {
return err
}
schema, err := compiler.Compile("schema.json")
if err != nil {
return err
}
return schema.Validate(instance)
}

View File

@@ -0,0 +1,134 @@
package enemyevents
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func newEnemyExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
if len(references) == 0 {
references = []contracts.ReferenceSet{groundingReferences(t, "Ashfang", dnd.SceneKindCombat)}
}
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
profiles:
- id: enemy-profile
endpoint: https://example.invalid/v1
model: occurrence-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
extractor, err := New(engine, Options{}, references...)
if err != nil {
t.Fatal(err)
}
return extractor
}
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
t.Helper()
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "combat_turns": "combat_turns", "npc_occurrences": "npc_occurrences"}[name]
for _, message := range request.Prompt.Messages {
for offset := 0; offset < len(message.Content); offset++ {
if message.Content[offset] != '{' {
continue
}
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
var fields map[string]json.RawMessage
if decoder.Decode(&fields) == nil && fields[key] != nil {
return []byte(message.Content[offset : offset+int(decoder.InputOffset())])
}
}
}
t.Fatalf("missing rendered %s input", name)
return nil
}
type fakeEnemyEventsLLMClient struct {
response extractionResponse
responses []string
raw string
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeEnemyEventsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if client.err != nil {
return nil, client.err
}
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
content := client.raw
if index < len(client.responses) {
content = client.responses[index]
}
if content == "" {
encoded, err := json.Marshal(client.response)
if err != nil {
return nil, err
}
content = string(encoded)
}
return &promptkit.GenerateResponse{Content: content}, nil
}
func mustGroundingResolver(t *testing.T, references contracts.ReferenceSet) *groundingResolver {
t.Helper()
resolver, err := newGroundingResolver(references)
if err != nil {
t.Fatal(err)
}
return resolver
}
func enemyExtractionRequest(t *testing.T) contracts.TypedExtractionRequest[dnd.EnemyEventList] {
t.Helper()
chunk := combatChunk()
chunk.Units = []source.SourceUnit{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}}
chunk.MediaType = "application/json"
chunk.Content = []byte(`{"id":"combat-scene","units":[1,2,3,4]}`)
return contracts.TypedExtractionRequest[dnd.EnemyEventList]{
Source: &source.SourceDocument{ID: "combat-session", Units: append([]source.SourceUnit(nil), chunk.Units...)},
Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", "application/json", chunk.Content, digest(chunk.Content), "file:///combat-session.json"),
References: groundingReferences(t, "Ashfang", dnd.SceneKindCombat),
LLMProfile: "enemy-profile",
SessionID: "session-123",
Operation: contracts.ProducerOperation[dnd.EnemyEventList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.EnemyEventList) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}},
}
}