From bb3153c1ad459943606e8af8e94c45936ce80fb0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 01:13:57 +0000 Subject: [PATCH] Migrate combat turns to native correction and retain deterministic scene gating --- assets/combat_turns_test.go | 176 +++++++++++ assets/dnd/combat-turns/prompts/correction.md | 7 + assets/dnd/combat-turns/prompts/prompt.yaml | 20 +- docs/roadmap/implementation.md | 25 ++ .../dnd/extract/combatturns/extractor.go | 190 +++++------- .../dnd/extract/combatturns/extractor_test.go | 284 ++---------------- .../combatturns/native_execution_test.go | 184 ++++++++++++ .../dnd/extract/combatturns/pipeline_test.go | 130 ++++++++ .../dnd/extract/combatturns/prompt_assets.go | 75 ----- .../extract/combatturns/prompt_assets_test.go | 58 ---- .../modules/dnd/extract/combatturns/schema.go | 25 -- .../dnd/extract/combatturns/schema_test.go | 173 ----------- .../extract/combatturns/test_helpers_test.go | 186 ++++++++++++ 13 files changed, 825 insertions(+), 708 deletions(-) create mode 100644 assets/combat_turns_test.go create mode 100644 assets/dnd/combat-turns/prompts/correction.md create mode 100644 internal/modules/dnd/extract/combatturns/native_execution_test.go create mode 100644 internal/modules/dnd/extract/combatturns/pipeline_test.go delete mode 100644 internal/modules/dnd/extract/combatturns/prompt_assets.go delete mode 100644 internal/modules/dnd/extract/combatturns/prompt_assets_test.go delete mode 100644 internal/modules/dnd/extract/combatturns/schema.go delete mode 100644 internal/modules/dnd/extract/combatturns/schema_test.go create mode 100644 internal/modules/dnd/extract/combatturns/test_helpers_test.go diff --git a/assets/combat_turns_test.go b/assets/combat_turns_test.go new file mode 100644 index 00000000..8e74072b --- /dev/null +++ b/assets/combat_turns_test.go @@ -0,0 +1,176 @@ +package assets_test + +import ( + "context" + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/promptkit" +) + +func TestCombatPromptPairPreservesNativeContractAndFixedGrounding(t *testing.T) { + _, engine := loadNativeAssets(t) + initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.combat_turns", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"units":[{"text":"combat-transcript-marker"}]}`), + "npc_registry": promptkit.Inline(`{"npcs":[{"name":"combat-npc-marker"}]}`), + "party": promptkit.Inline("combat-party-marker"), + "players": promptkit.Inline(" "), + "glossary": promptkit.Inline(" "), + }, + }) + if err != nil { + t.Fatal(err) + } + defer initial.Discard() + correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.combat_turns.repair", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Fix the actor citation.")}, + }) + if err != nil { + t.Fatal(err) + } + defer correction.Discard() + first, next := initial.Details(), correction.Details() + if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) { + t.Fatal("combat native pair lost its schema/default profile") + } + var system, all string + for _, message := range first.Messages { + all += message.Content + if message.Role == "system" { + system += message.Content + } + } + for _, principle := range []string{"Unmatched actors remain permitted.", "Exclude initiative setup without a turn", "Transcript units are the only evidence", "legendary_action", "lair_action"} { + if !strings.Contains(strings.Join(strings.Fields(system), " "), principle) { + t.Fatalf("missing enduring rule %q", principle) + } + } + for _, marker := range []string{"combat-transcript-marker", "combat-npc-marker", "combat-party-marker"} { + if strings.Count(all, marker) != 1 { + t.Fatalf("fixed input repeated: %q", marker) + } + } + if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Fix the actor citation.") || strings.Contains(next.Messages[0].Content, "combat-transcript-marker") { + t.Fatal("correction did not contain only findings and a replacement assignment") + } +} + +func TestCombatResponseSchemaUsesPrivateShape(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/combat-turns/schemas/dnd_combat_turns_llm.v1.json") + if err != nil { + t.Fatal(err) + } + + valid := validCombatResponse() + content, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err != nil { + t.Fatalf("valid combat response rejected: %v", err) + } + + withSourceID := validCombatResponse() + withSourceID["combat_turns"].([]any)[0].(map[string]any)["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session-alpha" + content, err = json.Marshal(withSourceID) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err == nil { + t.Fatal("response schema accepted source_id, want private source-reference shape") + } +} + +func TestCombatResponseSchemaLeavesNonCategoricalSemanticsToValidators(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/combat-turns/schemas/dnd_combat_turns_llm.v1.json") + if err != nil { + t.Fatal(err) + } + semanticCandidate := validCombatResponse() + turn := semanticCandidate["combat_turns"].([]any)[0].(map[string]any) + turn["actor"] = "" + ref := turn["source_refs"].([]any)[0].(map[string]any) + ref["start_unit_id"] = 0 + ref["end_unit_id"] = -1 + content, err := json.Marshal(semanticCandidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err != nil { + t.Fatalf("private schema rejected non-categorical validator-owned semantics: %v", err) + } + turn["source_refs"] = []any{} + content, err = json.Marshal(semanticCandidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err != nil { + t.Fatalf("private schema rejected empty validator-owned collections: %v", err) + } +} + +func TestCombatResponseSchemaRetainsStructuralBoundary(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/combat-turns/schemas/dnd_combat_turns_llm.v1.json") + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + mutate func(map[string]any) + }{ + {name: "missing actor", mutate: func(turn map[string]any) { delete(turn, "actor") }}, + {name: "wrong actor type", mutate: func(turn map[string]any) { turn["actor"] = 1 }}, + {name: "unknown field", mutate: func(turn map[string]any) { turn["unexpected"] = true }}, + {name: "missing source refs", mutate: func(turn map[string]any) { delete(turn, "source_refs") }}, + {name: "unsupported turn kind", mutate: func(turn map[string]any) { turn["turn_kind"] = "unsupported" }}, + } { + t.Run(test.name, func(t *testing.T) { + candidate := validCombatResponse() + test.mutate(candidate["combat_turns"].([]any)[0].(map[string]any)) + content, err := json.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err == nil { + t.Fatal("private schema accepted structurally invalid response") + } + }) + } +} + +func TestCombatResponseSchemaAcceptsEverySupportedTurnKind(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/combat-turns/schemas/dnd_combat_turns_llm.v1.json") + if err != nil { + t.Fatal(err) + } + for _, kind := range []string{"turn", "reaction", "legendary_action", "lair_action", "other"} { + candidate := validCombatResponse() + candidate["combat_turns"].([]any)[0].(map[string]any)["turn_kind"] = kind + content, err := json.Marshal(candidate) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err != nil { + t.Fatalf("supported turn kind %q was rejected: %v", kind, err) + } + } +} + +func validCombatResponse() map[string]any { + return map[string]any{ + "combat_turns": []any{ + map[string]any{ + "actor": "Aria", + "turn_kind": "reaction", + "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, + }, + }, + } +} diff --git a/assets/dnd/combat-turns/prompts/correction.md b/assets/dnd/combat-turns/prompts/correction.md new file mode 100644 index 00000000..b42183f1 --- /dev/null +++ b/assets/dnd/combat-turns/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous combat-turn list using these findings: + +{{ input "guidance" }} + +Return the complete replacement list using the original transcript and actor +grounding. Preserve transcript chronology and include only established combat +events. Return only JSON matching the original response contract. diff --git a/assets/dnd/combat-turns/prompts/prompt.yaml b/assets/dnd/combat-turns/prompts/prompt.yaml index c8313796..75cc80c6 100644 --- a/assets/dnd/combat-turns/prompts/prompt.yaml +++ b/assets/dnd/combat-turns/prompts/prompt.yaml @@ -20,7 +20,7 @@ prompts: messages: - role: system content_file: ../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../shared/prompts/common-dnd-references.md @@ -30,11 +30,11 @@ prompts: content_file: ../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-extraction-evidence.md - role: user content_file: ../../shared/prompts/common-dnd-npc-registry.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -42,3 +42,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/combat-turns/schemas/dnd_combat_turns_llm.v1.json + + - id: dnd.combat_turns.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/combat-turns/schemas/dnd_combat_turns_llm.v1.json diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6bbff124..fe378729 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -946,6 +946,31 @@ correction acceptance, empty fallback, and final semantic rejection. ## Stage 21: Migrate Combat-Turn Extraction +**Status:** Complete. Combat extraction now uses one fixed source/NPC projection, +native initial/correction requests, and the shared typed producer helper. Added +the same-schema/default-profile correction pair and static prompt dependencies; +removed private completion, schema/asset registration, and manifest/fingerprint +metadata. Actor/turn-kind semantics, unmatched actors, document-order endpoint +normalization, evidence canonicalization, and exact scene eligibility remain. +Noncombat and unavailable/mismatched scene paths assess their deterministic empty +result directly, retaining the existing warning where applicable, without native +preparation, a model call, or a synthetic cache entry. Explicit outcomes inherit +incoming reuse eligibility. Permissive model failures emit only a qualified, +codec-valid empty combat list; cancellation emits no fallback. +Retained domain/gating regressions with native provider fakes and added complete +ordered-chain correction, fixed eligibility/context/history, override, zero-turn +reversed-endpoint, strict/permissive, failed-continuation, structural-failure, and +cancellation cases. Added a leaf-assembled generic-chunker combat pipeline test +for combat, narrative, recap, meta, and mismatched scenes using provider invocation +counts. Native prompt-pair and unchanged structural-schema checks run in assets. +Asset/shape tests and vet/race checks, independent response-model tests/vet/race, +and `git diff --check` pass. Full combat extractor, source/invariant validator, +and integration package checks remain blocked at setup by shared legacy helpers +importing old ConfigKit's removed YAML `plugin/limit`; native module and assembled +runtime tests are not claimed passing. Scheduled dependency cleanup and final +verification must execute them; remaining old integration fixtures retain their +assigned migration scope. + **Goal:** native combat production preserves scene eligibility and evidence rules. **Inspect/change:** `internal/modules/dnd/extract/combatturns`, combat validators, diff --git a/internal/modules/dnd/extract/combatturns/extractor.go b/internal/modules/dnd/extract/combatturns/extractor.go index f1a9f191..540fd395 100644 --- a/internal/modules/dnd/extract/combatturns/extractor.go +++ b/internal/modules/dnd/extract/combatturns/extractor.go @@ -2,8 +2,12 @@ package combatturns import ( "context" + "encoding/json" "fmt" "sort" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -13,11 +17,9 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" ) -const ( - Key = "dnd/combat-turns" - mappingPolicy = "dnd.combat_turns.extract_mapping.v3" - sceneGatePolicy = "dnd.combat_turns.scene_gate.v1" -) +const Key = "dnd/combat-turns" +const PromptID = "notarius/dnd.combat_turns" +const CorrectionPromptID = "notarius/dnd.combat_turns.repair" const ( NPCRegistryReferenceSlot = npcregistry.ReferenceSlot @@ -64,22 +66,18 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.CombatTurnList] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} type Extractor struct { - llm contracts.StructuredLLMClient - npcResolver *npcregistry.Resolver - sceneResolver *sceneregistry.Resolver - promptSHA string - responseSchemaSHA string + engine *promptkit.Engine + npcResolver *npcregistry.Resolver + sceneResolver *sceneregistry.Resolver } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") @@ -96,79 +94,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr if err != nil { return nil, extractorErrorf("prepare scene eligibility: %w", err) } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{ - llm: llmClient, - npcResolver: npcResolver, - sceneResolver: sceneResolver, - promptSHA: promptSHA, - responseSchemaSHA: responseSchema.SHA256, - }, nil + return &Extractor{engine: engine, npcResolver: npcResolver, sceneResolver: sceneResolver}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - metadata := map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "mapping_policy": mappingPolicy, - "scene_gate_policy": sceneGatePolicy, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - } - seeded := e.npcResolver.Seeded() - if seeded.Bound() { - metadata["npc_registry_digest"] = seeded.Digest() - metadata["npc_count"] = seeded.Count() - } - seededScenes := e.sceneResolver.Seeded() - if seededScenes.Bound() { - metadata["scene_eligibility_digest"] = seededScenes.EligibilityDigest() - metadata["scene_description_count"] = seededScenes.Count() - } - return metadata -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - fingerprints := []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "mapping_policy", Value: mappingPolicy}, - {Name: "scene_gate_policy", Value: sceneGatePolicy}, - } - seeded := e.npcResolver.Seeded() - fingerprints = append(fingerprints, pipeline.CheckpointFingerprint{Name: "npc_registry", Value: seeded.ProjectionDigest()}) - seededScenes := e.sceneResolver.Seeded() - fingerprints = append(fingerprints, pipeline.CheckpointFingerprint{Name: "scene_eligibility", Value: seededScenes.EligibilityDigest()}) - return fingerprints -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.CombatTurnList], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.CombatTurnList]) (contracts.TypedExtractionResult[dnd.CombatTurnList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { @@ -185,19 +123,16 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe switch match.State { case sceneregistry.MatchExact: if match.Kind != dnd.SceneKindCombat { - return contracts.TypedExtractionResult[dnd.CombatTurnList]{Value: dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}}, nil + return e.withoutModel(ctx, req, nil) } case sceneregistry.MatchMissing, sceneregistry.MatchMismatched: - return contracts.TypedExtractionResult[dnd.CombatTurnList]{ - Value: dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}, - Diagnostics: []contracts.ProducerDiagnostic{{ - Disposition: contracts.DiagnosticDispositionWarning, - Category: contracts.DiagnosticCategoryDegradation, - ReasonCode: "scene_classification_unavailable", - OccurrenceCount: 1, - Samples: []contracts.DiagnosticSample{{Scope: SceneDescriptionReferenceSlot, Message: "No exact scene classification was available; combat extraction was skipped."}}, - }}, - }, nil + return e.withoutModel(ctx, req, []contracts.ProducerDiagnostic{{ + Disposition: contracts.DiagnosticDispositionWarning, + Category: contracts.DiagnosticCategoryDegradation, + ReasonCode: "scene_classification_unavailable", + OccurrenceCount: 1, + Samples: []contracts.DiagnosticSample{{Scope: SceneDescriptionReferenceSlot, Message: "No exact scene classification was available; combat extraction was skipped."}}, + }}) default: return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("unsupported scene eligibility match state %q", match.State) } @@ -207,40 +142,59 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("resolve NPC registry: %w", err) } - var response extractionResponse - inputs := shared.PromptInputs(sourceInput, req.References) - inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput() - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: inputs, - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("complete structured output: %w", err) + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + materials[NPCRegistryReferenceSlot] = npcRegistry.PromptInput() + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.CombatTurnList, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + canonicalizeResponse(&response, order, sourceID) + value := canonicalCombatTurnList(response, sourceID) + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.CombatTurnList) (dnd.CombatTurnList, error) { + return dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("produce combat turns: %w", err) } - canonicalizeResponse(&response, order, req.Source.ID) - return contracts.TypedExtractionResult[dnd.CombatTurnList]{Value: canonicalCombatTurnList(response, req.Source.ID), ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.CombatTurnList]{Value: value, Outcome: outcome}, nil +} + +// withoutModel assesses the deterministic empty result without native preparation. +func (e *Extractor) withoutModel(ctx context.Context, req contracts.TypedExtractionRequest[dnd.CombatTurnList], diagnostics []contracts.ProducerDiagnostic) (contracts.TypedExtractionResult[dnd.CombatTurnList], error) { + empty := dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}} + value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, empty, func(context.Context) (dnd.CombatTurnList, error) { + return dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}, nil + }) + if err != nil { + return contracts.TypedExtractionResult[dnd.CombatTurnList]{}, extractorErrorf("assess skipped combat extraction: %w", err) + } + return contracts.TypedExtractionResult[dnd.CombatTurnList]{Value: value, Outcome: outcome, Diagnostics: diagnostics}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ArtifactKind: dnd.CombatTurnListKind, - ReferenceSlots: referenceSlots(), + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.CombatTurnListKind, + ReferenceSlots: referenceSlots(), } } @@ -250,7 +204,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/combatturns/extractor_test.go b/internal/modules/dnd/extract/combatturns/extractor_test.go index 75db4c1a..433ff13b 100644 --- a/internal/modules/dnd/extract/combatturns/extractor_test.go +++ b/internal/modules/dnd/extract/combatturns/extractor_test.go @@ -2,7 +2,6 @@ package combatturns import ( "context" - "encoding/json" "errors" "reflect" "strings" @@ -14,7 +13,6 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" ) func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) { @@ -54,12 +52,11 @@ func TestExtractMapsAndOrdersCombatTurnsBySourcePosition(t *testing.T) { t.Fatalf("LLM calls = %d, want one call per chunk", len(client.requests)) } request := client.requests[0] - if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.SessionID != "session-123" || request.ProfileID != "profile-combat" { - t.Fatalf("LLM request identity = %#v", request) + if request.Prompt.SessionID != "session-123" { + t.Fatal("session selection lost") } - transcript := request.Inputs["transcript"] - if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || !reflect.DeepEqual(transcript.Content, extractionRequest(t).Chunk.Content) { - t.Fatalf("transcript input = %#v, want chunk-scoped source input", transcript) + if !reflect.DeepEqual(renderedInput(t, request, "transcript"), extractionRequest(t).Chunk.Content) { + t.Fatal("chunk-scoped transcript changed") } } @@ -137,38 +134,26 @@ func TestExtractPassesReferencesAndNPCGroundingWithoutUsingItAsEvidence(t *testi if _, err := newExtractor(t, client, references).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v, want nil", err) } - inputs := client.requests[0].Inputs - if string(inputs["players"].Content) != "Alice: Aria" || string(inputs["party"].Content) != "Aria: cleric" || string(inputs["glossary"].Content) != "ogre: a large foe" { - t.Fatalf("reference inputs = %#v, want configured references", inputs) - } - registryInput := inputs[NPCRegistryReferenceSlot] - if registryInput.Name != NPCRegistryReferenceSlot || registryInput.MediaType != "application/json" || !strings.Contains(string(registryInput.Content), "Mira Thorn") { - t.Fatalf("NPC registry input = %#v, want canonical registry grounding", registryInput) - } - if strings.Contains(string(inputs["transcript"].Content), "Aria: cleric") { - t.Fatal("transcript input contains reference content") - } - if _, ok := inputs[SceneDescriptionReferenceSlot]; ok { - t.Fatalf("prompt inputs include scene descriptions: %#v", inputs) - } - metadata := newExtractor(t, &fakeCombatTurnsLLMClient{}, references).ManifestMetadata() - if metadata["npc_count"] != 1 || !strings.HasPrefix(metadata["npc_registry_digest"].(string), "sha256:") { - t.Fatalf("bound registry metadata = %#v, want digest and count", metadata) + messages := client.requests[0].Prompt.Messages + var all string + for _, message := range messages { + all += message.Content } - if metadata["scene_description_count"] != 1 || !strings.HasPrefix(metadata["scene_eligibility_digest"].(string), "sha256:") { - t.Fatalf("bound scene metadata = %#v, want digest and count", metadata) + for _, material := range []string{"Alice: Aria", "Aria: cleric", "ogre: a large foe"} { + if strings.Count(all, material) != 1 { + t.Fatalf("campaign input not rendered once: %q", material) + } } - fingerprints := newExtractor(t, &fakeCombatTurnsLLMClient{}, references).CheckpointFingerprints() - if len(fingerprints) != 6 || fingerprints[3].Name != "scene_gate_policy" || fingerprints[4].Name != "npc_registry" || fingerprints[5].Name != "scene_eligibility" { - t.Fatalf("bound fingerprints = %#v, want local identities plus scene gate and NPC registry", fingerprints) + input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot) + if string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` { + t.Fatalf("NPC grounding=%s", input) } - encoded, err := json.Marshal(metadata) - if err != nil { - t.Fatal(err) + if strings.Contains(string(renderedInput(t, client.requests[0], "transcript")), "Aria: cleric") { + t.Fatal("campaign context became evidence") } - if strings.Contains(string(encoded), "Mira Thorn") || strings.Contains(string(encoded), "session-alpha") { - t.Fatalf("metadata leaked content: %s", encoded) + if strings.Contains(all, "Combat eligibility") || strings.Contains(all, "A valid scene description") { + t.Fatal("eligibility prose became model input") } } @@ -178,16 +163,8 @@ func TestExtractUnboundRegistryUsesExactEmptyPromptAndOmitsIdentity(t *testing.T if _, err := extractor.Extract(context.Background(), extractionRequest(t)); err != nil { t.Fatalf("Extract() error = %v, want nil", err) } - input := client.requests[0].Inputs[NPCRegistryReferenceSlot] - if string(input.Content) != `{"npcs":[]}` || input.Digest == "" || input.OriginURI != "" { - t.Fatalf("unbound registry input = %#v, want exact empty prompt without identity", input) - } - metadata := extractor.ManifestMetadata() - if _, ok := metadata["npc_registry_digest"]; ok { - t.Fatalf("unbound metadata has registry digest: %#v", metadata) - } - if fingerprints := extractor.CheckpointFingerprints(); len(fingerprints) != 6 || fingerprints[4].Name != "npc_registry" || fingerprints[4].Value != input.Digest || fingerprints[5].Name != "scene_eligibility" || fingerprints[5].Value == "" { - t.Fatalf("unbound fingerprints = %#v, want empty-projection identity", fingerprints) + if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[]}` { + t.Fatalf("unbound registry=%s", input) } } @@ -203,12 +180,8 @@ func TestExtractorResolvesOperationNPCOverrideWithoutSingletonMetadata(t *testin if _, err := extractor.Extract(context.Background(), request); err != nil { t.Fatalf("Extract() error = %v", err) } - input := client.requests[0].Inputs[NPCRegistryReferenceSlot] - if input.Digest == "" || string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" { - t.Fatalf("operation NPC input = %#v, want generated canonical grounding without provenance", input) - } - if metadata := extractor.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil || metadata["scene_eligibility_digest"] != nil || metadata["scene_description_count"] != nil { - t.Fatalf("singleton metadata = %#v, want no operation-varying NPC identity", metadata) + if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` { + t.Fatalf("generated grounding=%s", input) } } @@ -320,38 +293,6 @@ func TestExtractAppliesSceneEligibilityBeforePromptConstruction(t *testing.T) { } } -func TestSceneEligibilityMetadataAndFingerprintsTrackGatingValues(t *testing.T) { - chunk := extractionRequest(t).Chunk - base := sceneDescription(chunk, dnd.SceneKindCombat) - baseExtractor := newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, base)) - metadata := baseExtractor.ManifestMetadata() - if metadata["scene_gate_policy"] != sceneGatePolicy || metadata["scene_description_count"] != 1 { - t.Fatalf("metadata = %#v, want scene gate policy and count", metadata) - } - baseDigest := fingerprintValue(t, baseExtractor.CheckpointFingerprints(), "scene_eligibility") - proseChanged := base - proseChanged.Title = "Different title" - proseChanged.Summary = "Different summary." - if got := fingerprintValue(t, newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, proseChanged)).CheckpointFingerprints(), "scene_eligibility"); got != baseDigest { - t.Fatalf("prose-only eligibility digest = %q, want %q", got, baseDigest) - } - for _, test := range []struct { - name string - change func(*dnd.SceneDescription) - }{ - {"kind", func(value *dnd.SceneDescription) { value.Kind = dnd.SceneKindNarrative }}, - {"range", func(value *dnd.SceneDescription) { value.SourceRef.StartUnitID = 3 }}, - } { - t.Run(test.name, func(t *testing.T) { - changed := base - test.change(&changed) - if got := fingerprintValue(t, newExtractor(t, &fakeCombatTurnsLLMClient{}, sceneListReferences(t, changed)).CheckpointFingerprints(), "scene_eligibility"); got == baseDigest { - t.Fatal("eligibility digest did not change") - } - }) - } -} - func TestNewRejectsMalformedNPCRegistryBeforeLLMCallWithoutContent(t *testing.T) { client := &fakeCombatTurnsLLMClient{} references := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ @@ -360,7 +301,7 @@ func TestNewRejectsMalformedNPCRegistryBeforeLLMCallWithoutContent(t *testing.T) Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private transcript detail"}`)}}, }, }} - _, err := New(client, Options{}, references) + _, err := New(newExtractor(t, client).engine, Options{}, references) if err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private transcript detail") { t.Fatalf("New() error = %v, want bounded content-free registry failure", err) } @@ -377,7 +318,7 @@ func TestNewRejectsInvalidSceneReferenceBeforeLLMCallWithoutContent(t *testing.T Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: []byte(`{"scenes":[]}`)}}, }, }} - _, err := New(client, Options{}, references) + _, err := New(newExtractor(t, client).engine, Options{}, references) if err == nil || !strings.Contains(err.Error(), "prepare scene eligibility") || strings.Contains(err.Error(), `{"scenes":[]}`) { t.Fatalf("New() error = %v, want bounded content-free scene failure", err) } @@ -394,11 +335,11 @@ func TestExtractRejectsInvalidRequestsAndWrapsProviderFailures(t *testing.T) { name string extractor *Extractor ctx context.Context - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.CombatTurnList] want string }{ {name: "nil extractor", extractor: nilExtractor, ctx: context.Background(), req: validReq, want: "extractor"}, - {name: "nil LLM client", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "LLM client"}, + {name: "nil LLM client", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "native engine"}, {name: "wrapped preflight failure", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"}, } for _, test := range tests { @@ -415,40 +356,10 @@ func TestExtractRejectsInvalidRequestsAndWrapsProviderFailures(t *testing.T) { } } -func TestExtractorManifestMetadataAndFingerprints(t *testing.T) { - extractor := newExtractor(t, &fakeCombatTurnsLLMClient{}) - metadata := extractor.ManifestMetadata() - for key, want := range map[string]string{ - "prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy, - "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, - } { - if metadata[key] != want { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) - } - } - for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { - value, ok := metadata[key].(string) - if !ok || !strings.HasPrefix(value, "sha256:") { - t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key]) - } - } - wantNames := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "scene_gate_policy": {}, "npc_registry": {}, "scene_eligibility": {}} - for _, fingerprint := range extractor.CheckpointFingerprints() { - if _, ok := wantNames[fingerprint.Name]; !ok { - t.Fatalf("unexpected fingerprint = %#v", fingerprint) - } - delete(wantNames, fingerprint.Name) - } - if len(wantNames) != 0 { - t.Fatalf("missing fingerprints = %#v", wantNames) - } -} - func TestModuleSpecAndRegistration(t *testing.T) { wantSlots := referenceSlots() got := ModuleSpec() - if got.Key != Key || got.Stage != pipeline.StageExtract || got.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || got.ArtifactKind != dnd.CombatTurnListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.combat_turns"}) || !reflect.DeepEqual(got.ReferenceSlots, wantSlots) { + if got.Key != Key || got.Stage != pipeline.StageExtract || !reflect.DeepEqual(got.Prompts, []string{PromptID, CorrectionPromptID}) || got.ArtifactKind != dnd.CombatTurnListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.combat_turns"}) || !reflect.DeepEqual(got.ReferenceSlots, wantSlots) { t.Fatalf("ModuleSpec() = %#v, want combat extractor contract", got) } got.Requires[0] = "changed" @@ -469,142 +380,3 @@ func TestModuleSpecAndRegistration(t *testing.T) { t.Fatalf("registry missing spec for %q", Key) } } - -func extractionRequest(t *testing.T) contracts.TypedExtractionRequest { - t.Helper() - doc := combatSourceDocument() - chunk := &source.Chunk{ - ID: "session-alpha:chunk:0", - SourceID: doc.ID, - Index: 0, - Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 10}, - Content: []byte(`{"units":[10,2]}`), - MediaType: "application/json", - Units: append([]source.SourceUnit(nil), doc.Units...), - } - request := contracts.TypedExtractionRequest{ - Source: doc, - Chunk: chunk, - SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), - SessionID: "session-123", - LLMProfile: "profile-combat", - } - request.References = sceneReferences(t, chunk, dnd.SceneKindCombat) - return request -} - -func combatSourceDocument() *source.SourceDocument { - return &source.SourceDocument{ - ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source", - Units: []source.SourceUnit{ - {ID: 10, Kind: "transcript_segment", Text: "Aria reacts and strikes the ogre.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 10, EndUnitID: 10}}, - {ID: 2, Kind: "transcript_segment", Text: "Borin retreats from the ogre.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}, - }, - } -} - -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { - req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"other":true}`), "sha256:other", "") - return req -} - -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { - t.Helper() - extractor, err := New(client, Options{}, references...) - if err != nil { - t.Fatalf("New() error = %v", err) - } - return extractor -} - -func npcRegistryJSON(t *testing.T) []byte { - t.Helper() - value := dnd.NPCRegistry{NPCs: []dnd.NPC{{ID: identity.DeriveID("Mira Thorn"), Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: 1, EndUnitID: 1}}}}} - content, err := npccodec.New().Encode(value) - if err != nil { - t.Fatalf("encode NPC registry: %v", err) - } - return content -} - -func sceneReferences(t *testing.T, chunk *source.Chunk, kind dnd.SceneKind) contracts.ReferenceSet { - t.Helper() - return sceneListReferences(t, sceneDescription(chunk, kind)) -} - -func sceneListReferences(t *testing.T, scenes ...dnd.SceneDescription) contracts.ReferenceSet { - t.Helper() - content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: scenes}) - if err != nil { - t.Fatalf("encode scene descriptions: %v", err) - } - return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ - SceneDescriptionReferenceSlot: { - Slot: contracts.ReferenceSlot{Name: SceneDescriptionReferenceSlot}, - Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: content}}, - }, - }} -} - -func sceneDescription(chunk *source.Chunk, kind dnd.SceneKind) dnd.SceneDescription { - return dnd.SceneDescription{ - ID: chunk.ID, - SourceRef: chunk.Ref, - Kind: kind, - Title: "Combat eligibility", - Summary: "A valid scene description for combat extraction.", - } -} - -func fingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name string) string { - t.Helper() - for _, fingerprint := range fingerprints { - if fingerprint.Name == name { - return fingerprint.Value - } - } - t.Fatalf("fingerprints = %#v, missing %q", fingerprints, name) - return "" -} - -type fakeCombatTurnsLLMClient struct { - response extractionResponse - err error - requests []contracts.StructuredCompletionRequest -} - -func (client *fakeCombatTurnsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) - if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err - } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") - } - *target = client.response - content, err := json.Marshal(client.response) - if err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil -} - -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) - } - req.Correction = correction - if len(req.Vars) == 0 { - req.Vars = nil - return req - } - vars := make(map[string]any, len(req.Vars)) - for key, value := range req.Vars { - vars[key] = value - } - req.Vars = vars - return req -} diff --git a/internal/modules/dnd/extract/combatturns/native_execution_test.go b/internal/modules/dnd/extract/combatturns/native_execution_test.go new file mode 100644 index 00000000..25393c4c --- /dev/null +++ b/internal/modules/dnd/extract/combatturns/native_execution_test.go @@ -0,0 +1,184 @@ +package combatturns + +import ( + "context" + "errors" + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape" + sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_relatedness" + "go.maximumdirect.net/promptkit" +) + +const missingActor = `{"combat_turns":[{"actor":"","turn_kind":"reaction","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}` +const invalidCitation = `{"combat_turns":[{"actor":"Aria","turn_kind":"reaction","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}` +const knownTurn = `{"combat_turns":[{"actor":"Aria","turn_kind":"reaction","source_refs":[{"start_unit_id":2,"end_unit_id":10}]}]}` + +func bindCombatAssessment(request *contracts.TypedExtractionRequest[dnd.CombatTurnList]) *[]string { + validators := []contracts.TypedValidator[dnd.CombatTurnList]{ + shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{}), + } + var calls []string + request.Operation.Assess = func(ctx context.Context, value dnd.CombatTurnList) (contracts.CandidateAssessment, error) { + var assessment contracts.CandidateAssessment + for _, validator := range validators { + calls = append(calls, validator.Name()) + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.CombatTurnList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value}) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) + } + return assessment, nil + } + return &calls +} + +func TestCombatNativeCorrectionKeepsEligibilityAndSourceContextFixed(t *testing.T) { + for _, first := range []string{missingActor, invalidCitation} { + request := extractionRequest(t) + request.References.Slots[NPCRegistryReferenceSlot] = contracts.ResolvedReferenceSlot{Items: []contracts.ReferenceItem{ + {SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: npcRegistryJSON(t)}, + }} + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.ReuseEligible = false + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "combat-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + calls := bindCombatAssessment(&request) + client := &fakeCombatTurnsLLMClient{responses: []string{first, knownTurn}} + client.onGenerate = func(context.Context, promptkit.GenerateRequest) { + if len(client.requests) == 1 { + // Eligibility and projection were already captured. Neither may be resolved again. + request.References.Slots[SceneDescriptionReferenceSlot].Items[0].Content[0] = '!' + request.References.Slots[NPCRegistryReferenceSlot].Items[0].Content[0] = '!' + } + } + result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) + chain := []string{shape.Key, sourcerefs.Key, relatedness.Key} + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || + !reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.CombatTurns) != 1 { + t.Fatalf("combat correction=%#v %v, validators=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared)) + } + turn := result.Value.CombatTurns[0] + if turn.Actor != "Aria" || turn.SourceRefs[0].SourceID != request.Source.ID || turn.SourceRefs[0].StartUnitID != 10 || turn.SourceRefs[0].EndUnitID != 2 { + t.Fatalf("document-order endpoints=%#v", turn) + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "combat-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native selection lost") + } + if string(renderedInput(t, call, NPCRegistryReferenceSlot)) != `{"npcs":[{"name":"Mira Thorn"}]}` { + t.Fatal("names-only projection changed") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first { + t.Fatal("native continuation lost fixed context or defective response") + } + if _, err := combatcodec.New().Encode(result.Value); err != nil { + t.Fatal(err) + } + } +} + +func TestResolvableReversedCombatEndpointsNeedNoSemanticCorrection(t *testing.T) { + request := extractionRequest(t) + request.Operation.Controls.MaxAdditionalTurns = 0 + calls := bindCombatAssessment(&request) + client := &fakeCombatTurnsLLMClient{raw: knownTurn} + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(client.requests) != 1 || len(*calls) != 3 || + result.Value.CombatTurns[0].SourceRefs[0].StartUnitID != 10 || result.Value.CombatTurns[0].SourceRefs[0].EndUnitID != 2 { + t.Fatalf("endpoint reliability=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls) + } +} + +func TestSkippedCombatIsAssessedWithoutNativePreparation(t *testing.T) { + request := extractionRequest(t) + request.References = sceneReferences(t, request.Chunk, dnd.SceneKindNarrative) + request.LLMProfile = "missing-profile-is-not-needed" + request.Operation.ReuseEligible = false + request.Operation.Controls.Cache = contracts.CacheAuto + prepared := 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ } + calls := bindCombatAssessment(&request) + client := &fakeCombatTurnsLLMClient{err: errors.New("must not execute")} + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || + result.Value.CombatTurns == nil || len(result.Value.CombatTurns) != 0 || len(*calls) != 3 || len(client.requests) != 0 || prepared != 0 { + t.Fatalf("no-model outcome=%#v %v, native=%d/%d assessments=%#v", result, err, len(client.requests), prepared, *calls) + } +} + +func TestCombatTerminalModesAndCancellation(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + response string + failures []error + assessments, runs int + wantError bool + }{ + {"strict exhaustion", contracts.ModeStrict, invalidCitation, nil, 2, 2, true}, + {"permissive exhaustion", contracts.ModePermissive, invalidCitation, nil, 2, 2, false}, + {"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false}, + {"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false}, + {"structural failure", contracts.ModePermissive, `{"combat_turns":null}`, nil, 0, 1, false}, + } { + t.Run(test.name, func(t *testing.T) { + request := extractionRequest(t) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + calls := bindCombatAssessment(&request) + runs := 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ } + client := &fakeCombatTurnsLLMClient{raw: test.response, errors: test.failures} + result, err := newExtractor(t, client).Extract(context.Background(), request) + if (err != nil) != test.wantError || len(*calls) != test.assessments*3 || runs != test.runs { + t.Fatalf("terminal=%#v %v, validators=%#v runs=%d", result, err, *calls, runs) + } + if test.wantError { + if result.Outcome.Disposition != "" { + t.Fatal("strict failure returned output") + } + return + } + if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.CombatTurns == nil || len(result.Value.CombatTurns) != 0 { + t.Fatalf("fallback=%#v", result) + } + if _, err := combatcodec.New().Encode(result.Value); err != nil { + t.Fatal(err) + } + }) + } + for _, kind := range []dnd.SceneKind{dnd.SceneKindCombat, dnd.SceneKindNarrative} { + ctx, cancel := context.WithCancel(context.Background()) + request := extractionRequest(t) + request.References = sceneReferences(t, request.Chunk, kind) + request.Operation.Controls.Mode = contracts.ModePermissive + client := &fakeCombatTurnsLLMClient{raw: knownTurn, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + if kind == dnd.SceneKindNarrative { + cancel() + } + result, err := newExtractor(t, client).Extract(ctx, request) + cancel() + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" { + t.Fatalf("cancellation=%#v %v", result, err) + } + } +} diff --git a/internal/modules/dnd/extract/combatturns/pipeline_test.go b/internal/modules/dnd/extract/combatturns/pipeline_test.go new file mode 100644 index 00000000..0d2e7947 --- /dev/null +++ b/internal/modules/dnd/extract/combatturns/pipeline_test.go @@ -0,0 +1,130 @@ +package combatturns + +import ( + "context" + "os" + "path/filepath" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" + scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape" + sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_relatedness" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" + "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" + "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" + "go.maximumdirect.net/promptkit" +) + +type combatPipelineOutput struct{} + +func (combatPipelineOutput) Key() string { return "test/combat-output" } +func (combatPipelineOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) { + return contracts.OutputResult{}, nil +} + +func TestNativeCombatPipelineHonorsSceneEligibilityWithoutSyntheticRequests(t *testing.T) { + for _, test := range []struct { + name string + kind dnd.SceneKind + mismatched bool + wantCalls int + }{ + {"combat", dnd.SceneKindCombat, false, 1}, + {"narrative", dnd.SceneKindNarrative, false, 0}, + {"recap", dnd.SceneKindRecap, false, 0}, + {"meta", dnd.SceneKindMeta, false, 0}, + {"mismatched combat", dnd.SceneKindCombat, true, 0}, + } { + t.Run(test.name, func(t *testing.T) { + client := &fakeCombatTurnsLLMClient{raw: `{"combat_turns":[{"actor":"Aria","turn_kind":"turn","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`} + engine := newExtractor(t, client).engine + registries := pipeline.Registries{ + Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), + ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), + Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), + Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), + Outputs: pipeline.NewOutputEncoderRegistry(), + } + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + check(transcript.Register(registries.Inputs)) + check(units.Register(registries.Chunkers)) + check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, combatcodec.New())) + check(Register(registries.Extractors)) + check(appendorder.RegisterTyped(registries.Mergers, dnd.CombatTurnListKind, func(values []dnd.CombatTurnList) (dnd.CombatTurnList, error) { + result := dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}} + for _, value := range values { + result.CombatTurns = append(result.CombatTurns, value.CombatTurns...) + } + return result, nil + })) + check(noop.RegisterTyped[dnd.CombatTurnList](registries.Normalizers, dnd.CombatTurnListKind)) + check(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/combat-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, + func() (contracts.OutputEncoder, error) { return combatPipelineOutput{}, nil })) + check(shape.Register(registries.Validators)) + check(sourcerefs.Register(registries.Validators)) + check(relatedness.Register(registries.Validators)) + scene := dnd.SceneDescription{ + ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "combat-session", StartUnitID: 1, EndUnitID: 1}, + Kind: test.kind, Title: "Eligibility only", Summary: "This prose must not become combat evidence.", + } + if test.mismatched { + scene.SourceRef.EndUnitID = 2 + } + content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{scene}}) + check(err) + path := filepath.Join(t.TempDir(), "scenes.json") + check(os.WriteFile(path, content, 0600)) + profile := pipeline.PipelineProfile{ + ID: "native-combat", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding("test/combat-output"), + LLMProfile: "profile-combat", + Artifacts: map[string]pipeline.ArtifactLaneProfile{"combat": { + Extract: pipeline.ModuleBinding{Module: Key, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{ + pipeline.Binding(shape.Key), pipeline.Binding(sourcerefs.Key), pipeline.Binding(relatedness.Key), + }}, References: map[string]pipeline.ReferenceSource{SceneDescriptionReferenceSlot: pipeline.ExternalReference(path)}}, + Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key), + }}, + } + catalog := pipeline.ModuleCatalog{ + Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, + Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers, + Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs, + } + resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog) + check(err) + resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{}) + check(err) + prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine}) + check(err) + preparations := 0 + result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ + Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1, + RawInput: []byte(`{"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Aria strikes the ogre."}]}`), + Execution: contracts.ExecutionOptions{OnPrepared: func(context.Context, promptkit.PreparedRun) { preparations++ }}, + }) + check(err) + if len(client.requests) != test.wantCalls || preparations != test.wantCalls || result.FallbackCount != 0 || len(result.NormalizeOutputs) != 1 { + t.Fatalf("pipeline=%#v native=%d/%d, want %d", result, len(client.requests), preparations, test.wantCalls) + } + value, err := combatcodec.New().Decode(result.NormalizeOutputs[0].Artifact.Content) + check(err) + if value.CombatTurns == nil || len(value.CombatTurns) != test.wantCalls { + t.Fatalf("combat artifact=%#v", value) + } + if test.wantCalls > 0 && (value.CombatTurns[0].Actor != "Aria" || value.CombatTurns[0].SourceRefs[0].SourceID != "combat-session") { + t.Fatalf("combat evidence=%#v", value) + } + }) + } +} diff --git a/internal/modules/dnd/extract/combatturns/prompt_assets.go b/internal/modules/dnd/extract/combatturns/prompt_assets.go deleted file mode 100644 index a9de82a2..00000000 --- a/internal/modules/dnd/extract/combatturns/prompt_assets.go +++ /dev/null @@ -1,75 +0,0 @@ -package combatturns - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.combat_turns", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-transcript-chunk.md", - "common-dnd-references.md", - "common-dnd-npc-registry.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/combat-turns") - if err != nil { - return nil, fmt.Errorf("scope combat-turn assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare combat-turn prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope combat-turn schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/combatturns/prompt_assets_test.go b/internal/modules/dnd/extract/combatturns/prompt_assets_test.go deleted file mode 100644 index 757f2a47..00000000 --- a/internal/modules/dnd/extract/combatturns/prompt_assets_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package combatturns - -import ( - "context" - "io/fs" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsAndPrepareCombatPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v", err) - } - schemaFS, err := registry.SchemaFS() - if err != nil { - t.Fatal(err) - } - if _, err := fs.ReadFile(schemaFS, "dnd_combat_turns_llm.v1.json"); err != nil { - t.Fatalf("response schema asset: %v", err) - } - hash, err := promptAssetMetadata() - if err != nil || !strings.HasPrefix(hash, "sha256:") { - t.Fatalf("promptAssetMetadata() = %q, %v; want digest", hash, err) - } -} - -func TestPromptAssetsPrepareRequiredInputs(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "combat-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-test-model"), - }) - transcript := `{"units":[{"sentinel":"combat-transcript"}]}` - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", transcript), - "players": promptkit.Inline("combat-player"), - "party": promptkit.Inline(" "), - "glossary": promptkit.Inline(" "), - "npc_registry": promptkit.Inline(`{"npcs":[{"name":"combat-npc"}]}`), - }, - }) - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_combat_turns_llm.v1.json" { - t.Fatalf("prepared prompt = %#v, want combat prompt identity and schema", prepared) - } -} diff --git a/internal/modules/dnd/extract/combatturns/schema.go b/internal/modules/dnd/extract/combatturns/schema.go deleted file mode 100644 index 5021c865..00000000 --- a/internal/modules/dnd/extract/combatturns/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package combatturns - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.combat_turns" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_combat_turns_llm") - ResponseSchemaID = "notarius.dnd.combat_turns.llm" - ResponseSchemaName = "notarius_dnd_combat_turns_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_combat_turns_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/combatturns/schema_test.go b/internal/modules/dnd/extract/combatturns/schema_test.go deleted file mode 100644 index 302e7f16..00000000 --- a/internal/modules/dnd/extract/combatturns/schema_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package combatturns - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" - - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" -) - -func TestLoadResponseSchemaUsesPrivateCombatShape(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v", err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema metadata = %#v, want private combat schema identity", schema) - } - valid := validCombatResponse() - content, err := json.Marshal(valid) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("valid combat response rejected: %v", err) - } - - withSourceID := validCombatResponse() - withSourceID["combat_turns"].([]any)[0].(map[string]any)["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session-alpha" - content, err = json.Marshal(withSourceID) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err == nil { - t.Fatal("response schema accepted source_id, want private source-reference shape") - } -} - -func TestResponseSchemaLeavesNonCategoricalSemanticsToDeterministicValidators(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - semanticCandidate := validCombatResponse() - turn := semanticCandidate["combat_turns"].([]any)[0].(map[string]any) - turn["actor"] = "" - ref := turn["source_refs"].([]any)[0].(map[string]any) - ref["start_unit_id"] = 0 - ref["end_unit_id"] = -1 - content, err := json.Marshal(semanticCandidate) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("private schema rejected non-categorical validator-owned semantics: %v", err) - } - turn["source_refs"] = []any{} - content, err = json.Marshal(semanticCandidate) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("private schema rejected empty validator-owned collections: %v", err) - } -} - -func TestResponseSchemaRetainsStructuralBoundary(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - for _, test := range []struct { - name string - mutate func(map[string]any) - }{ - {name: "missing actor", mutate: func(turn map[string]any) { delete(turn, "actor") }}, - {name: "wrong actor type", mutate: func(turn map[string]any) { turn["actor"] = 1 }}, - {name: "unknown field", mutate: func(turn map[string]any) { turn["unexpected"] = true }}, - {name: "missing source refs", mutate: func(turn map[string]any) { delete(turn, "source_refs") }}, - {name: "unsupported turn kind", mutate: func(turn map[string]any) { turn["turn_kind"] = "unsupported" }}, - } { - t.Run(test.name, func(t *testing.T) { - candidate := validCombatResponse() - test.mutate(candidate["combat_turns"].([]any)[0].(map[string]any)) - content, err := json.Marshal(candidate) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err == nil { - t.Fatal("private schema accepted structurally invalid response") - } - }) - } -} - -func TestResponseSchemaAcceptsEverySupportedTurnKind(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - for _, kind := range []string{string(dnd.CombatTurnKindTurn), string(dnd.CombatTurnKindReaction), string(dnd.CombatTurnKindLegendaryAction), string(dnd.CombatTurnKindLairAction), string(dnd.CombatTurnKindOther)} { - candidate := validCombatResponse() - candidate["combat_turns"].([]any)[0].(map[string]any)["turn_kind"] = kind - content, err := json.Marshal(candidate) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("supported turn kind %q was rejected: %v", kind, err) - } - } -} - -func TestResponseSchemaJSONIsMutationSafe(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("second schema = %s, %v; want defensive valid copy", second.JSONSchema, err) - } -} - -func TestResponseSchemaDiagnosticsOmitRawSchema(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - diagnostics := schema.DiagnosticsMap() - if diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID { - t.Fatalf("diagnostics = %#v, want schema identity", diagnostics) - } - if _, ok := diagnostics["json_schema"]; ok { - t.Fatalf("diagnostics include raw schema: %#v", diagnostics) - } -} - -func validCombatResponse() map[string]any { - return map[string]any{ - "combat_turns": []any{ - map[string]any{ - "actor": "Aria", - "turn_kind": "reaction", - "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, - }, - }, - } -} - -func validateJSONSchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", schemaDocument); err != nil { - return err - } - schema, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return schema.Validate(instance) -} diff --git a/internal/modules/dnd/extract/combatturns/test_helpers_test.go b/internal/modules/dnd/extract/combatturns/test_helpers_test.go new file mode 100644 index 00000000..7db07306 --- /dev/null +++ b/internal/modules/dnd/extract/combatturns/test_helpers_test.go @@ -0,0 +1,186 @@ +package combatturns + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" + scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func extractionRequest(t *testing.T) contracts.TypedExtractionRequest[dnd.CombatTurnList] { + t.Helper() + doc := combatSourceDocument() + chunk := &source.Chunk{ + ID: "session-alpha:chunk:0", + SourceID: doc.ID, + Index: 0, + Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 10}, + Content: []byte(`{"units":[10,2]}`), + MediaType: "application/json", + Units: append([]source.SourceUnit(nil), doc.Units...), + } + request := contracts.TypedExtractionRequest[dnd.CombatTurnList]{ + Source: doc, + Chunk: chunk, + SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), + SessionID: "session-123", + LLMProfile: "profile-combat", + Operation: contracts.ProducerOperation[dnd.CombatTurnList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.CombatTurnList) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, + } + request.References = sceneReferences(t, chunk, dnd.SceneKindCombat) + return request +} + +func combatSourceDocument() *source.SourceDocument { + return &source.SourceDocument{ + ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source", + Units: []source.SourceUnit{ + {ID: 10, Kind: "transcript_segment", Text: "Aria reacts and strikes the ogre.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 10, EndUnitID: 10}}, + {ID: 2, Kind: "transcript_segment", Text: "Borin retreats from the ogre.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}, + }, + } +} + +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.CombatTurnList]) contracts.TypedExtractionRequest[dnd.CombatTurnList] { + req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"other":true}`), "sha256:other", "") + return req +} + +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit: + profiles: + - id: profile-combat + endpoint: https://example.invalid/v1 + model: occurrence-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) + } + return extractor +} + +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name] + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } + } + t.Fatalf("missing rendered %s input", name) + return nil +} + +type fakeCombatTurnsLLMClient struct { + response extractionResponse + responses []string + raw string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) +} + +func (client *fakeCombatTurnsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } + if err := ctx.Err(); err != nil { + return nil, err + } + if client.err != nil { + return nil, client.err + } + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] + } + content := client.raw + if index < len(client.responses) { + content = client.responses[index] + } + if content == "" { + encoded, err := json.Marshal(client.response) + if err != nil { + return nil, err + } + content = string(encoded) + } + return &promptkit.GenerateResponse{Content: content}, nil +} + +func npcRegistryJSON(t *testing.T) []byte { + t.Helper() + value := dnd.NPCRegistry{NPCs: []dnd.NPC{{ID: identity.DeriveID("Mira Thorn"), Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: 1, EndUnitID: 1}}}}} + content, err := npccodec.New().Encode(value) + if err != nil { + t.Fatalf("encode NPC registry: %v", err) + } + return content +} + +func sceneReferences(t *testing.T, chunk *source.Chunk, kind dnd.SceneKind) contracts.ReferenceSet { + t.Helper() + return sceneListReferences(t, sceneDescription(chunk, kind)) +} + +func sceneListReferences(t *testing.T, scenes ...dnd.SceneDescription) contracts.ReferenceSet { + t.Helper() + content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: scenes}) + if err != nil { + t.Fatalf("encode scene descriptions: %v", err) + } + return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + SceneDescriptionReferenceSlot: { + Slot: contracts.ReferenceSlot{Name: SceneDescriptionReferenceSlot}, + Items: []contracts.ReferenceItem{{SlotName: SceneDescriptionReferenceSlot, MediaType: scenecodec.MediaType, Content: content}}, + }, + }} +} + +func sceneDescription(chunk *source.Chunk, kind dnd.SceneKind) dnd.SceneDescription { + return dnd.SceneDescription{ + ID: chunk.ID, + SourceRef: chunk.Ref, + Kind: kind, + Title: "Combat eligibility", + Summary: "A valid scene description for combat extraction.", + } +}