Migrate enemy events to native correction with fixed multi-artifact grounding
This commit is contained in:
8
assets/dnd/enemy-events/prompts/correction.md
Normal file
8
assets/dnd/enemy-events/prompts/correction.md
Normal file
@@ -0,0 +1,8 @@
|
||||
Revise the previous enemy-event list using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement event list using the original combat transcript
|
||||
and fixed contextual grounding. Include only directly established engagements and
|
||||
outcomes, retaining their source evidence and chronology. Return only JSON
|
||||
matching the original response contract.
|
||||
@@ -26,7 +26,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -36,13 +36,13 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
content_file: ./combat-grounding.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -50,3 +50,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json
|
||||
|
||||
- id: dnd.enemy_events.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json
|
||||
|
||||
151
assets/enemy_events_test.go
Normal file
151
assets/enemy_events_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestEnemyResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
valid := validEnemyResponse()
|
||||
content, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err != nil {
|
||||
t.Fatalf("valid response rejected: %v", err)
|
||||
}
|
||||
semantic := validEnemyResponse()
|
||||
event := semantic["events"].([]any)[0].(map[string]any)
|
||||
event["name"] = ""
|
||||
ref := event["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
content, err = json.Marshal(semantic)
|
||||
if err != nil || validateJSONSchema(content, schema) != nil {
|
||||
t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnemyResponseSchemaRejectsInvalidStructure(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(event map[string]any) { delete(event, "name") },
|
||||
func(event map[string]any) { event["kind"] = 1 },
|
||||
func(event map[string]any) { event["kind"] = "unsupported" },
|
||||
func(event map[string]any) { event["unexpected"] = true },
|
||||
func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" },
|
||||
} {
|
||||
candidate := validEnemyResponse()
|
||||
mutate(candidate["events"].([]any)[0].(map[string]any))
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err == nil {
|
||||
t.Fatal("private schema accepted structurally invalid response")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestEnemyResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/enemy-events/schemas/dnd_enemy_events_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{"engaged", "killed", "fled", "captured", "incapacitated"} {
|
||||
candidate := validEnemyResponse()
|
||||
candidate["events"].([]any)[0].(map[string]any)["kind"] = kind
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err != nil {
|
||||
t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validEnemyResponse() map[string]any {
|
||||
return map[string]any{"events": []any{map[string]any{
|
||||
"name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
}}}
|
||||
}
|
||||
|
||||
func TestEnemyPromptPairRetainsRequiredGroundingAndNativeDefaults(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: enemyEventPromptInputs()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer initial.Discard()
|
||||
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "notarius/dnd.enemy_events.repair", APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Combine the duplicate engagement evidence.")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer correction.Discard()
|
||||
first, next := initial.Details(), correction.Details()
|
||||
if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
|
||||
t.Fatal("enemy-event prompt pair lost schema/defaults")
|
||||
}
|
||||
var system, all string
|
||||
for _, message := range first.Messages {
|
||||
all += message.Content
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
}
|
||||
for _, rule := range []string{"Emit at most one", "A hostile creature without a registry entry is allowed.", "Do not infer an engagement or outcome", "Transcript units are the only evidence"} {
|
||||
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
|
||||
t.Fatalf("missing enduring rule %q", rule)
|
||||
}
|
||||
}
|
||||
for _, marker := range []string{"enemy-transcript", "enemy-player", "enemy-party", "enemy-glossary", "enemy-npc", "enemy-turn", "enemy-opponent"} {
|
||||
if strings.Count(all, marker) != 1 {
|
||||
t.Fatalf("fixed grounding repeated: %q", marker)
|
||||
}
|
||||
}
|
||||
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Combine the duplicate engagement evidence.") || strings.Contains(next.Messages[0].Content, "enemy-transcript") {
|
||||
t.Fatal("correction repeated fixed inputs or omitted findings")
|
||||
}
|
||||
for _, name := range []string{"npc_registry", "combat_turns", "npc_occurrences"} {
|
||||
missing := enemyEventPromptInputs()
|
||||
delete(missing, name)
|
||||
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.enemy_events", APIKey: "test-only", Inputs: missing})
|
||||
if handle != nil {
|
||||
handle.Discard()
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("missing %s input=%v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
|
||||
return map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"value":"enemy-transcript"}`),
|
||||
"players": promptkit.Inline("enemy-player"),
|
||||
"party": promptkit.Inline("enemy-party"),
|
||||
"glossary": promptkit.Inline("enemy-glossary"),
|
||||
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`),
|
||||
"combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`),
|
||||
"npc_occurrences": promptkit.Inline(`{"npc_occurrences":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`),
|
||||
}
|
||||
}
|
||||
@@ -992,6 +992,29 @@ invocations, not by obsolete completion-interface mocks.
|
||||
|
||||
## Stage 22: Migrate Enemy-Event Extraction
|
||||
|
||||
**Status:** Complete. Enemy extraction now uses fixed source/reference projections,
|
||||
native initial/correction requests, and the shared typed producer helper. Added
|
||||
same-schema/default-profile correction assets and static dependencies; removed
|
||||
private completion, schema/asset registration, and manifest/fingerprint metadata.
|
||||
NPC, scene, combat-turn, and combat-opponent reference compatibility, source-free
|
||||
grounding, contextual subject names, engagement checks, and source-order mapping
|
||||
remain unchanged. Ineligible scenes produce directly assessed empty results
|
||||
without native requests, preserving existing unavailable-classification warnings.
|
||||
Permissive terminal output is an explicit codec-valid empty event list; accepted
|
||||
work consuming fallback references retains incoming reuse ineligibility.
|
||||
Retained grounding/domain regressions with native provider fakes and added
|
||||
full-chain engagement/source correction, fixed multi-artifact context/history,
|
||||
run selections, empty predecessor handoffs, strict/permissive exhaustion,
|
||||
failed continuation, structural failure, and cancellation cases. Added a
|
||||
leaf-assembled native pipeline test retaining the intrinsic codec boundary even
|
||||
with explicit empty validator chains, and migrated the existing CLI codec
|
||||
contract's typed request. Native prompt-pair, required-input, and unchanged-schema
|
||||
checks run in assets. Asset tests/vet/race checks, focused pipeline tests, and
|
||||
`git diff --check` pass. Extractor, validator, codec, and assembled CLI package
|
||||
checks remain blocked at setup by shared legacy helpers importing old ConfigKit's
|
||||
removed YAML `plugin/limit`; their new runtime tests are not claimed passing.
|
||||
Scheduled dependency cleanup and final verification must execute those tests.
|
||||
|
||||
**Goal:** preserve multi-artifact grounding while changing execution ownership.
|
||||
|
||||
**Inspect/change:** `internal/modules/dnd/extract/enemyevents`, grounding/mapping
|
||||
|
||||
@@ -66,7 +66,7 @@ func (invalidEnemyEventExtractor) Key() string { return invalidEnemyEventExtract
|
||||
|
||||
func (invalidEnemyEventExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (invalidEnemyEventExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
|
||||
func (invalidEnemyEventExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, err
|
||||
}
|
||||
|
||||
@@ -2,7 +2,11 @@ package enemyevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
@@ -11,11 +15,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/enemy-events"
|
||||
mappingPolicy = "dnd.enemy_events.extract_mapping.v2"
|
||||
sceneGatePolicy = "dnd.enemy_events.scene_gate.v1"
|
||||
)
|
||||
const Key = "dnd/enemy-events"
|
||||
const PromptID = "notarius/dnd.enemy_events"
|
||||
const CorrectionPromptID = "notarius/dnd.enemy_events.repair"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
@@ -27,22 +29,17 @@ var providedCapabilities = []string{
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.EnemyEventList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
engine *promptkit.Engine
|
||||
grounding *groundingResolver
|
||||
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
@@ -55,62 +52,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("prepare grounding: %w", err)
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{
|
||||
llm: llmClient,
|
||||
grounding: grounding,
|
||||
promptSHA: promptSHA,
|
||||
responseSchemaSHA: responseSchema.SHA256,
|
||||
}, nil
|
||||
return &Extractor{engine: engine, grounding: grounding}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"mapping_policy": mappingPolicy,
|
||||
"scene_gate_policy": sceneGatePolicy,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
{Name: "scene_gate_policy", Value: sceneGatePolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if e.grounding == nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("grounding resolver must not be nil")
|
||||
@@ -126,10 +80,10 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
switch match.State {
|
||||
case sceneregistry.MatchExact:
|
||||
if match.Kind != dnd.SceneKindCombat {
|
||||
return emptyResult(), nil
|
||||
return e.withoutModel(ctx, req, emptyResult())
|
||||
}
|
||||
case sceneregistry.MatchMissing, sceneregistry.MatchMismatched:
|
||||
return unavailableSceneResult(), nil
|
||||
return e.withoutModel(ctx, req, unavailableSceneResult())
|
||||
default:
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("unsupported scene eligibility match state %q", match.State)
|
||||
}
|
||||
@@ -137,32 +91,38 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("resolve enemy-event grounding: %w", err)
|
||||
}
|
||||
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||
for name, input := range grounding.PromptInputs() {
|
||||
inputs[name] = input
|
||||
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
sourceID := req.Source.ID
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
for name, material := range grounding.PromptInputs() {
|
||||
materials[name] = material
|
||||
}
|
||||
var response extractionResponse
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction,
|
||||
Inputs: inputs,
|
||||
}, &response)
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.EnemyEventList, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
value := canonicalEnemyEventList(response, order, sourceID)
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *dnd.EnemyEventList) (dnd.EnemyEventList, error) {
|
||||
return emptyResult().Value, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("produce enemy events: %w", err)
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{
|
||||
Value: canonicalEnemyEventList(response, shared.NewSourceRefOrder(req.Source), req.Source.ID),
|
||||
ModelCandidate: candidate,
|
||||
}, nil
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func emptyResult() contracts.TypedExtractionResult[dnd.EnemyEventList] {
|
||||
@@ -181,16 +141,26 @@ func unavailableSceneResult() contracts.TypedExtractionResult[dnd.EnemyEventList
|
||||
return result
|
||||
}
|
||||
|
||||
// withoutModel assesses a deterministic skip without a native request or cache key.
|
||||
func (e *Extractor) withoutModel(ctx context.Context, req contracts.TypedExtractionRequest[dnd.EnemyEventList], result contracts.TypedExtractionResult[dnd.EnemyEventList]) (contracts.TypedExtractionResult[dnd.EnemyEventList], error) {
|
||||
value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, result.Value, func(context.Context) (dnd.EnemyEventList, error) { return emptyResult().Value, nil })
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.EnemyEventList]{}, extractorErrorf("assess skipped enemy extraction: %w", err)
|
||||
}
|
||||
result.Value, result.Outcome = value, outcome
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.EnemyEventListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.EnemyEventListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,7 +170,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -2,12 +2,13 @@ package enemyevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
@@ -46,14 +47,14 @@ func TestExtractMapsEnemyEventsInSourceOrder(t *testing.T) {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
request := client.requests[0]
|
||||
if request.StageName != Key || request.PromptID != PromptID || request.PromptVersion != SchemaVersion || request.ProfileID != "enemy-profile" || request.SessionID != "session-123" {
|
||||
t.Fatalf("LLM request identity = %#v", request)
|
||||
if request.Prompt.SessionID != "session-123" {
|
||||
t.Fatal("session selection lost")
|
||||
}
|
||||
if got := string(request.Inputs[CombatTurnReferenceSlot].Content); !strings.Contains(got, `"actor":"Ashfang"`) || strings.Contains(got, "source_ref") {
|
||||
t.Fatalf("combat grounding = %s", got)
|
||||
if got := string(renderedInput(t, request, CombatTurnReferenceSlot)); !strings.Contains(got, `"actor":"Ashfang"`) || strings.Contains(got, "source_ref") {
|
||||
t.Fatalf("combat grounding=%s", got)
|
||||
}
|
||||
if got := string(request.Inputs[NPCOccurrenceReferenceSlot].Content); !strings.Contains(got, `"kind":"combat_opponent"`) || strings.Contains(got, "Aria") {
|
||||
t.Fatalf("occurrence grounding = %s", got)
|
||||
if got := string(renderedInput(t, request, NPCOccurrenceReferenceSlot)); !strings.Contains(got, `"kind":"combat_opponent"`) || strings.Contains(got, "Aria") {
|
||||
t.Fatalf("opponent grounding=%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +94,13 @@ func TestExtractSkipsModelForIneligibleScenes(t *testing.T) {
|
||||
if test.change != nil {
|
||||
test.change(request.Chunk)
|
||||
}
|
||||
request.Operation.ReuseEligible = false
|
||||
prepared, assessed := 0, 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
request.Operation.Assess = func(context.Context, dnd.EnemyEventList) (contracts.CandidateAssessment, error) {
|
||||
assessed++
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}
|
||||
result, err := newEnemyExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -100,6 +108,9 @@ func TestExtractSkipsModelForIneligibleScenes(t *testing.T) {
|
||||
if len(client.requests) != 0 || result.Value.Events == nil || len(result.Value.Events) != 0 {
|
||||
t.Fatalf("result = %#v, calls = %d", result, len(client.requests))
|
||||
}
|
||||
if result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || prepared != 0 || assessed != 1 {
|
||||
t.Fatalf("deterministic skip=%#v, prepared/assessed=%d/%d", result.Outcome, prepared, assessed)
|
||||
}
|
||||
if test.wantWarning {
|
||||
if len(result.Diagnostics) != 1 || result.Diagnostics[0].ReasonCode != "scene_classification_unavailable" || result.Diagnostics[0].Disposition != contracts.DiagnosticDispositionWarning || result.Diagnostics[0].Category != contracts.DiagnosticCategoryDegradation || result.Diagnostics[0].Samples[0].Scope != SceneDescriptionReferenceSlot {
|
||||
t.Fatalf("diagnostics = %#v", result.Diagnostics)
|
||||
@@ -115,21 +126,21 @@ func TestExtractReportsRequiredGroundingAndProviderFailures(t *testing.T) {
|
||||
client := &fakeEnemyEventsLLMClient{}
|
||||
request := enemyExtractionRequest(t)
|
||||
request.References = withoutSlot(request.References, NPCRegistryReferenceSlot)
|
||||
if _, err := (&Extractor{llm: client, grounding: mustGroundingResolver(t, request.References)}).Extract(context.Background(), request); err == nil || !strings.Contains(err.Error(), "NPC registry") {
|
||||
if _, err := (&Extractor{engine: newEnemyExtractor(t, client).engine, grounding: mustGroundingResolver(t, request.References)}).Extract(context.Background(), request); err == nil || !strings.Contains(err.Error(), "NPC registry") {
|
||||
t.Fatalf("Extract() error = %v, want required grounding context", err)
|
||||
}
|
||||
|
||||
provider := &fakeEnemyEventsLLMClient{err: errors.New("provider unavailable")}
|
||||
if _, err := newEnemyExtractor(t, provider).Extract(context.Background(), enemyExtractionRequest(t)); err == nil || !strings.Contains(err.Error(), "complete structured output") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
if _, err := newEnemyExtractor(t, provider).Extract(context.Background(), enemyExtractionRequest(t)); err == nil || !strings.Contains(err.Error(), "produce enemy events") || !strings.Contains(err.Error(), "provider unavailable") {
|
||||
t.Fatalf("Extract() error = %v, want provider context", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
func TestConstructorSpecOptionsAndNativeDependencies(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v", err)
|
||||
}
|
||||
if _, err := New(&fakeEnemyEventsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
|
||||
if _, err := New(newEnemyExtractor(t, &fakeEnemyEventsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil || !strings.Contains(err.Error(), "unknown option") {
|
||||
@@ -140,7 +151,7 @@ func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) {
|
||||
first.Requires[0] = "changed"
|
||||
first.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
second := ModuleSpec()
|
||||
if second.Requires[0] != "chunks" || second.ArtifactKind != dnd.EnemyEventListKind || second.ExecutionClass != contracts.ExecutionClassLLMBacked || second.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || second.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
|
||||
if second.Requires[0] != "chunks" || second.ArtifactKind != dnd.EnemyEventListKind || second.ExecutionClass != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(second.Prompts, []string{PromptID, CorrectionPromptID}) || second.ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
|
||||
t.Fatalf("ModuleSpec() reused mutable state: %#v", second)
|
||||
}
|
||||
registry := pipeline.NewExtractorRegistry()
|
||||
@@ -151,82 +162,4 @@ func TestConstructorSpecOptionsAndSafeMetadata(t *testing.T) {
|
||||
t.Fatalf("registered spec = %#v, %t", spec, ok)
|
||||
}
|
||||
|
||||
extractor := newEnemyExtractor(t, &fakeEnemyEventsLLMClient{}, groundingReferences(t, "Ashfang", dnd.SceneKindCombat))
|
||||
metadata := extractor.ManifestMetadata()
|
||||
encoded, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(encoded), "Ashfang") || metadata["mapping_policy"] != mappingPolicy || metadata["scene_gate_policy"] != sceneGatePolicy {
|
||||
t.Fatalf("unsafe or incomplete metadata = %s", encoded)
|
||||
}
|
||||
fingerprints := extractor.CheckpointFingerprints()
|
||||
if len(fingerprints) != 4 || fingerprints[0].Value != metadata["prompt_sha256"] || fingerprints[1].Value != metadata["response_schema_sha256"] || fingerprints[2].Value != mappingPolicy || fingerprints[3].Value != sceneGatePolicy {
|
||||
t.Fatalf("fingerprints = %#v", fingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
func newEnemyExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
if len(references) == 0 {
|
||||
references = []contracts.ReferenceSet{groundingReferences(t, "Ashfang", dnd.SceneKindCombat)}
|
||||
}
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func mustGroundingResolver(t *testing.T, references contracts.ReferenceSet) *groundingResolver {
|
||||
t.Helper()
|
||||
resolver, err := newGroundingResolver(references)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resolver
|
||||
}
|
||||
|
||||
func enemyExtractionRequest(t *testing.T) contracts.TypedExtractionRequest {
|
||||
t.Helper()
|
||||
chunk := combatChunk()
|
||||
chunk.Units = []source.SourceUnit{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}}
|
||||
chunk.MediaType = "application/json"
|
||||
chunk.Content = []byte(`{"id":"combat-scene","units":[1,2,3,4]}`)
|
||||
return contracts.TypedExtractionRequest{
|
||||
Source: &source.SourceDocument{ID: "combat-session", Units: append([]source.SourceUnit(nil), chunk.Units...)},
|
||||
Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", "application/json", chunk.Content, digest(chunk.Content), "file:///combat-session.json"),
|
||||
References: groundingReferences(t, "Ashfang", dnd.SceneKindCombat),
|
||||
LLMProfile: "enemy-profile",
|
||||
SessionID: "session-123",
|
||||
}
|
||||
}
|
||||
|
||||
type fakeEnemyEventsLLMClient struct {
|
||||
response extractionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeEnemyEventsLLMClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneEnemyRequest(request))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
}
|
||||
*target = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneEnemyRequest(request contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
request.Inputs = request.Inputs.Clone()
|
||||
return request
|
||||
}
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
package enemyevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
||||
eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
|
||||
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/engagements"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/shape"
|
||||
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/enemyevents/source_relatedness"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const duplicateEngagements = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":1,"end_unit_id":1}]},{"name":"ASHFANG","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
|
||||
const invalidEventCitation = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||||
const knownEvents = `{"events":[{"name":"Ashfang","kind":"engaged","source_refs":[{"start_unit_id":2,"end_unit_id":1}]},{"name":"Orcs","kind":"fled","source_refs":[{"start_unit_id":4,"end_unit_id":4}]}]}`
|
||||
|
||||
func bindEnemyAssessment(request *contracts.TypedExtractionRequest[dnd.EnemyEventList]) *[]string {
|
||||
validators := []contracts.TypedValidator[dnd.EnemyEventList]{
|
||||
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
|
||||
relatedness.New(relatedness.Options{}), engagements.New(engagements.Options{}),
|
||||
}
|
||||
var calls []string
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.EnemyEventList) (contracts.CandidateAssessment, error) {
|
||||
var assessment contracts.CandidateAssessment
|
||||
for _, validator := range validators {
|
||||
calls = append(calls, validator.Name())
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.EnemyEventList]{
|
||||
Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value, Execution: request.Operation.Execution,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
return &calls
|
||||
}
|
||||
|
||||
func TestEnemyNativeCorrectionKeepsAllGroundingFixedAndSourceFree(t *testing.T) {
|
||||
for _, first := range []string{duplicateEngagements, invalidEventCitation} {
|
||||
request := enemyExtractionRequest(t)
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.ReuseEligible = false
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "enemy-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
calls := bindEnemyAssessment(&request)
|
||||
client := &fakeEnemyEventsLLMClient{responses: []string{first, knownEvents}}
|
||||
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
|
||||
if len(client.requests) == 1 {
|
||||
for _, name := range []string{NPCRegistryReferenceSlot, SceneDescriptionReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} {
|
||||
request.References.Slots[name].Items[0].Content[0] = '!'
|
||||
}
|
||||
}
|
||||
}
|
||||
result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, engagements.Key}
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Events) != 2 || len(client.requests) != 2 || len(prepared) != 2 {
|
||||
t.Fatalf("enemy correction=%#v %v, validators=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
|
||||
}
|
||||
if result.Value.Events[0].SourceRefs[0].StartUnitID != 1 || result.Value.Events[0].SourceRefs[0].EndUnitID != 2 || result.Value.Events[1].Name != "Orcs" || result.Value.Events[1].Kind != dnd.EnemyEventKindFled {
|
||||
t.Fatalf("contextual names/evidence=%#v", result.Value)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "enemy-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native selection lost")
|
||||
}
|
||||
for _, name := range []string{NPCRegistryReferenceSlot, CombatTurnReferenceSlot, NPCOccurrenceReferenceSlot} {
|
||||
material := string(renderedInput(t, call, name))
|
||||
if !strings.Contains(material, "Ashfang") {
|
||||
t.Fatalf("missing %s grounding", name)
|
||||
}
|
||||
for _, forbidden := range []string{"source_ref", "source_id", "start_unit_id", "npc:sha256:", "combat-session", "summary"} {
|
||||
if strings.Contains(material, forbidden) {
|
||||
t.Fatalf("%s exposed %q", name, forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
|
||||
t.Fatal("native history changed fixed grounding")
|
||||
}
|
||||
guidance := revised[len(initial)+1].Content
|
||||
if first == duplicateEngagements && !strings.Contains(guidance, "at most one") {
|
||||
t.Fatal("engagement guidance lost")
|
||||
}
|
||||
if first == invalidEventCitation && !strings.Contains(guidance, "source range endpoints") {
|
||||
t.Fatal("source guidance lost")
|
||||
}
|
||||
content, err := eventcodec.New().Encode(result.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decoded, err := eventcodec.New().Decode(content)
|
||||
if err != nil || !reflect.DeepEqual(result.Value, decoded) {
|
||||
t.Fatalf("event wire round trip=%#v %v", decoded, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnemyTerminalModesAndCancellation(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
response string
|
||||
failures []error
|
||||
assessments, runs int
|
||||
wantError bool
|
||||
}{
|
||||
{"strict engagement rejection", contracts.ModeStrict, duplicateEngagements, nil, 2, 2, true},
|
||||
{"permissive engagement rejection", contracts.ModePermissive, duplicateEngagements, nil, 2, 2, false},
|
||||
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
|
||||
{"failed continuation", contracts.ModePermissive, invalidEventCitation, []error{nil, failure}, 1, 2, false},
|
||||
{"structural failure", contracts.ModePermissive, `{"events":null}`, nil, 0, 1, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request := enemyExtractionRequest(t)
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
calls := bindEnemyAssessment(&request)
|
||||
runs := 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
|
||||
client := &fakeEnemyEventsLLMClient{raw: test.response, errors: test.failures}
|
||||
result, err := newEnemyExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
|
||||
t.Fatalf("terminal=%#v %v, validators=%#v runs=%d", result, err, *calls, runs)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" {
|
||||
t.Fatal("strict failure published output")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 {
|
||||
t.Fatalf("fallback invented events=%#v", result)
|
||||
}
|
||||
if _, err := eventcodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
request := enemyExtractionRequest(t)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
client := &fakeEnemyEventsLLMClient{raw: knownEvents, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
result, err := newEnemyExtractor(t, client).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("canceled outcome=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnemyAcceptsEmptyFallbackReferencesWithoutInventingEvents(t *testing.T) {
|
||||
request := enemyExtractionRequest(t)
|
||||
npc, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
turns, err := combatcodec.New().Encode(dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
occurrences, err := occurrencecodec.New().Encode(dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range map[string][]byte{NPCRegistryReferenceSlot: npc, CombatTurnReferenceSlot: turns, NPCOccurrenceReferenceSlot: occurrences} {
|
||||
request.References = replaceItem(request.References, name, newReferenceItem(name, content))
|
||||
}
|
||||
request.Operation.ReuseEligible = false
|
||||
bindEnemyAssessment(&request)
|
||||
client := &fakeEnemyEventsLLMClient{raw: `{"events":[]}`}
|
||||
result, err := newEnemyExtractor(t, client, contracts.ReferenceSet{}).Extract(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || result.Value.Events == nil || len(result.Value.Events) != 0 || len(client.requests) != 1 {
|
||||
t.Fatalf("empty reference handoff=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
120
internal/modules/dnd/extract/enemyevents/pipeline_test.go
Normal file
120
internal/modules/dnd/extract/enemyevents/pipeline_test.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package enemyevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
type enemyPipelineOutput struct{}
|
||||
|
||||
func (enemyPipelineOutput) Key() string { return "test/enemy-output" }
|
||||
func (enemyPipelineOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{}, nil
|
||||
}
|
||||
|
||||
func TestNativeEnemyPipelineRetainsIntrinsicCodecBoundaryWithEmptyValidators(t *testing.T) {
|
||||
for _, name := range []string{"Ashfang", " "} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
client := &fakeEnemyEventsLLMClient{response: extractionResponse{Events: []enemyEventResponse{
|
||||
{Name: name, Kind: "engaged", SourceRefs: []enemySourceRefResponse{{StartUnitID: 1, EndUnitID: 1}}},
|
||||
}}}
|
||||
engine := newEnemyExtractor(t, client).engine
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
check := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
check(transcript.Register(registries.Inputs))
|
||||
check(units.Register(registries.Chunkers))
|
||||
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, eventcodec.New()))
|
||||
check(Register(registries.Extractors))
|
||||
check(appendorder.RegisterTyped(registries.Mergers, dnd.EnemyEventListKind, func(values []dnd.EnemyEventList) (dnd.EnemyEventList, error) {
|
||||
result := dnd.EnemyEventList{Events: []dnd.EnemyEvent{}}
|
||||
for _, value := range values {
|
||||
result.Events = append(result.Events, value.Events...)
|
||||
}
|
||||
return result, nil
|
||||
}))
|
||||
check(noop.RegisterTyped[dnd.EnemyEventList](registries.Normalizers, dnd.EnemyEventListKind))
|
||||
check(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/enemy-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||
func() (contracts.OutputEncoder, error) { return enemyPipelineOutput{}, nil }))
|
||||
references := groundingReferences(t, "Ashfang", dnd.SceneKindCombat)
|
||||
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{
|
||||
ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "combat-session", StartUnitID: 1, EndUnitID: 1},
|
||||
Kind: dnd.SceneKindCombat, Title: "Combat", Summary: "Ashfang opposes the party.",
|
||||
}}})
|
||||
check(err)
|
||||
references = replaceItem(references, SceneDescriptionReferenceSlot, newReferenceItem(SceneDescriptionReferenceSlot, content))
|
||||
referenceSources := make(map[string]pipeline.ReferenceSource)
|
||||
root := t.TempDir()
|
||||
for name, slot := range references.Slots {
|
||||
path := filepath.Join(root, name+".json")
|
||||
check(os.WriteFile(path, slot.Items[0].Content, 0600))
|
||||
referenceSources[name] = pipeline.ExternalReference(path)
|
||||
}
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: "native-enemies", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding("test/enemy-output"),
|
||||
LLMProfile: "enemy-profile",
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{"enemies": {
|
||||
Extract: pipeline.ModuleBinding{Module: Key, References: referenceSources, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{}}},
|
||||
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
|
||||
}},
|
||||
}
|
||||
catalog := pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
|
||||
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
|
||||
}
|
||||
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
|
||||
check(err)
|
||||
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
|
||||
check(err)
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
|
||||
check(err)
|
||||
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared, SourceID: "combat-session", ExtractWorkers: 1,
|
||||
RawInput: []byte(`{"segments":[{"id":1,"speaker":"GM","start":0,"end":1,"text":"Ashfang attacks the party."}]}`),
|
||||
})
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("native calls=%d", len(client.requests))
|
||||
}
|
||||
if strings.TrimSpace(name) == "" {
|
||||
if err == nil || !strings.Contains(err.Error(), "serialize accepted extract output") || !strings.Contains(err.Error(), "name must not be empty") {
|
||||
t.Fatalf("intrinsic codec error=%v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
check(err)
|
||||
if result.FallbackCount != 0 || len(result.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("assembled outcome=%#v", result)
|
||||
}
|
||||
value, err := eventcodec.New().Decode(result.NormalizeOutputs[0].Artifact.Content)
|
||||
check(err)
|
||||
if len(value.Events) != 1 || value.Events[0].Name != name || value.Events[0].SourceRefs[0].SourceID != "combat-session" {
|
||||
t.Fatalf("durable events=%#v", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package enemyevents
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.enemy_events",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "combat-grounding.md", Path: "prompts/combat-grounding.md"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-npc-registry.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/enemy-events")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope enemy-event assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare enemy-event prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope enemy-event schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,82 +0,0 @@
|
||||
package enemyevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareEnemyEventPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schemaFS, err := registry.SchemaFS()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := fs.ReadFile(schemaFS, "dnd_enemy_events_llm.v1.json"); err != nil {
|
||||
t.Fatalf("response schema asset: %v", err)
|
||||
}
|
||||
|
||||
prepared := prepareEnemyEventPrompt(t)
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_enemy_events_llm.v1.json" || prepared.SelectedProfileID != "dnd-extraction" {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnemyEventPromptRequiresGroundingInputs(t *testing.T) {
|
||||
engine := newEnemyEventPromptEngine(t)
|
||||
for _, inputName := range []string{"npc_registry", "combat_turns", "npc_occurrences"} {
|
||||
t.Run(inputName, func(t *testing.T) {
|
||||
inputs := enemyEventPromptInputs()
|
||||
delete(inputs, inputName)
|
||||
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: inputs,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), inputName) {
|
||||
t.Fatalf("Prepare() error = %v, want required %q input", err, inputName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func prepareEnemyEventPrompt(t *testing.T) *promptkit.PreparedRun {
|
||||
t.Helper()
|
||||
prepared, err := newEnemyEventPromptEngine(t).Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, Inputs: enemyEventPromptInputs(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return prepared
|
||||
}
|
||||
|
||||
func newEnemyEventPromptEngine(t *testing.T) *promptkit.Engine {
|
||||
t.Helper()
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"dnd-extraction": promptkitfixture.Profile("http://127.0.0.1:1/v1", "enemy-test-model"),
|
||||
})
|
||||
}
|
||||
|
||||
func enemyEventPromptInputs() map[string]promptkit.ArtifactRef {
|
||||
return map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"value":"enemy-transcript"}`),
|
||||
"players": promptkit.Inline("enemy-player"),
|
||||
"party": promptkit.Inline("enemy-party"),
|
||||
"glossary": promptkit.Inline("enemy-glossary"),
|
||||
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"enemy-npc"}]}`),
|
||||
"combat_turns": promptkit.Inline(`{"combat_turns":[{"actor":"enemy-turn","turn_kind":"turn"}]}`),
|
||||
"npc_occurrences": promptkit.Inline(`{"npc_occurrences":[{"name":"enemy-opponent","kind":"combat_opponent"}]}`),
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package enemyevents
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.enemy_events"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_enemy_events_llm")
|
||||
ResponseSchemaID = "notarius.dnd.enemy_events.llm"
|
||||
ResponseSchemaName = "notarius_dnd_enemy_events_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_enemy_events_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package enemyevents
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestResponseSchemaDefinesPrivateStructuralBoundary(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v", schema)
|
||||
}
|
||||
valid := validEnemyResponse()
|
||||
content, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateEnemySchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid response rejected: %v", err)
|
||||
}
|
||||
semantic := validEnemyResponse()
|
||||
event := semantic["events"].([]any)[0].(map[string]any)
|
||||
event["name"] = ""
|
||||
ref := event["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
content, err = json.Marshal(semantic)
|
||||
if err != nil || validateEnemySchema(content, schema.JSONSchema) != nil {
|
||||
t.Fatalf("non-categorical validator-owned semantics were rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaRejectsInvalidStructure(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(event map[string]any) { delete(event, "name") },
|
||||
func(event map[string]any) { event["kind"] = 1 },
|
||||
func(event map[string]any) { event["kind"] = "unsupported" },
|
||||
func(event map[string]any) { event["unexpected"] = true },
|
||||
func(event map[string]any) { event["source_refs"].([]any)[0].(map[string]any)["source_id"] = "session" },
|
||||
} {
|
||||
candidate := validEnemyResponse()
|
||||
mutate(candidate["events"].([]any)[0].(map[string]any))
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateEnemySchema(content, schema.JSONSchema); err == nil {
|
||||
t.Fatal("private schema accepted structurally invalid response")
|
||||
}
|
||||
}
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("schema defensive copy = %s, %v", second.JSONSchema, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{string(dnd.EnemyEventKindEngaged), string(dnd.EnemyEventKindKilled), string(dnd.EnemyEventKindFled), string(dnd.EnemyEventKindCaptured), string(dnd.EnemyEventKindIncapacitated)} {
|
||||
candidate := validEnemyResponse()
|
||||
candidate["events"].([]any)[0].(map[string]any)["kind"] = kind
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateEnemySchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("supported enemy-event kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validEnemyResponse() map[string]any {
|
||||
return map[string]any{"events": []any{map[string]any{
|
||||
"name": "Ashfang", "kind": "engaged", "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
}}}
|
||||
}
|
||||
|
||||
func validateEnemySchema(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", document); err != nil {
|
||||
return err
|
||||
}
|
||||
schema, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schema.Validate(instance)
|
||||
}
|
||||
134
internal/modules/dnd/extract/enemyevents/test_helpers_test.go
Normal file
134
internal/modules/dnd/extract/enemyevents/test_helpers_test.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package enemyevents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func newEnemyExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
if len(references) == 0 {
|
||||
references = []contracts.ReferenceSet{groundingReferences(t, "Ashfang", dnd.SceneKindCombat)}
|
||||
}
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: enemy-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: occurrence-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "combat_turns": "combat_turns", "npc_occurrences": "npc_occurrences"}[name]
|
||||
for _, message := range request.Prompt.Messages {
|
||||
for offset := 0; offset < len(message.Content); offset++ {
|
||||
if message.Content[offset] != '{' {
|
||||
continue
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
|
||||
var fields map[string]json.RawMessage
|
||||
if decoder.Decode(&fields) == nil && fields[key] != nil {
|
||||
return []byte(message.Content[offset : offset+int(decoder.InputOffset())])
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeEnemyEventsLLMClient struct {
|
||||
response extractionResponse
|
||||
responses []string
|
||||
raw string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeEnemyEventsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return nil, client.err
|
||||
}
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
content := client.raw
|
||||
if index < len(client.responses) {
|
||||
content = client.responses[index]
|
||||
}
|
||||
if content == "" {
|
||||
encoded, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = string(encoded)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: content}, nil
|
||||
}
|
||||
func mustGroundingResolver(t *testing.T, references contracts.ReferenceSet) *groundingResolver {
|
||||
t.Helper()
|
||||
resolver, err := newGroundingResolver(references)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resolver
|
||||
}
|
||||
|
||||
func enemyExtractionRequest(t *testing.T) contracts.TypedExtractionRequest[dnd.EnemyEventList] {
|
||||
t.Helper()
|
||||
chunk := combatChunk()
|
||||
chunk.Units = []source.SourceUnit{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}}
|
||||
chunk.MediaType = "application/json"
|
||||
chunk.Content = []byte(`{"id":"combat-scene","units":[1,2,3,4]}`)
|
||||
return contracts.TypedExtractionRequest[dnd.EnemyEventList]{
|
||||
Source: &source.SourceDocument{ID: "combat-session", Units: append([]source.SourceUnit(nil), chunk.Units...)},
|
||||
Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", "application/json", chunk.Content, digest(chunk.Content), "file:///combat-session.json"),
|
||||
References: groundingReferences(t, "Ashfang", dnd.SceneKindCombat),
|
||||
LLMProfile: "enemy-profile",
|
||||
SessionID: "session-123",
|
||||
Operation: contracts.ProducerOperation[dnd.EnemyEventList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.EnemyEventList) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user