From 433ed420f8e7470d8a650ef721ff4bbe10a69d62 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 01:26:49 +0000 Subject: [PATCH] Migrate scene descriptions to native correction with invariant-preserving final fallbacks --- .../scene-descriptions/prompts/correction.md | 7 + .../scene-descriptions/prompts/prompt.yaml | 20 +- assets/scene_descriptions_test.go | 120 ++++++++++++ docs/roadmap/implementation.md | 25 +++ .../extract/scenedescriptions/extractor.go | 126 +++++------- .../scenedescriptions/extractor_test.go | 41 ++-- .../dnd/extract/scenedescriptions/fallback.go | 35 ++++ .../scenedescriptions/fallback_test.go | 61 ++++++ .../native_execution_test.go | 184 ++++++++++++++++++ .../scenedescriptions/prompt_assets.go | 74 ------- .../scenedescriptions/prompt_assets_test.go | 71 ------- .../scenedescriptions/registry_test.go | 35 +--- .../dnd/extract/scenedescriptions/schema.go | 26 --- .../extract/scenedescriptions/schema_test.go | 109 ----------- .../scenedescriptions/test_helpers_test.go | 129 ++++++++---- 15 files changed, 604 insertions(+), 459 deletions(-) create mode 100644 assets/dnd/scene-descriptions/prompts/correction.md create mode 100644 assets/scene_descriptions_test.go create mode 100644 internal/modules/dnd/extract/scenedescriptions/fallback.go create mode 100644 internal/modules/dnd/extract/scenedescriptions/fallback_test.go create mode 100644 internal/modules/dnd/extract/scenedescriptions/native_execution_test.go delete mode 100644 internal/modules/dnd/extract/scenedescriptions/prompt_assets.go delete mode 100644 internal/modules/dnd/extract/scenedescriptions/prompt_assets_test.go delete mode 100644 internal/modules/dnd/extract/scenedescriptions/schema.go delete mode 100644 internal/modules/dnd/extract/scenedescriptions/schema_test.go diff --git a/assets/dnd/scene-descriptions/prompts/correction.md b/assets/dnd/scene-descriptions/prompts/correction.md new file mode 100644 index 00000000..8ab30dbb --- /dev/null +++ b/assets/dnd/scene-descriptions/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous scene description using these findings: + +{{ input "guidance" }} + +Return one complete replacement kind, title, and summary for the original +transcript chunk. Do not change its boundary or invent details to justify a +classification. Return only JSON matching the original response contract. diff --git a/assets/dnd/scene-descriptions/prompts/prompt.yaml b/assets/dnd/scene-descriptions/prompts/prompt.yaml index c5c60c8b..580e7ff6 100644 --- a/assets/dnd/scene-descriptions/prompts/prompt.yaml +++ b/assets/dnd/scene-descriptions/prompts/prompt.yaml @@ -17,7 +17,7 @@ prompts: messages: - role: system content_file: ../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../shared/prompts/common-dnd-references.md @@ -27,9 +27,9 @@ prompts: content_file: ../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-scene-combat-policy.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -37,3 +37,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json + + - id: dnd.scene_descriptions.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json diff --git a/assets/scene_descriptions_test.go b/assets/scene_descriptions_test.go new file mode 100644 index 00000000..98657e72 --- /dev/null +++ b/assets/scene_descriptions_test.go @@ -0,0 +1,120 @@ +package assets_test + +import ( + "context" + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/promptkit" +) + +func TestSceneResponseSchemaUsesStrictPrivateContract(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json") + if err != nil { + t.Fatal(err) + } + + valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."} + if err := validateSceneResponse(t, valid, schema); err != nil { + t.Fatalf("valid private response rejected: %v", err) + } + for _, test := range []struct { + name string + response map[string]any + valid bool + }{ + {name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}}, + {name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}}, + {name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}}, + {name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}}, + {name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}}, + {name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true}, + {name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true}, + {name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}}, + {name: "wrong title type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}}, + {name: "wrong summary type", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": 7}}, + } { + t.Run(test.name, func(t *testing.T) { + err := validateSceneResponse(t, test.response, schema) + if (err == nil) != test.valid { + t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid) + } + }) + } + if err := validateJSONSchema([]byte(`{"kind":`), schema); err == nil { + t.Fatal("validateJSONSchema() error = nil, want malformed JSON rejected") + } +} + +func TestSceneResponseSchemaAcceptsEverySupportedKind(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/schemas/dnd_scene_descriptions_llm.v1.json") + if err != nil { + t.Fatal(err) + } + for _, kind := range []string{"combat", "narrative", "recap", "meta"} { + response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"} + if err := validateSceneResponse(t, response, schema); err != nil { + t.Fatalf("supported scene kind %q was rejected: %v", kind, err) + } + } +} + +func validateSceneResponse(t *testing.T, instance map[string]any, schemaContent []byte) error { + t.Helper() + content, err := json.Marshal(instance) + if err != nil { + t.Fatal(err) + } + return validateJSONSchema(content, schemaContent) +} + +func TestSceneDescriptionPromptPairRetainsPolicyAndFixedInputs(t *testing.T) { + _, engine := loadNativeAssets(t) + initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.scene_descriptions", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline(`{"units":[{"text":"scene-description-transcript"}]}`), + "players": promptkit.Inline("scene-description-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "), + }, + }) + if err != nil { + t.Fatal(err) + } + defer initial.Discard() + correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{ + PromptID: "notarius/dnd.scene_descriptions.repair", APIKey: "test-only", + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Return kind: combat for this scene.")}, + }) + if err != nil { + t.Fatal(err) + } + defer correction.Discard() + first, next := initial.Details(), correction.Details() + if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID || first.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) { + t.Fatal("scene pair lost schema/defaults") + } + var system, all string + for _, message := range first.Messages { + all += message.Content + if message.Role == "system" { + system += message.Content + } + } + for _, rule := range []string{"complete chunk is the evidence boundary", "substantive active combat materially organizes the", "must not invent a proper noun"} { + if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) { + t.Fatalf("missing enduring rule %q", rule) + } + } + for _, material := range []string{"scene-description-transcript", "scene-description-player", "substantive active combat materially organizes the"} { + if strings.Count(all, material) != 1 { + t.Fatalf("fixed input/policy repeated: %q", material) + } + } + if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Return kind: combat for this scene.") || strings.Contains(next.Messages[0].Content, "scene-description-transcript") { + t.Fatal("correction assignment lost findings or repeated source") + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 848503e0..c3db0a75 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1036,6 +1036,31 @@ enemy-event codec contract, using a native provider fake and current typed codec ## Stage 23: Migrate Scene-Description Extraction +**Status:** Complete. Scene descriptions now use native initial/correction +requests through the shared typed producer helper with fixed transcript/reference +materials and captured chunk identity/range. Added matching schema/default-profile +correction assets and static dependencies; removed private completion, schema/ +asset registration, and manifest/fingerprint metadata. The existing one-scene +mapping, kind/title/summary contract, and trimming remain unchanged. +Permissive fallback construction uses only the final usable candidate and checks +the unchanged durable codec, exact current chunk identity/range, trimmed fields, +and current-source addressing. It does not rerun ordinary semantic validators, +invent a classification/prose, publish an empty collection, or recover an earlier +candidate after a result-less/structurally failed correction. Strict handling and +cancellation publish no fallback. Added native-provider coverage with the real +combat-semantics validator for ordered complete-chain correction, validator versus +producer profiles, shared run overrides/session, exact history, qualified final-only +semantic fallback, unusable final responses, failed continuation, and cancellation. +Retained mapping/reference/registration regressions and moved native prompt-pair/ +unchanged-schema checks to assets. Independently runnable fallback tests cover +missing/empty/multiple scenes, identity/range/shape failures, and defensive ownership. +Asset, shape, source-reference, invariant, codec, and isolated fallback tests and +vet/race checks pass, as do focused pipeline tests and `git diff --check`. +Full extractor and combat-semantics package checks remain blocked at setup by +shared legacy helpers importing old ConfigKit's removed YAML `plugin/limit`; +their new native runtime tests are not claimed passing. Scheduled cleanup and +final assembled verification must execute them. + **Goal:** retain the required scene artifact while making fallback limits explicit. **Inspect/change:** `internal/modules/dnd/extract/scenedescriptions`, its codec, diff --git a/internal/modules/dnd/extract/scenedescriptions/extractor.go b/internal/modules/dnd/extract/scenedescriptions/extractor.go index 6abb33fd..a8558222 100644 --- a/internal/modules/dnd/extract/scenedescriptions/extractor.go +++ b/internal/modules/dnd/extract/scenedescriptions/extractor.go @@ -2,9 +2,12 @@ package scenedescriptions import ( "context" + "encoding/json" "fmt" "strings" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -13,6 +16,8 @@ import ( ) const Key = "dnd/scene-descriptions" +const PromptID = "notarius/dnd.scene_descriptions" +const CorrectionPromptID = "notarius/dnd.scene_descriptions.repair" var requiredCapabilities = []string{ "chunks", @@ -41,102 +46,65 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.SceneDescriptionList] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} -type Extractor struct { - llm contracts.StructuredLLMClient - promptSHA string - responseSchemaSHA string -} +type Extractor struct{ engine *promptkit.Engine } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{ - llm: llmClient, - promptSHA: promptSHA, - responseSchemaSHA: responseSchema.SHA256, - }, nil + return &Extractor{engine: engine}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - return map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - "mapping_policy": mappingPolicy, - } -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "mapping_policy", Value: mappingPolicy}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("%w", err) } - var response extractionResponse - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, - PromptID: PromptID, - PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, - SessionID: req.SessionID, - StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, - Inputs: shared.PromptInputs(sourceInput, req.References), - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("complete structured output: %w", err) + chunk := source.Chunk{ID: req.Chunk.ID, Ref: req.Chunk.Ref} + index := source.NewDocumentIndex(req.Source) + materials := shared.PromptInputs(sourceInput, req.References) + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.SceneDescriptionList, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + value := mapResponse(response, &chunk) + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(_ context.Context, final *dnd.SceneDescriptionList) (dnd.SceneDescriptionList, error) { + return sceneFallback(final, chunk, index) + }) if err != nil { - return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("produce scene description: %w", err) } - return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: mapResponse(response, req.Chunk), ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: value, Outcome: outcome}, nil } func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDescriptionList { @@ -151,14 +119,14 @@ func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDesc func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ArtifactKind: dnd.SceneDescriptionListKind, - ReferenceSlots: referenceSlots(), + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.SceneDescriptionListKind, + ReferenceSlots: referenceSlots(), } } @@ -168,7 +136,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/scenedescriptions/extractor_test.go b/internal/modules/dnd/extract/scenedescriptions/extractor_test.go index 8923d7e3..db12c37f 100644 --- a/internal/modules/dnd/extract/scenedescriptions/extractor_test.go +++ b/internal/modules/dnd/extract/scenedescriptions/extractor_test.go @@ -32,12 +32,11 @@ func TestExtractMapsExactlyOneSceneToTheChunk(t *testing.T) { t.Fatalf("LLM calls = %d, want 1", len(client.requests)) } completed := client.requests[0] - if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != "profile-scene-descriptions" || completed.SessionID != "session-123" { - t.Fatalf("LLM request identity = %#v", completed) + if completed.Prompt.SessionID != request.SessionID || result.Outcome.Disposition != contracts.ProducerAccepted { + t.Fatal("native session/outcome lost") } - transcript := completed.Inputs["transcript"] - if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(request.Chunk.Content) { - t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript) + if string(renderedInput(t, completed, "transcript")) != string(request.Chunk.Content) { + t.Fatal("chunk-scoped material changed") } } @@ -47,12 +46,6 @@ func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) { if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v, want nil", err) } - for _, slot := range []string{"players", "party", "glossary"} { - if got := string(client.requests[0].Inputs[slot].Content); got != " " { - t.Fatalf("empty %s input = %q, want explicit placeholder", slot, got) - } - } - req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ "players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Mira")}}}, "party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Mira: ranger")}}}, @@ -61,23 +54,21 @@ func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) { if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil { t.Fatalf("Extract() error = %v, want nil", err) } - inputs := client.requests[1].Inputs - if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" { - t.Fatalf("reference inputs = %#v", inputs) + var text string + for _, message := range client.requests[1].Prompt.Messages { + text += message.Content } - if strings.Contains(string(inputs["transcript"].Content), "local title") { - t.Fatal("transcript input contains reference content") + for _, reference := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: local title"} { + if strings.Count(text, reference) != 1 { + t.Fatalf("optional input not rendered once: %q", reference) + } + } + if strings.Contains(string(renderedInput(t, client.requests[1], "transcript")), "local title") { + t.Fatal("campaign context became evidence") } } func TestExtractReturnsSemanticallyInvalidResponseForDeterministicValidation(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v", err) - } - if err := validateJSONSchema(t, map[string]any{"kind": "narrative", "title": " ", "summary": ""}, schema.JSONSchema); err != nil { - t.Fatalf("semantic candidate rejected by private schema: %v", err) - } client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{ Kind: dnd.SceneKindNarrative, Title: " ", Summary: "", }} @@ -98,11 +89,11 @@ func TestExtractValidatesRequestsAndSurfacesProviderFailures(t *testing.T) { for _, test := range []struct { name string extractor *Extractor - req contracts.TypedExtractionRequest + req contracts.TypedExtractionRequest[dnd.SceneDescriptionList] want string }{ {name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"}, - {name: "nil LLM client", extractor: &Extractor{}, req: request, want: "LLM client"}, + {name: "nil LLM client", extractor: &Extractor{}, req: request, want: "native engine"}, {name: "wrapped preflight failure", extractor: extractor, req: mismatchedSourceInputRequest(request), want: "must match chunk"}, } { t.Run(test.name, func(t *testing.T) { diff --git a/internal/modules/dnd/extract/scenedescriptions/fallback.go b/internal/modules/dnd/extract/scenedescriptions/fallback.go new file mode 100644 index 00000000..3d2ce828 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/fallback.go @@ -0,0 +1,35 @@ +package scenedescriptions + +import ( + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" +) + +// sceneFallback checks construction invariants, not the configured validator chain. +func sceneFallback(final *dnd.SceneDescriptionList, chunk source.Chunk, index source.DocumentIndex) (dnd.SceneDescriptionList, error) { + if final == nil { + return dnd.SceneDescriptionList{}, fmt.Errorf("no usable final scene description is available") + } + if len(final.Scenes) != 1 { + return dnd.SceneDescriptionList{}, fmt.Errorf("scene fallback requires exactly one final scene") + } + scene := final.Scenes[0] + if scene.ID != chunk.ID || scene.SourceRef != chunk.Ref { + return dnd.SceneDescriptionList{}, fmt.Errorf("final scene does not match the current chunk identity and range") + } + if scene.ID != strings.TrimSpace(scene.ID) || scene.SourceRef.SourceID != strings.TrimSpace(scene.SourceRef.SourceID) || + scene.Title != strings.TrimSpace(scene.Title) || scene.Summary != strings.TrimSpace(scene.Summary) { + return dnd.SceneDescriptionList{}, fmt.Errorf("final scene fields must be trimmed") + } + if _, err := scenecodec.New().Encode(*final); err != nil { + return dnd.SceneDescriptionList{}, fmt.Errorf("final scene cannot be published: %w", err) + } + if err := index.ValidateRef(scene.SourceRef); err != nil { + return dnd.SceneDescriptionList{}, fmt.Errorf("final scene source range is unusable: %w", err) + } + return dnd.SceneDescriptionList{Scenes: append([]dnd.SceneDescription(nil), final.Scenes...)}, nil +} diff --git a/internal/modules/dnd/extract/scenedescriptions/fallback_test.go b/internal/modules/dnd/extract/scenedescriptions/fallback_test.go new file mode 100644 index 00000000..c24f291f --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/fallback_test.go @@ -0,0 +1,61 @@ +package scenedescriptions + +import ( + "reflect" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" +) + +func TestSceneFallbackRequiresOneUsableFinalSceneWithCurrentIdentity(t *testing.T) { + doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1}, {ID: 2}}} + chunk := source.Chunk{ID: "chunk", Ref: source.SourceRef{SourceID: "session", StartUnitID: 1, EndUnitID: 2}} + index := source.NewDocumentIndex(doc) + valid := dnd.SceneDescription{ID: chunk.ID, SourceRef: chunk.Ref, Kind: dnd.SceneKindNarrative, Title: "Arrival", Summary: "The party arrives."} + for _, test := range []struct { + name string + change func(*dnd.SceneDescriptionList, *source.Chunk) + }{ + {"empty", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes = []dnd.SceneDescription{} }}, + {"many", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { + value.Scenes = append(value.Scenes, value.Scenes[0]) + }}, + {"blank title", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Title = "" }}, + {"blank summary", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Summary = "" }}, + {"unsupported kind", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Kind = "unknown" }}, + {"wrong identity", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].ID = "another-chunk" }}, + {"partial range", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].SourceRef.EndUnitID = 1 }}, + {"wrong source", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].SourceRef.SourceID = "other" }}, + {"unknown endpoint", func(value *dnd.SceneDescriptionList, current *source.Chunk) { + value.Scenes[0].SourceRef.EndUnitID = 99 + current.Ref = value.Scenes[0].SourceRef + }}, + {"untrimmed identity", func(value *dnd.SceneDescriptionList, current *source.Chunk) { + value.Scenes[0].ID = " chunk " + current.ID = value.Scenes[0].ID + }}, + {"untrimmed prose", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Summary = " Untrimmed " }}, + } { + t.Run(test.name, func(t *testing.T) { + value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{valid}} + current := chunk + test.change(&value, ¤t) + if result, err := sceneFallback(&value, current, index); err == nil || result.Scenes != nil { + t.Fatalf("unusable fallback=%#v %v", result, err) + } + }) + } + if result, err := sceneFallback(nil, chunk, index); err == nil || result.Scenes != nil { + t.Fatalf("missing final scene=%#v %v", result, err) + } + value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{valid}} + result, err := sceneFallback(&value, chunk, index) + if err != nil || !reflect.DeepEqual(result, value) { + t.Fatalf("usable final scene=%#v %v", result, err) + } + result.Scenes[0].Title = "changed" + if value.Scenes[0] != valid { + t.Fatal("fallback aliases the candidate or alters its classification/prose") + } +} diff --git a/internal/modules/dnd/extract/scenedescriptions/native_execution_test.go b/internal/modules/dnd/extract/scenedescriptions/native_execution_test.go new file mode 100644 index 00000000..ba993430 --- /dev/null +++ b/internal/modules/dnd/extract/scenedescriptions/native_execution_test.go @@ -0,0 +1,184 @@ +package scenedescriptions + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" + semantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape" + sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_relatedness" + "go.maximumdirect.net/promptkit" +) + +const narrativeScene = `{"kind":"narrative","title":"The watchtower","summary":"The party faces the bandits."}` +const combatScene = `{"kind":"combat","title":"Watchtower fight","summary":"Bandits attack and the party drives them back."}` +const blankScene = `{"kind":"combat","title":"","summary":""}` +const combatJudgment = `{"classification":"combat","explanation":"Bandits attack the party during the current scene."}` + +func bindSceneAssessment(t *testing.T, extractor *Extractor, request *contracts.TypedExtractionRequest[dnd.SceneDescriptionList], withSemantic bool) *[]string { + t.Helper() + validators := []contracts.TypedValidator[dnd.SceneDescriptionList]{shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{})} + if withSemantic { + validator, err := semantics.New(extractor.engine, semantics.Options{}) + if err != nil { + t.Fatal(err) + } + validators = append(validators, validator) + } + var calls []string + request.Operation.Assess = func(ctx context.Context, value dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) { + var assessment contracts.CandidateAssessment + for _, validator := range validators { + calls = append(calls, validator.Name()) + profile := request.LLMProfile + if validator.Name() == semantics.Key { + profile = "scene-validator-profile" + } + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.SceneDescriptionList]{ + Stage: "extract", Source: request.Source, Chunk: request.Chunk, SourceInput: request.SourceInput, + LLMProfile: profile, SessionID: request.SessionID, Execution: request.Operation.Execution, Value: value, + }) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) + } + return assessment, nil + } + return &calls +} + +func TestSceneCorrectionUsesRealNativeCombatJudgmentsAndPreservesSourceIdentity(t *testing.T) { + request := extractionRequest() + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.ReuseEligible = false + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "run-wide-scene-model"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) } + client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, combatScene, combatJudgment}} + extractor := newExtractor(t, client) + calls := bindSceneAssessment(t, extractor, &request, true) + result, err := extractor.Extract(context.Background(), request) + chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, semantics.Key} + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || + !reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 4 || len(prepared) != 4 || len(result.Value.Scenes) != 1 { + t.Fatalf("scene correction=%#v %v calls=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared)) + } + scene := result.Value.Scenes[0] + if scene.ID != request.Chunk.ID || scene.SourceRef != request.Chunk.Ref || scene.Kind != dnd.SceneKindCombat || scene.Title != "Watchtower fight" { + t.Fatalf("mapped scene=%#v", scene) + } + for i, call := range client.requests { + id, profile := PromptID, request.LLMProfile + if i == 2 { + id = CorrectionPromptID + } + if i%2 == 1 { + id, profile = semantics.PromptID, "scene-validator-profile" + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != profile || call.Target.Model != "run-wide-scene-model" || call.Prompt.SessionID != request.SessionID { + t.Fatal("producer/validator native selections lost") + } + } + initial, revised := client.requests[0].Prompt.Messages, client.requests[2].Prompt.Messages + if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != narrativeScene { + t.Fatal("continuation includes wrong history or regenerated context") + } + if !strings.Contains(revised[len(initial)+1].Content, "Return kind: combat for this scene.") || !strings.Contains(revised[len(initial)+1].Content, "Bandits attack the party") { + t.Fatal("combat judgment guidance lost") + } + if _, err := scenecodec.New().Encode(result.Value); err != nil { + t.Fatal(err) + } +} + +func TestPermissiveSceneRetainsOnlyFinalUsableCandidateWithoutSemanticRevalidation(t *testing.T) { + request := extractionRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + request.Operation.Controls.MaxAdditionalTurns = 1 + // Both scenes remain non-combat, but the final title/summary must win. + final := `{"kind":"narrative","title":"Final watchtower description","summary":"The party is attacked by bandits."}` + client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, final, combatJudgment}} + extractor := newExtractor(t, client) + calls := bindSceneAssessment(t, extractor, &request, true) + result, err := extractor.Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Outcome.FallbackReason == "" || len(client.requests) != 4 || len(*calls) != 8 || len(result.Value.Scenes) != 1 { + t.Fatalf("semantic fallback=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls) + } + scene := result.Value.Scenes[0] + if scene.Title != "Final watchtower description" || scene.Kind != dnd.SceneKindNarrative || scene.SourceRef != request.Chunk.Ref || scene.ID != request.Chunk.ID { + t.Fatalf("fallback recovered earlier scene or invented classification=%#v", scene) + } + if _, err := scenecodec.New().Encode(result.Value); err != nil { + t.Fatal(err) + } +} + +func TestSceneCannotFallbackAfterMissingOrIntrinsicallyUnusableFinalResponse(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + responses []string + failures []error + wantAssessments int + }{ + {"no first response", nil, []error{failure}, 0}, + {"structural failure", []string{`{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 0}, + {"blank final scene", []string{blankScene, blankScene}, nil, 2}, + {"failed correction", []string{blankScene}, []error{nil, failure}, 1}, + // An initially usable scene cannot be recovered after a result-less correction. + {"lost usable prior scene", []string{narrativeScene}, []error{nil, failure}, 1}, + {"structurally unusable revision", []string{narrativeScene, `{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 1}, + } { + t.Run(test.name, func(t *testing.T) { + request := extractionRequest() + request.Operation.Controls.Mode = contracts.ModePermissive + request.Operation.Controls.MaxAdditionalTurns = 1 + assessments := 0 + request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) { + assessments++ + return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "test/reject", Decision: contracts.AssessmentRejected, + Result: contracts.ValidationResult{ReasonCode: "test_rejection", Message: "Scene rejected.", CorrectionGuidance: "Correct the scene description."}}}}, nil + } + client := &fakeSceneDescriptionsLLMClient{responses: test.responses, errors: test.failures} + if len(test.responses) == 1 { + client.raw = test.responses[0] + } + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err == nil || result.Outcome.Disposition != "" || len(result.Value.Scenes) != 0 || assessments != test.wantAssessments { + t.Fatalf("unusable scene=%#v %v, assessments=%d", result, err, assessments) + } + }) + } +} + +func TestStrictSceneRejectionAndCancellationDoNotPublishFallback(t *testing.T) { + request := extractionRequest() + request.Operation.Controls.MaxAdditionalTurns = 0 + client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment}} + extractor := newExtractor(t, client) + bindSceneAssessment(t, extractor, &request, true) + result, err := extractor.Extract(context.Background(), request) + if err == nil || result.Outcome.Disposition != "" || len(client.requests) != 2 { + t.Fatalf("strict scene=%#v %v", result, err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request.Operation.Controls.Mode = contracts.ModePermissive + canceled := &fakeSceneDescriptionsLLMClient{raw: combatScene, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + result, err = newExtractor(t, canceled).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(canceled.requests) != 1 { + t.Fatalf("cancellation=%#v %v", result, err) + } +} diff --git a/internal/modules/dnd/extract/scenedescriptions/prompt_assets.go b/internal/modules/dnd/extract/scenedescriptions/prompt_assets.go deleted file mode 100644 index f03a10ed..00000000 --- a/internal/modules/dnd/extract/scenedescriptions/prompt_assets.go +++ /dev/null @@ -1,74 +0,0 @@ -package scenedescriptions - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.scene_descriptions", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-identity.md", - "common-dnd-references.md", - "common-dnd-transcript-chunk.md", - "common-dnd-scene-combat-policy.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions") - if err != nil { - return nil, fmt.Errorf("scope scene-description assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare scene-description prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope scene-description schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/scenedescriptions/prompt_assets_test.go b/internal/modules/dnd/extract/scenedescriptions/prompt_assets_test.go deleted file mode 100644 index 5435c308..00000000 --- a/internal/modules/dnd/extract/scenedescriptions/prompt_assets_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package scenedescriptions - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "scene-description-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-description-test-model"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"scene-description-transcript"}`), - "players": promptkit.Inline("scene-description-player"), - "party": promptkit.Inline(" "), - "glossary": promptkit.Inline(" "), - }, - }) - if err != nil { - t.Fatalf("Prepare() error = %v, want nil", err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" { - t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared) - } - const combatPolicySentinel = "substantive active combat materially organizes the" - transcriptIndex := -1 - policyIndex := -1 - instructionsIndex := len(prepared.Messages) - 1 - policyOccurrences := 0 - for index, message := range prepared.Messages { - if strings.Contains(message.Content, "scene-description-transcript") { - transcriptIndex = index - } - if count := strings.Count(message.Content, combatPolicySentinel); count > 0 { - policyIndex = index - policyOccurrences += count - } - } - if policyOccurrences != 1 || policyIndex <= transcriptIndex || policyIndex >= instructionsIndex { - t.Fatalf("combat policy appeared %d times at message %d; want once after transcript message %d and before instructions message %d", policyOccurrences, policyIndex, transcriptIndex, instructionsIndex) - } -} - -func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) { - hash, err := promptAssetMetadata() - if err != nil || !strings.HasPrefix(hash, "sha256:") { - t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err) - } - payload, err := json.Marshal(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata()) - if err != nil { - t.Fatal(err) - } - for _, forbidden := range []string{"common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} { - if strings.Contains(string(payload), forbidden) { - t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload) - } - } -} diff --git a/internal/modules/dnd/extract/scenedescriptions/registry_test.go b/internal/modules/dnd/extract/scenedescriptions/registry_test.go index bf4d6e66..9587dfaf 100644 --- a/internal/modules/dnd/extract/scenedescriptions/registry_test.go +++ b/internal/modules/dnd/extract/scenedescriptions/registry_test.go @@ -11,10 +11,10 @@ import ( ) func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) { - if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") { + if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") { t.Fatalf("New(nil) error = %v, want dependency error", err) } - if _, err := New(&fakeSceneDescriptionsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { + if _, err := New(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") { t.Fatalf("New() error = %v, want reference-set error", err) } if got := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).Key(); got != Key { @@ -24,7 +24,7 @@ func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) { func TestModuleSpecAndReferenceSlots(t *testing.T) { want := pipeline.ModuleSpec{ - Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind, + Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind, ReferenceSlots: []contracts.ReferenceSlot{ {Name: "glossary", Description: "Optional campaign glossary reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, {Name: "party", Description: "Optional party roster reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}}, @@ -61,32 +61,3 @@ func TestRegisterStoresTypedModuleSpecAndOptionsAreStrict(t *testing.T) { t.Fatalf("DecodeOptions() error = %v, want strict options error", err) } } - -func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) { - metadata := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata() - for key, want := range map[string]string{ - "prompt_id": PromptID, "prompt_version": SchemaVersion, - "response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion, - "mapping_policy": mappingPolicy, - } { - if metadata[key] != want { - t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want) - } - } - for _, key := range []string{"prompt_sha256", "response_schema_sha256"} { - if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") { - t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key]) - } - } - want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "mapping_policy": mappingPolicy} - fingerprints := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).CheckpointFingerprints() - if len(fingerprints) != len(want) { - t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want)) - } - for _, fingerprint := range fingerprints { - if fingerprint.Value != want[fingerprint.Name] { - t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want) - } - } -} diff --git a/internal/modules/dnd/extract/scenedescriptions/schema.go b/internal/modules/dnd/extract/scenedescriptions/schema.go deleted file mode 100644 index 143ae455..00000000 --- a/internal/modules/dnd/extract/scenedescriptions/schema.go +++ /dev/null @@ -1,26 +0,0 @@ -package scenedescriptions - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.scene_descriptions" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_descriptions_llm") - ResponseSchemaID = "notarius.dnd.scene_descriptions.llm" - ResponseSchemaName = "notarius_dnd_scene_descriptions_llm_v1" - SchemaVersion = "v1" - mappingPolicy = "dnd.scene_descriptions.mapping.v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_scene_descriptions_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/scenedescriptions/schema_test.go b/internal/modules/dnd/extract/scenedescriptions/schema_test.go deleted file mode 100644 index 86f1aba9..00000000 --- a/internal/modules/dnd/extract/scenedescriptions/schema_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package scenedescriptions - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" - - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" -) - -func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatalf("loadResponseSchema() error = %v, want nil", err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v, want private scene-description schema identity", schema) - } - valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."} - if err := validateJSONSchema(t, valid, schema.JSONSchema); err != nil { - t.Fatalf("valid private response rejected: %v", err) - } - for _, test := range []struct { - name string - response map[string]any - valid bool - }{ - {name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}}, - {name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}}, - {name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}}, - {name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}}, - {name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}}, - {name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true}, - {name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true}, - {name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}}, - {name: "wrong title type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}}, - {name: "wrong summary type", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": 7}}, - } { - t.Run(test.name, func(t *testing.T) { - err := validateJSONSchema(t, test.response, schema.JSONSchema) - if (err == nil) != test.valid { - t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid) - } - }) - } - if err := validateJSONSchemaContent([]byte(`{"kind":`), schema.JSONSchema); err == nil { - t.Fatal("validateJSONSchemaContent() error = nil, want malformed JSON rejected") - } -} - -func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) { - first, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - first.JSONSchema[0] = '[' - second, err := loadResponseSchema() - if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { - t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err) - } - if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil { - t.Fatalf("schema diagnostics included raw content: %#v", diagnostics) - } -} - -func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - for _, kind := range []string{string(dnd.SceneKindCombat), string(dnd.SceneKindNarrative), string(dnd.SceneKindRecap), string(dnd.SceneKindMeta)} { - response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"} - if err := validateJSONSchema(t, response, schema.JSONSchema); err != nil { - t.Fatalf("supported scene kind %q was rejected: %v", kind, err) - } - } -} - -func validateJSONSchema(t *testing.T, instance map[string]any, schemaContent []byte) error { - t.Helper() - content, err := json.Marshal(instance) - if err != nil { - t.Fatal(err) - } - return validateJSONSchemaContent(content, schemaContent) -} - -func validateJSONSchemaContent(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", schemaDocument); err != nil { - return err - } - compiled, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return compiled.Validate(instance) -} diff --git a/internal/modules/dnd/extract/scenedescriptions/test_helpers_test.go b/internal/modules/dnd/extract/scenedescriptions/test_helpers_test.go index 0c288e2e..bda96107 100644 --- a/internal/modules/dnd/extract/scenedescriptions/test_helpers_test.go +++ b/internal/modules/dnd/extract/scenedescriptions/test_helpers_test.go @@ -3,14 +3,19 @@ package scenedescriptions import ( "context" "encoding/json" - "errors" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" ) -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[dnd.SceneDescriptionList] { doc := sourceDocument() chunk := &source.Chunk{ ID: "session-alpha:chunk:0", @@ -22,12 +27,15 @@ func extractionRequest() contracts.TypedExtractionRequest { Units: append([]source.SourceUnit(nil), doc.Units...), Metadata: map[string]any{"ignored": "chunk metadata"}, } - return contracts.TypedExtractionRequest{ + return contracts.TypedExtractionRequest[dnd.SceneDescriptionList]{ Source: doc, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), SessionID: "session-123", LLMProfile: "profile-scene-descriptions", + Operation: contracts.ProducerOperation[dnd.SceneDescriptionList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, } } @@ -45,58 +53,99 @@ func sourceDocument() *source.SourceDocument { } } -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { - t.Helper() - extractor, err := New(client, Options{}, references...) - if err != nil { - t.Fatalf("New() error = %v, want nil", err) - } - return extractor -} - -func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest { +func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]) contracts.TypedExtractionRequest[dnd.SceneDescriptionList] { req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json") return req } -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { + t.Helper() + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit: + profiles: + - id: profile-scene-descriptions + endpoint: https://example.invalid/v1 + model: occurrence-test-model + api_key_required: false + - id: scene-validator-profile + endpoint: https://example.invalid/v1 + model: validator-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - panic(err) + t.Fatal(err) } - req.Correction = correction - if len(req.Vars) == 0 { - req.Vars = nil - return req + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) } - vars := make(map[string]any, len(req.Vars)) - for key, value := range req.Vars { - vars[key] = value + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) } - req.Vars = vars - return req + return extractor +} + +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name] + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } + } + t.Fatalf("missing rendered %s input", name) + return nil } type fakeSceneDescriptionsLLMClient struct { - response extractionResponse - err error - requests []contracts.StructuredCompletionRequest + response extractionResponse + responses []string + raw string + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeSceneDescriptionsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) +func (client *fakeSceneDescriptionsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } + if err := ctx.Err(); err != nil { + return nil, err + } if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err + return nil, client.err } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] } - *target = client.response - content, err := json.Marshal(client.response) - if err != nil { - return contracts.StructuredCompletionResponse{}, err + content := client.raw + if index < len(client.responses) { + content = client.responses[index] } - return contracts.StructuredCompletionResponse{Content: content}, nil + if content == "" { + encoded, err := json.Marshal(client.response) + if err != nil { + return nil, err + } + content = string(encoded) + } + return &promptkit.GenerateResponse{Content: content}, nil }