Migrate spell extraction to native execution and semantic continuation

This commit is contained in:
2026-11-22 00:02:13 +00:00
parent 923cbe594f
commit 92c2382806
21 changed files with 563 additions and 654 deletions

View File

@@ -1,29 +1,18 @@
package spells
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
)
func newCatalogPromptInput(effective spellcatalog.EffectiveCatalog) (contracts.LLMInputMaterial, error) {
func newCatalogPromptInput(effective spellcatalog.EffectiveCatalog) ([]byte, error) {
content, err := json.Marshal(struct {
Spells []spellcatalog.PromptSpell `json:"spells"`
}{Spells: effective.PromptSpells()})
if err != nil {
return contracts.LLMInputMaterial{}, fmt.Errorf("encode spell recognition catalog: %w", err)
return nil, fmt.Errorf("encode spell recognition catalog: %w", err)
}
sum := sha256.Sum256(content)
digest := "sha256:" + hex.EncodeToString(sum[:])
return contracts.NewLLMInputMaterial(
spellcatalog.SpellCatalogReferenceSlot,
"application/json",
content,
digest,
"",
), nil
return content, nil
}

View File

@@ -2,8 +2,12 @@ package spells
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -14,9 +18,8 @@ import (
)
const Key = "dnd/spells"
const SchemaVersion = "v1"
const mappingPolicy = "dnd.spells.extract_mapping.v3"
const PromptID = "notarius/dnd.spells"
const CorrectionPromptID = "notarius/dnd.spells.repair"
const (
NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
@@ -58,22 +61,18 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.SpellList] = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
effectiveCatalog spellcatalog.EffectiveCatalog
catalogPromptInput contracts.LLMInputMaterial
engine *promptkit.Engine
catalogPromptInput []byte
npcResolver *npcregistry.Resolver
promptSHA string
responseSchemaSHA string
}
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
@@ -94,22 +93,7 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare NPC registry prompt input: %w", err)
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{
llm: llmClient,
effectiveCatalog: effectiveCatalog,
catalogPromptInput: catalogPromptInput,
npcResolver: npcResolver,
promptSHA: promptSHA,
responseSchemaSHA: responseSchema.SHA256,
}, nil
return &Extractor{engine: engine, catalogPromptInput: catalogPromptInput, npcResolver: npcResolver}, nil
}
func (e *Extractor) Key() string {
@@ -120,99 +104,68 @@ func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot {
return referenceSlots()
}
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"catalog_base_id": e.effectiveCatalog.BaseID(),
"catalog_digest": e.effectiveCatalog.Digest(),
"catalog_overlay_ids": e.effectiveCatalog.OverlayIDs(),
"mapping_policy": mappingPolicy,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
}
seeded := e.npcResolver.Seeded()
if seeded.Bound() {
metadata["npc_registry_digest"] = seeded.Digest()
metadata["npc_count"] = seeded.Count()
}
return metadata
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
fingerprints := []pipeline.CheckpointFingerprint{
{Name: "effective_catalog", Value: e.effectiveCatalog.Digest()},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
}
seeded := e.npcResolver.Seeded()
fingerprints = append(fingerprints, pipeline.CheckpointFingerprint{Name: "npc_registry", Value: seeded.ProjectionDigest()})
return fingerprints
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("native engine must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("%w", err)
}
sourceID := req.Source.ID
order := shared.NewSourceRefOrder(req.Source)
npcRegistry, err := e.npcResolver.Resolve(req.References)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("resolve NPC registry: %w", err)
}
var response extractionResponse
inputs := shared.PromptInputs(sourceInput, req.References)
inputs[spellcatalog.SpellCatalogReferenceSlot] = e.catalogPromptInput.Clone()
inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile,
SessionID: req.SessionID,
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction,
Inputs: inputs,
}, &response)
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("complete structured output: %w", err)
materials := shared.PromptInputs(sourceInput, req.References)
inputs := make(map[string]promptkit.ArtifactRef, len(materials)+2)
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
inputs[spellcatalog.SpellCatalogReferenceSlot] = promptkit.Inline(string(e.catalogPromptInput))
registryInput := npcRegistry.PromptInput()
inputs[NPCRegistryReferenceSlot] = promptkit.Inline(string(registryInput.Content))
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.SpellList, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
canonicalizeResponse(&response, order, sourceID)
value := canonicalSpellList(response, sourceID)
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{
PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))},
}, nil
},
func(context.Context, *dnd.SpellList) (dnd.SpellList, error) {
return dnd.SpellList{SpellCasts: []dnd.SpellCast{}}, nil
})
if err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("capture model candidate: %w", err)
return contracts.TypedExtractionResult[dnd.SpellList]{}, extractorErrorf("produce spells: %w", err)
}
canonicalizeResponse(&response, order, req.Source.ID)
return contracts.TypedExtractionResult[dnd.SpellList]{Value: canonicalSpellList(response, req.Source.ID), ModelCandidate: candidate}, nil
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, Outcome: outcome}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.SpellListKind,
ReferenceSlots: referenceSlots(),
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.SpellListKind,
ReferenceSlots: referenceSlots(),
}
}
@@ -222,7 +175,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
"go.maximumdirect.net/promptkit"
)
func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
@@ -25,8 +26,11 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
},
}}}
req := extractionRequest()
var prepared promptkit.PreparedRun
req.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value }
result, err := newExtractor(t, client).Extract(context.Background(), req)
extractor := newExtractor(t, client)
result, err := extractor.Extract(context.Background(), req)
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
@@ -44,27 +48,20 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
llmReq := client.requests[0]
if llmReq.StageName != Key || llmReq.PromptID != PromptID || llmReq.PromptVersion != SchemaVersion {
t.Fatalf("LLM request identity = %#v, want spell prompt", llmReq)
if prepared.PromptID != PromptID || prepared.SelectedProfileID != "profile-spells" || prepared.SessionID != "session-123" || llmReq.Target.Model != "spell-test-model" {
t.Fatalf("native selection lost: %#v", prepared)
}
if llmReq.SessionID != "session-123" || llmReq.ProfileID != "profile-spells" {
t.Fatalf("session/profile = %q/%q, want session-123/profile-spells", llmReq.SessionID, llmReq.ProfileID)
if !strings.Contains(promptText(llmReq), string(req.Chunk.Content)) {
t.Fatal("missing exact chunk bytes")
}
transcript := llmReq.Inputs["transcript"]
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" {
t.Fatalf("transcript metadata = %#v", transcript)
}
if got := string(transcript.Content); got != string(req.Chunk.Content) {
t.Fatalf("transcript content = %q, want chunk content %q", got, req.Chunk.Content)
}
catalogInput := llmReq.Inputs[spellcatalog.SpellCatalogReferenceSlot]
if catalogInput.Name != spellcatalog.SpellCatalogReferenceSlot || catalogInput.MediaType != "application/json" || catalogInput.OriginURI != "" || !strings.HasPrefix(catalogInput.Digest, "sha256:") {
t.Fatalf("catalog prompt input metadata = %#v", catalogInput)
catalogInput := extractor.catalogPromptInput
if !strings.Contains(promptText(llmReq), string(catalogInput)) {
t.Fatal("missing exact catalog projection in native request")
}
var catalogPayload struct {
Spells []spellcatalog.PromptSpell `json:"spells"`
}
if err := json.Unmarshal(catalogInput.Content, &catalogPayload); err != nil {
if err := json.Unmarshal(catalogInput, &catalogPayload); err != nil {
t.Fatalf("decode catalog prompt input: %v", err)
}
base, err := spellcatalog.LoadSRD5E2014()
@@ -90,15 +87,16 @@ func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
func TestExtractPromptProjectsCanonicalNamesAndAliasesWithoutMetadata(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
if _, err := newExtractor(t, client, overlaySpellCatalogReference()).Extract(context.Background(), extractionRequest()); err != nil {
extractor := newExtractor(t, client, overlaySpellCatalogReference())
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
input := client.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
content := string(input.Content)
input := extractor.catalogPromptInput
content := string(input)
var payload struct {
Spells []spellcatalog.PromptSpell `json:"spells"`
}
if err := json.Unmarshal(input.Content, &payload); err != nil {
if err := json.Unmarshal(input, &payload); err != nil {
t.Fatalf("decode catalog prompt input: %v", err)
}
var aegis *spellcatalog.PromptSpell
@@ -116,45 +114,6 @@ func TestExtractPromptProjectsCanonicalNamesAndAliasesWithoutMetadata(t *testing
t.Fatalf("catalog prompt input leaked %q: %s", forbidden, content)
}
}
metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata()
if metadata["mapping_policy"] != mappingPolicy {
t.Fatalf("mapping policy metadata = %#v, want %q", metadata["mapping_policy"], mappingPolicy)
}
if metadata["catalog_base_id"] != spellcatalog.SRD5E2014ID {
t.Fatalf("catalog base metadata = %#v", metadata["catalog_base_id"])
}
if digest, ok := metadata["catalog_digest"].(string); !ok || !strings.HasPrefix(digest, "sha256:") {
t.Fatalf("catalog digest metadata = %#v", metadata["catalog_digest"])
}
if got, ok := metadata["catalog_overlay_ids"].([]string); !ok || !reflect.DeepEqual(got, []string{"campaign.example"}) {
t.Fatalf("catalog overlay metadata = %#v", metadata["catalog_overlay_ids"])
}
fingerprints := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).CheckpointFingerprints()
wantFingerprints := map[string]any{
"effective_catalog": metadata["catalog_digest"],
"mapping_policy": mappingPolicy,
"prompt": metadata["prompt_sha256"],
"response_schema": metadata["response_schema_sha256"],
"npc_registry": checkpointFingerprintMap(newExtractor(t, &fakeSpellsLLMClient{}).CheckpointFingerprints())["npc_registry"],
}
if len(fingerprints) != len(wantFingerprints) {
t.Fatalf("checkpoint fingerprints = %#v, want prompt, response schema, catalog, and NPC projection identities", fingerprints)
}
for _, fingerprint := range fingerprints {
if want, ok := wantFingerprints[fingerprint.Name]; !ok || fingerprint.Value != want {
t.Fatalf("checkpoint fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want)
}
}
encoded, err := json.Marshal(metadata)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Private campaign source", "file:///private-source.json"} {
if strings.Contains(string(encoded), forbidden) {
t.Fatalf("manifest metadata leaked %q: %s", forbidden, encoded)
}
}
}
func TestExtractUsesAliasRecognitionToRequestCanonicalSpellNames(t *testing.T) {
@@ -174,13 +133,13 @@ func TestExtractUsesAliasRecognitionToRequestCanonicalSpellNames(t *testing.T) {
if got := result.Value.SpellCasts; len(got) != 1 || got[0].Spell != "Aegis of Emberfall" {
t.Fatalf("spell casts = %#v, want canonical spell name", got)
}
input := client.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
if !strings.Contains(string(input.Content), `"aliases":["Emberfall Aegis"]`) {
t.Fatalf("catalog prompt input = %s, want transcript alias recognition", input.Content)
content := promptText(client.requests[0])
if !strings.Contains(content, `"aliases":["Emberfall Aegis"]`) {
t.Fatalf("catalog prompt input = %s, want transcript alias recognition", content)
}
}
func TestExtractAliasOnlyCatalogChangesPromptMaterialAndCheckpointFingerprint(t *testing.T) {
func TestExtractAliasOnlyCatalogChangesPromptMaterial(t *testing.T) {
aliasReference := spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v1","catalogs":[{"id":"campaign.example","ruleset":"dnd-5e-2014","source":{"title":"Private campaign source","version":"1","url":"file:///private-source.json","license":"private"},"spells":[{"name":"Cure Wounds","aliases":["Campaign Woundweave"]}]}]}`)
baseClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
aliasClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
@@ -193,27 +152,16 @@ func TestExtractAliasOnlyCatalogChangesPromptMaterialAndCheckpointFingerprint(t
t.Fatalf("alias Extract() error = %v", err)
}
baseInput := baseClient.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
aliasInput := aliasClient.requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
if baseInput.Digest == aliasInput.Digest || string(baseInput.Content) == string(aliasInput.Content) {
t.Fatalf("alias-only catalog did not change prompt material: %q / %q", baseInput.Digest, aliasInput.Digest)
}
if !strings.Contains(string(aliasInput.Content), "Campaign Woundweave") {
t.Fatalf("alias prompt input = %s, want alias recognition", aliasInput.Content)
}
baseFingerprints := checkpointFingerprintMap(baseExtractor.CheckpointFingerprints())
aliasFingerprints := checkpointFingerprintMap(aliasExtractor.CheckpointFingerprints())
if baseFingerprints["effective_catalog"] == aliasFingerprints["effective_catalog"] {
t.Fatalf("effective catalog fingerprint did not change: %#v", aliasFingerprints)
}
if baseExtractor.ManifestMetadata()["prompt_id"] != aliasExtractor.ManifestMetadata()["prompt_id"] || baseExtractor.ManifestMetadata()["prompt_version"] != aliasExtractor.ManifestMetadata()["prompt_version"] {
t.Fatal("catalog-only change altered prompt identity")
baseInput := promptText(baseClient.requests[0])
aliasInput := promptText(aliasClient.requests[0])
if baseInput == aliasInput || !strings.Contains(aliasInput, "Campaign Woundweave") {
t.Fatal("alias-only catalog change was not reflected in native messages")
}
}
func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) {
client := &fakeSpellsLLMClient{}
_, err := New(client, Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`))
_, err := New(newSpellEngine(t, client), Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`))
if err == nil || !strings.Contains(err.Error(), "resolve effective spell catalog") {
t.Fatalf("New() error = %v, want effective catalog error", err)
}
@@ -222,33 +170,6 @@ func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) {
}
}
func TestExtractorManifestMetadataIncludesLLMSchemaProvenance(t *testing.T) {
metadata := newExtractor(t, &fakeSpellsLLMClient{}).ManifestMetadata()
tests := map[string]string{
"prompt_id": PromptID, "prompt_version": SchemaVersion,
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
}
for key, want := range tests {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
value, ok := metadata[key].(string)
if !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want sha256 value", key, metadata[key])
}
}
}
func TestNilExtractorManifestMetadata(t *testing.T) {
var extractor *Extractor
if metadata := extractor.ManifestMetadata(); metadata != nil {
t.Fatalf("nil extractor metadata = %#v, want nil", metadata)
}
}
func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
req := extractionRequest()
@@ -261,12 +182,11 @@ func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
inputs := client.requests[0].Inputs
if string(inputs["players"].Content) != "Alice: Aria Brightmantle" || string(inputs["party"].Content) != "Aria Brightmantle: party cleric" || string(inputs["glossary"].Content) != "Brightmantle: local temple name" {
t.Fatalf("reference inputs = %#v, want configured content", inputs)
}
if strings.Contains(string(inputs["transcript"].Content), "party cleric") {
t.Fatal("transcript input contains reference content")
text := promptText(client.requests[0])
for _, content := range []string{"Alice: Aria Brightmantle", "Aria Brightmantle: party cleric", "Brightmantle: local temple name", string(req.Chunk.Content)} {
if !strings.Contains(text, content) {
t.Fatalf("missing fixed input %q", content)
}
}
}
@@ -282,23 +202,35 @@ func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
}
}
func TestExtractPreservesEmptyAndMalformedValuesForTypedValidators(t *testing.T) {
tests := []struct {
name string
response extractionResponse
wantNil bool
}{
{name: "empty", response: extractionResponse{SpellCasts: []spellCastResponse{}}},
{name: "missing", response: extractionResponse{}, wantNil: true},
func TestExtractPreservesValidEmptyCollection(t *testing.T) {
result, err := newExtractor(t, &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[]}`)}).Extract(context.Background(), extractionRequest())
if err != nil || result.Value.SpellCasts == nil || len(result.Value.SpellCasts) != 0 {
t.Fatalf("empty collection: %#v %v", result, err)
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result, err := newExtractor(t, &fakeSpellsLLMClient{response: test.response}).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
func TestExtractDoesNotAssessNativeStructuralFailure(t *testing.T) {
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
t.Run(string(mode), func(t *testing.T) {
client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":null}`)}
request := extractionRequest()
request.Operation.Controls.Mode = mode
assessed := 0
request.Operation.Assess = func(context.Context, dnd.SpellList) (contracts.CandidateAssessment, error) {
assessed++
return contracts.CandidateAssessment{}, nil
}
if (result.Value.SpellCasts == nil) != test.wantNil || len(result.Value.SpellCasts) != 0 {
t.Fatalf("SpellCasts = %#v, want empty with nil=%t", result.Value.SpellCasts, test.wantNil)
result, err := newExtractor(t, client).Extract(context.Background(), request)
if assessed != 0 {
t.Fatal("structurally invalid native result reached typed validation")
}
if mode == contracts.ModeStrict {
if err == nil {
t.Fatal("strict native structural failure succeeded")
}
} else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible ||
result.Value.SpellCasts == nil || len(result.Value.SpellCasts) != 0 {
t.Fatalf("permissive native failure = %#v, %v", result, err)
}
})
}
@@ -319,11 +251,11 @@ func TestExtractRejectsInvalidRequests(t *testing.T) {
name string
extractor *Extractor
ctx context.Context
req contracts.TypedExtractionRequest
req contracts.TypedExtractionRequest[dnd.SpellList]
want string
}{
{name: "nil extractor", extractor: nilExtractor, ctx: context.Background(), req: validReq, want: "extractor"},
{name: "nil LLM client", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "LLM client"},
{name: "nil native engine", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "native engine"},
{name: "wrapped preflight failure", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"},
}
for _, test := range tests {
@@ -340,7 +272,7 @@ func TestExtractOrdersAndDeduplicatesEvidence(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
{Caster: "Borin", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(2, 2)},
{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []spellSourceRefResponse{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 1, EndUnitID: 2}}},
{Caster: "Narrator", Spell: "Unknown"},
{Caster: "Narrator", Spell: "Unknown", SourceRefs: []spellSourceRefResponse{}},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {

View File

@@ -1,7 +1,6 @@
package spells
import (
"bytes"
"context"
"encoding/json"
"strings"
@@ -9,101 +8,74 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
)
func TestSpellExtractorUsesExactUnboundNPCPromptAndOmitsRegistryIdentity(t *testing.T) {
extractor := newExtractor(t, &fakeSpellsLLMClient{})
metadata := extractor.ManifestMetadata()
if metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil {
t.Fatalf("unbound extractor metadata = %#v, want no NPC registry fields", metadata)
}
fingerprints := checkpointFingerprintMap(extractor.CheckpointFingerprints())
if fingerprints["npc_registry"] == "" {
t.Fatalf("unbound checkpoint fingerprints = %#v, want empty-projection identity", fingerprints)
}
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
extractor = newExtractor(t, client)
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
if input.Name != NPCRegistryReferenceSlot || input.MediaType != npccodec.MediaType || input.Digest != fingerprints["npc_registry"] || input.OriginURI != "" || string(input.Content) != `{"npcs":[]}` {
t.Fatalf("NPC prompt input = %#v, want exact empty registry material", input)
}
}
func TestSpellExtractorPreservesSemanticNPCRegistryFingerprintAndPromptWiring(t *testing.T) {
func TestSpellExtractorProjectsNPCNamesWithoutIdentityOrProvenance(t *testing.T) {
value := registryFixture()
canonical, err := npccodec.New().Encode(value)
if err != nil {
t.Fatalf("encode NPC registry: %v", err)
t.Fatal(err)
}
pretty, err := json.MarshalIndent(value, "", " ")
if err != nil {
t.Fatalf("MarshalIndent() error = %v", err)
t.Fatal(err)
}
first := newExtractor(t, &fakeSpellsLLMClient{}, spellNPCRegistryReference(canonical, "file:///one.json"))
second := newExtractor(t, &fakeSpellsLLMClient{}, spellNPCRegistryReference(pretty, "file:///two.json"))
firstFingerprints := checkpointFingerprintMap(first.CheckpointFingerprints())
secondFingerprints := checkpointFingerprintMap(second.CheckpointFingerprints())
if firstFingerprints["npc_registry"] == "" || firstFingerprints["npc_registry"] != secondFingerprints["npc_registry"] {
t.Fatalf("semantic NPC fingerprints = %#v and %#v, want same npc_registry value", firstFingerprints, secondFingerprints)
}
metadata := first.ManifestMetadata()
if metadata["npc_registry_digest"] == "" || metadata["npc_registry_digest"] == firstFingerprints["npc_registry"] || metadata["npc_count"] != 1 {
t.Fatalf("NPC registry metadata = %#v, want digest and count only", metadata)
}
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
extractor := newExtractor(t, client, spellNPCRegistryReference(append([]byte("\n"), canonical...), "file:///npc-session.json"))
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
if input.Name != NPCRegistryReferenceSlot || input.MediaType != npccodec.MediaType || input.Digest != firstFingerprints["npc_registry"] || input.OriginURI != "" {
t.Fatalf("NPC prompt input metadata = %#v, want semantic metadata without provenance", input)
}
if !bytes.Equal(input.Content, []byte(`{"npcs":[{"name":"Mira Thorn"}]}`)) {
t.Fatalf("NPC prompt input = %s, want names-only projection", input.Content)
}
encoded, err := json.Marshal(map[string]any{"metadata": metadata, "fingerprints": firstFingerprints})
if err != nil {
t.Fatalf("marshal metadata: %v", err)
}
for _, forbidden := range []string{"Mira Thorn", "The Greencloak", "one.json", "two.json"} {
if strings.Contains(string(encoded), forbidden) {
t.Fatalf("metadata or fingerprints leaked %q: %s", forbidden, encoded)
}
var previous string
for _, test := range []struct {
name string
references contracts.ReferenceSet
want string
}{
{"unbound", contracts.ReferenceSet{}, `{"npcs":[]}`},
{"canonical", spellNPCRegistryReference(canonical, "file:///one.json"), `{"npcs":[{"name":"Mira Thorn"}]}`},
{"different formatting and origin", spellNPCRegistryReference(pretty, "file:///two.json"), `{"npcs":[{"name":"Mira Thorn"}]}`},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[]}`)}
if _, err := newExtractor(t, client, test.references).Extract(context.Background(), extractionRequest()); err != nil {
t.Fatal(err)
}
text := promptText(client.requests[0])
if !strings.Contains(text, test.want) {
t.Fatalf("missing names-only projection %q", test.want)
}
for _, private := range []string{value.NPCs[0].ID, "npc-session", "file:///one.json", "file:///two.json"} {
if strings.Contains(text, private) {
t.Fatalf("NPC prompt leaked %q", private)
}
}
if test.name == "different formatting and origin" && text != previous {
t.Fatal("semantic NPC projection changed with provenance/formatting")
}
previous = text
})
}
}
func TestSpellExtractorResolvesOperationNPCOverrideWithoutSingletonMetadata(t *testing.T) {
func TestSpellExtractorResolvesOperationNPCOverride(t *testing.T) {
canonical, err := npccodec.New().Encode(registryFixture())
if err != nil {
t.Fatalf("encode NPC registry: %v", err)
t.Fatal(err)
}
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[]}`)}
extractor := newExtractor(t, client)
request := extractionRequest()
request.References = spellNPCRegistryReference(canonical, "file:///generated.json")
if _, err := extractor.Extract(context.Background(), request); err != nil {
t.Fatalf("Extract() error = %v", err)
t.Fatal(err)
}
input := client.requests[0].Inputs[NPCRegistryReferenceSlot]
if input.Digest == "" || string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" {
t.Fatalf("operation NPC input = %#v, want generated canonical grounding without provenance", input)
if !strings.Contains(promptText(client.requests[0]), `{"npcs":[{"name":"Mira Thorn"}]}`) {
t.Fatal("missing operation NPC projection")
}
if metadata := extractor.ManifestMetadata(); metadata["npc_registry_digest"] != nil || metadata["npc_count"] != nil {
t.Fatalf("singleton metadata = %#v, want no operation-varying NPC identity", metadata)
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatal(err)
}
if checkpointFingerprintMap(extractor.CheckpointFingerprints())["npc_registry"] == "" {
t.Fatalf("singleton fingerprints = %#v, want empty-projection identity", extractor.CheckpointFingerprints())
if !strings.Contains(promptText(client.requests[1]), `{"npcs":[]}`) {
t.Fatal("operation override mutated seeded resolver")
}
}
@@ -128,11 +100,3 @@ func spellNPCRegistryReference(content []byte, origin string) contracts.Referenc
},
}}
}
func checkpointFingerprintMap(values []pipeline.CheckpointFingerprint) map[string]string {
result := make(map[string]string, len(values))
for _, value := range values {
result[value.Name] = value.Value
}
return result
}

View File

@@ -0,0 +1,203 @@
package spells
import (
"context"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
spellcatalogvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
spellsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
// Count the real validator boundary, not a second implementation of its rules.
type observedSpellValidator struct {
contracts.TypedValidator[dnd.SpellList]
record func(string)
}
func (v observedSpellValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
v.record(v.Name())
return v.TypedValidator.Validate(ctx, req)
}
type spellPipelineOutput struct{}
func (spellPipelineOutput) Key() string { return "test/spell-output" }
func (spellPipelineOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) {
response := func(name string) string {
return fmt.Sprintf(`{"spell_casts":[{"caster":"Aria","spell":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name)
}
for _, test := range []struct {
name string
mode contracts.ExecutionMode
responses []string
wantError bool
fallbacks int
}{
{"initial acceptance", contracts.ModeStrict, []string{response("Cure Wounds")}, false, 0},
{"final correction accepted", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Cure Wounds")}, false, 0},
{"strict exhaustion", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, true, 0},
{"permissive empty fallback", contracts.ModePermissive, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, false, 1},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeSpellsLLMClient{outputs: append([]string(nil), test.responses...)}
engine := newSpellEngine(t, client)
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
check(transcript.Register(registries.Inputs))
check(units.Register(registries.Chunkers))
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, spellcodec.New()))
check(Register(registries.Extractors))
check(appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, func(values []dnd.SpellList) (dnd.SpellList, error) {
result := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
for _, value := range values {
result.SpellCasts = append(result.SpellCasts, value.SpellCasts...)
}
return result, nil
}))
check(noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind))
check(registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/spell-output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic},
func() (contracts.OutputEncoder, error) { return spellPipelineOutput{}, nil }))
catalogValidator, err := spellcatalogvalidator.New(spellcatalogvalidator.Options{})
check(err)
validators := []contracts.TypedValidator[dnd.SpellList]{
spellshape.New(spellshape.Options{}), spellsourcerefs.New(spellsourcerefs.Options{}),
catalogValidator, spellsourcerelatedness.New(spellsourcerelatedness.Options{}),
}
var mu sync.Mutex
var calls []string
record := func(name string) { mu.Lock(); defer mu.Unlock(); calls = append(calls, name) }
var bindings []pipeline.ModuleBinding
var names []string
for _, validator := range validators {
check(pipeline.RegisterTypedValidator(registries.Validators, dnd.SpellListKind,
pipeline.ValidatorSpec{Key: validator.Name(), ExecutionClass: contracts.ExecutionClassDeterministic},
func() (contracts.TypedValidator[dnd.SpellList], error) {
return observedSpellValidator{validator, record}, nil
}))
bindings = append(bindings, pipeline.Binding(validator.Name()))
names = append(names, validator.Name())
}
partyFile := filepath.Join(t.TempDir(), "party.txt")
check(os.WriteFile(partyFile, []byte("party-grounding-sentinel: Aria"), 0600))
turns := 2
profile := pipeline.PipelineProfile{
ID: "native-spells", Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding("test/spell-output"),
LLMProfile: "profile-spells",
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {
Extract: pipeline.ModuleBinding{Module: Key, Mode: &test.mode, MaxAdditionalTurns: &turns,
Validators: pipeline.ValidatorOverride{Set: true, Validators: bindings},
References: map[string]pipeline.ReferenceSource{"party": pipeline.ExternalReference(partyFile)}},
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
}},
}
catalog := pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
check(err)
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
check(err)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
check(err)
model := "run-wide-spell-model"
var preparations []promptkit.PreparedRun
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", ExtractWorkers: 1,
RawInput: []byte(`{"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`),
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model},
OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }},
})
if (err != nil) != test.wantError {
t.Fatalf("pipeline error = %v, want error %t", err, test.wantError)
}
wantCalls := len(test.responses)
if len(client.requests) != wantCalls || len(preparations) != wantCalls {
t.Fatalf("provider/preparation calls = %d/%d, want %d", len(client.requests), len(preparations), wantCalls)
}
var expected []string
for range wantCalls {
expected = append(expected, names...)
}
if !reflect.DeepEqual(calls, expected) {
t.Fatalf("ordered assessments = %v, want %v (no outer assessment)", calls, expected)
}
for i, request := range client.requests {
if request.Target.Model != model || request.Prompt.SessionID != "spell-session-override" || preparations[i].SelectedProfileID != "profile-spells" {
t.Fatal("native profile/session/execution override lost")
}
wantID := PromptID
if i > 0 {
wantID = CorrectionPromptID
}
if preparations[i].PromptID != wantID {
t.Fatalf("prompt = %q, want %q", preparations[i].PromptID, wantID)
}
if i == 0 {
continue
}
previous := client.requests[i-1].Prompt.Messages
current := request.Prompt.Messages
if len(current) != len(previous)+2 || !reflect.DeepEqual(current[:len(previous)], previous) {
t.Fatal("native continuation did not retain exact ordered history")
}
if current[len(previous)].Role != "assistant" || current[len(previous)].Content != test.responses[i-1] ||
!strings.Contains(current[len(previous)+1].Content, "recognized D&D spell") {
t.Fatal("correction lost the actual candidate or catalog guidance")
}
if strings.Count(promptText(request), "party-grounding-sentinel") != 1 {
t.Fatal("reference bytes were regenerated in correction")
}
}
if test.wantError {
return
}
if output.FallbackCount != test.fallbacks || len(output.NormalizeOutputs) != 1 {
t.Fatalf("outcome = %#v", output)
}
value, err := spellcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content)
check(err)
if test.fallbacks > 0 {
if value.SpellCasts == nil || len(value.SpellCasts) != 0 {
t.Fatalf("invalid empty fallback: %#v", value)
}
} else if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" || value.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("durable spell artifact = %#v", value)
}
})
}
}

View File

@@ -1,76 +0,0 @@
package spells
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: "dnd.spells",
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "spell-catalog.md", Path: "prompts/spell-catalog.md"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-transcript-chunk.md",
"common-dnd-references.md",
"common-dnd-npc-registry.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/spells")
if err != nil {
return nil, fmt.Errorf("scope spell assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare spell prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope spell schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -2,111 +2,57 @@ package spells
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
"go.maximumdirect.net/promptkit"
)
func TestPromptAssetsPrepareSpellPrompt(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[{"id":"u1","text":"spell-transcript"}]}`)
prepared := prepareSpellsPrompt(t, transcript, "spell-player", "Mira: wizard", "Shield: abjuration")
if prepared.PromptID != PromptID {
t.Fatalf("prompt id = %q, want %q", prepared.PromptID, PromptID)
}
if prepared.OutputContract.SchemaPath != "dnd_spells_llm.v1.json" {
t.Fatalf("schema path = %q, want LLM-only schema", prepared.OutputContract.SchemaPath)
}
}
func TestPromptAssetsPrepareWithMissingOptionalReferences(t *testing.T) {
transcript := []byte(`{"id":"session-1","segments":[]}`)
prepared := prepareSpellsPrompt(t, transcript, " ", " ", " ")
for _, message := range prepared.Messages {
if strings.Contains(message.Content, "Player list reference:") && strings.Contains(message.Content, "Party roster reference:") && strings.Contains(message.Content, "Glossary reference:") {
return
func TestSpellPromptPairSharesNativeContractAndDefaultProfile(t *testing.T) {
client := &fakeSpellsLLMClient{}
engine := newSpellEngine(t, client)
var initial promptkit.PreparedRun
for _, id := range []string{PromptID, CorrectionPromptID} {
inputs := map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Fix the rejected cast.")}
if id == PromptID {
inputs = map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"units":[1,2]}`),
"spell_catalog": promptkit.Inline(`{"spells":[]}`),
"npc_registry": promptkit.Inline(`{"npcs":[]}`),
"players": promptkit.Inline(" "), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
}
}
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: id, APIKey: "test-only", Inputs: inputs})
if err != nil {
t.Fatal(err)
}
prepared := handle.Details()
handle.Discard()
if prepared.SelectedProfileID != "notarius/dnd-extraction" {
t.Fatalf("unexpected default profile %q", prepared.SelectedProfileID)
}
if prepared.StructuredOutput == nil || prepared.OutputContract.SchemaPath != "dnd/spells/schemas/dnd_spells_llm.v1.json" {
t.Fatal("missing native spell response schema")
}
if id == PromptID {
initial = prepared
var system string
for _, message := range prepared.Messages {
if message.Role == "system" {
system += message.Content
}
}
if !strings.Contains(system, "canonical spelling exactly") || !strings.Contains(system, "in-world caster") {
t.Fatal("enduring spell constraints missing from initial system context")
}
} else {
if !reflect.DeepEqual(initial.OutputContract, prepared.OutputContract) || len(prepared.Messages) != 1 || prepared.Messages[0].Role != "user" || !strings.Contains(prepared.Messages[0].Content, "Fix the rejected cast.") {
t.Fatal("correction must supply only findings and a replacement assignment with the same contract")
}
}
}
t.Fatalf("reference message did not render empty optional reference placeholders")
}
func TestPromptAssetDiagnosticsOmitRawMaterials(t *testing.T) {
transcript := []byte(`{"secret":"source text"}`)
reference := "private party note"
prepared := prepareSpellsPrompt(t, transcript, "private player note", reference, " ")
metadata := newExtractor(t, &fakeSpellsLLMClient{}, overlaySpellCatalogReference()).ManifestMetadata()
payload, err := json.Marshal(map[string]any{
"prepared": map[string]any{
"prompt_id": prepared.PromptID,
"prompt_version": prepared.PromptVersion,
"prompt_hash": prepared.PromptHash,
"rendered_prompt_hash": prepared.RenderedPromptHash,
"selected_profile_id": prepared.SelectedProfileID,
"output_contract": prepared.OutputContract,
"input_hashes": prepared.InputHashes,
"effective_model_params": prepared.EffectiveModelParams,
},
"manifest": metadata,
})
if err != nil {
t.Fatalf("marshal diagnostics: %v", err)
}
diagnostics := string(payload)
for _, forbidden := range []string{
"source text",
"private player note",
reference,
"Cure Wounds",
"Aegis of Emberfall",
"Emberfall Aegis",
"Private campaign source",
"file:///private-source.json",
`"properties"`,
"spell_casts",
} {
if strings.Contains(diagnostics, forbidden) {
t.Fatalf("diagnostics leaked %q: %s", forbidden, diagnostics)
}
}
if metadata["prompt_id"] != PromptID || metadata["prompt_version"] != SchemaVersion {
t.Fatalf("manifest prompt metadata = %#v", metadata)
}
if !strings.HasPrefix(metadata["prompt_sha256"].(string), "sha256:") {
t.Fatalf("manifest prompt hash = %#v, want sha256-prefixed", metadata["prompt_sha256"])
if len(client.requests) != 0 {
t.Fatal("preparation invoked generation")
}
}
func prepareSpellsPrompt(t *testing.T, transcript []byte, players string, party string, glossary string) *promptkit.PreparedRun {
t.Helper()
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("register spell prompt assets: %v", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"spell-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "spell-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: "spell-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", string(transcript)),
"spell_catalog": promptkit.Inline(`{"spells":[{"canonical_name":"spell-catalog-sentinel","aliases":["spell-alias-sentinel"]}]}`),
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"spell-npc-sentinel"}]}`),
"players": promptkit.Inline(players),
"party": promptkit.Inline(party),
"glossary": promptkit.Inline(glossary),
},
})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
return prepared
}

View File

@@ -10,9 +10,9 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestNewRequiresLLMClientAndReturnsExtractor(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
t.Fatalf("New(nil) error = %v, want LLM client error", err)
func TestNewRequiresNativeEngineAndReturnsExtractor(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v, want native engine error", err)
}
extractor := newExtractor(t, &fakeSpellsLLMClient{})
if extractor.Key() != Key {
@@ -23,10 +23,10 @@ func TestNewRequiresLLMClientAndReturnsExtractor(t *testing.T) {
func TestModuleSpec(t *testing.T) {
got := ModuleSpec()
want := pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: []string{
"chunks",
"source.transcript",
@@ -75,6 +75,7 @@ func TestModuleSpec(t *testing.T) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
got.Prompts[0] = "changed"
got.Requires[0] = "changed"
got.Provides[0] = "changed"
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"

View File

@@ -1,24 +0,0 @@
package spells
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.spells"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_spells_llm")
ResponseSchemaID = "notarius.dnd.spells.llm"
ResponseSchemaName = "notarius_dnd_spells_llm_v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_spells_llm.v1.json",
})
}

View File

@@ -3,38 +3,18 @@ package spells
import (
"bytes"
"encoding/json"
"strings"
"io/fs"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"github.com/santhosh-tekuri/jsonschema/v6"
)
func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) {
schema, err := loadResponseSchema()
func TestSpellResponseSchemaPreservesPrivateSelectionContract(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/spells/schemas/dnd_spells_llm.v1.json")
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if got, want := string(schema.Key), "dnd_spells_llm"; got != want {
t.Fatalf("schema key = %q, want %q", got, want)
}
if got, want := schema.ID, "notarius.dnd.spells.llm"; got != want {
t.Fatalf("schema ID = %q, want %q", got, want)
}
if got, want := schema.Version, "v1"; got != want {
t.Fatalf("schema version = %q, want %q", got, want)
}
if got, want := schema.Name, "notarius_dnd_spells_llm_v1"; got != want {
t.Fatalf("schema name = %q, want %q", got, want)
}
if !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema metadata = %#v, want valid hashed JSON", schema)
}
var schemaDocument map[string]any
if err := json.Unmarshal(schema.JSONSchema, &schemaDocument); err != nil {
t.Fatalf("Unmarshal(schema.JSONSchema) error = %v, want nil", err)
}
if schemaDocument["$id"] != "notarius.dnd.spells.llm" {
t.Fatalf("schema $id = %#v, want private LLM schema ID", schemaDocument["$id"])
t.Fatal(err)
}
valid := validSpellsResponse()
@@ -42,7 +22,7 @@ func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) {
if err != nil {
t.Fatalf("Marshal(validSpellsResponse()) error = %v, want nil", err)
}
if err := validateJSONSchema(validJSON, schema.JSONSchema); err != nil {
if err := validateJSONSchema(validJSON, schema); err != nil {
t.Fatalf("valid private spells response rejected: %v", err)
}
for _, test := range []struct {
@@ -96,13 +76,13 @@ func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = validateJSONSchema(content, schema.JSONSchema)
err = validateJSONSchema(content, schema)
if (err == nil) != test.valid {
t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid)
}
})
}
if err := validateJSONSchema([]byte(`{"spell_casts":`), schema.JSONSchema); err == nil {
if err := validateJSONSchema([]byte(`{"spell_casts":`), schema); err == nil {
t.Fatal("validateJSONSchema() error = nil, want malformed JSON rejected")
}
@@ -112,37 +92,11 @@ func TestLoadResponseSchemaUsesExtractorOwnedLLMSchema(t *testing.T) {
if err != nil {
t.Fatalf("Marshal(response with source_id) error = %v, want nil", err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
if err := validateJSONSchema(content, schema); err == nil {
t.Fatal("validateJSONSchema() error = nil, want canonical source_id rejected")
}
}
func TestResponseSchemaJSONIsMutationSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v", err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
t.Fatalf("second schema = %s, %v; want defensive valid copy", second.JSONSchema, err)
}
}
func TestResponseSchemaDiagnosticsOmitRawSchema(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v", err)
}
diagnostics := schema.DiagnosticsMap()
if diagnostics["key"] != ResponseSchemaKey {
t.Fatalf("diagnostics = %#v, want response key", diagnostics)
}
if _, ok := diagnostics["json_schema"]; ok {
t.Fatalf("diagnostics include raw schema: %#v", diagnostics)
}
}
func validSpellsResponse() map[string]any {
return map[string]any{
"spell_casts": []any{

View File

@@ -3,15 +3,20 @@ package spells
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
)
func promptExtractionRequest() contracts.TypedExtractionRequest {
func promptExtractionRequest() contracts.TypedExtractionRequest[dnd.SpellList] {
doc := promptSourceDocument()
chunk := &source.Chunk{
ID: "session-alpha:chunk:0",
@@ -27,9 +32,12 @@ func promptExtractionRequest() contracts.TypedExtractionRequest {
Units: append([]source.SourceUnit(nil), doc.Units...),
Metadata: map[string]any{"ignored": "chunk metadata"},
}
return contracts.TypedExtractionRequest{
return contracts.TypedExtractionRequest[dnd.SpellList]{
Source: doc,
Chunk: chunk,
Operation: contracts.ProducerOperation[dnd.SpellList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.SpellList) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}},
}
}
@@ -91,7 +99,7 @@ func spellChunkInput(chunk *source.Chunk) contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json")
}
func extractionRequest() contracts.TypedExtractionRequest {
func extractionRequest() contracts.TypedExtractionRequest[dnd.SpellList] {
req := promptExtractionRequest()
req.SourceInput = spellChunkInput(req.Chunk)
req.SessionID = "session-123"
@@ -99,14 +107,14 @@ func extractionRequest() contracts.TypedExtractionRequest {
return req
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.SpellList]) contracts.TypedExtractionRequest[dnd.SpellList] {
req.SourceInput = spellSourceInput()
return req
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
extractor, err := New(newSpellEngine(t, client), Options{}, references...)
if err != nil {
t.Fatalf("New() error = %v, want nil", err)
}
@@ -133,49 +141,64 @@ type fakeSpellsLLMClient struct {
response extractionResponse
content []byte
err error
requests []contracts.StructuredCompletionRequest
requests []promptkit.GenerateRequest
outputs []string
}
func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
func (client *fakeSpellsLLMClient) Generate(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.requests = append(client.requests, req)
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
return nil, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
content := append([]byte(nil), client.content...)
if len(content) != 0 {
if err := json.Unmarshal(content, target); err != nil {
return contracts.StructuredCompletionResponse{}, err
if len(client.outputs) > 0 {
content := client.outputs[0]
if len(client.outputs) > 1 {
client.outputs = client.outputs[1:]
}
} else {
*target = client.response
return &promptkit.GenerateResponse{Content: content}, nil
}
content := client.content
if len(content) == 0 {
var err error
content, err = json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
}
return contracts.StructuredCompletionResponse{Content: content}, nil
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
func promptText(request promptkit.GenerateRequest) string {
var text []string
for _, message := range request.Prompt.Messages {
text = append(text, message.Content)
}
req.Correction = correction
if len(req.Vars) == 0 {
req.Vars = nil
return req
}
vars := make(map[string]any, len(req.Vars))
for key, value := range req.Vars {
vars[key] = value
}
req.Vars = vars
return req
return strings.Join(text, "\n")
}
func newSpellEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
t.Helper()
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
config, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("spell provider fixture", []byte(`promptkit:
profiles:
- id: profile-spells
endpoint: https://example.invalid/v1
model: spell-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(config, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
return engine
}