Migrate item occurrences to native correction while preserving holder semantics

This commit is contained in:
2026-11-22 01:08:02 +00:00
parent 2c1d727232
commit 1fe231b6fb
14 changed files with 519 additions and 486 deletions

View File

@@ -0,0 +1,7 @@
Revise the previous item occurrence list using these findings:
{{ input "guidance" }}
Return the complete replacement list using the original transcript and registry
context. Preserve stated quantities and possession changes without inventing an
item or holder. Return only JSON matching the original response contract.

View File

@@ -20,7 +20,7 @@ prompts:
messages:
- role: system
content_file: ../../shared/prompts/common-dnd-system.md
- role: user
- role: system
content_file: ../../shared/prompts/common-dnd-identity.md
- role: user
content_file: ../../shared/prompts/common-dnd-references.md
@@ -30,11 +30,11 @@ prompts:
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
cache_control:
type: ephemeral
- role: user
- role: system
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
- role: user
content_file: ./item-registry.md
- role: user
- role: system
content_file: ./instructions.md
cache_control:
type: ephemeral
@@ -42,3 +42,17 @@ prompts:
format: json
validation_mode: json_schema
schema_path: dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json
- id: dnd.item_occurrences.repair
default_profile: notarius/dnd-extraction
inputs:
- name: guidance
required: true
content_type: text/plain
messages:
- role: user
content_file: ./correction.md
output:
format: json
validation_mode: json_schema
schema_path: dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json

View File

@@ -0,0 +1,79 @@
package assets_test
import (
"encoding/json"
"io/fs"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
)
func TestItemOccurrenceSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json")
if err != nil {
t.Fatal(err)
}
valid := map[string]any{"occurrences": []any{
map[string]any{
"name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party",
"source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}},
},
map[string]any{
"name": "Hidden Cache", "kind": "discovered", "quantity": nil, "from": nil, "to": nil,
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
},
}}
content, err := json.Marshal(valid)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema); err != nil {
t.Fatalf("private schema rejected validator-owned values: %v", err)
}
for _, test := range []struct {
name string
value map[string]any
}{
{"missing occurrences", map[string]any{}},
{"missing occurrence name", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "name")}}},
{"missing nullable field", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "quantity")}}},
{"missing from", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "from")}}},
{"missing to", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "to")}}},
{"noninteger quantity", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "quantity", 1.5)}}},
{"unsupported kind", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "kind", "unsupported")}}},
{"opaque item identifier", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "item_id", "item:sha256:opaque")}}},
{"unknown occurrence field", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "extra", true)}}},
{"segment-named range", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}},
{"opaque source identifier", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "source_id": "session-alpha"}})}}},
{"unknown reference field", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "extra": true}})}}},
{"noninteger range", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 1}})}}},
} {
t.Run(test.name, func(t *testing.T) {
content, err := json.Marshal(test.value)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema); err == nil {
t.Fatal("private schema accepted invalid structure")
}
})
}
}
func itemOccurrenceRecord() map[string]any {
return map[string]any{
"name": "Ring", "kind": "acquired", "quantity": nil, "from": nil, "to": "party", "source_refs": []any{},
}
}
func itemOccurrenceWithoutField(value map[string]any, name string) map[string]any {
delete(value, name)
return value
}
func itemOccurrenceWithField(value map[string]any, name string, fieldValue any) map[string]any {
value[name] = fieldValue
return value
}

View File

@@ -18,6 +18,15 @@ func TestOccurrencePromptPairsPreserveRegistryAndTranscriptBoundaries(t *testing
family, registry string
rules []string
}{
{"item", `{"items":[{"name":"occurrence-registry-name"}]}`, []string{
"Return both `from` and `to` for every occurrence.",
"| `discovered` | `null` | `null` |",
"| `acquired` | `null` | `party` or the named party member gaining possession |",
"| `lost` | `party` or the named party member losing possession | `null` |",
"| `consumed` | `party` or the named party member consuming the item | `null` |",
"| `transferred` | one named party member | a different named party member |",
"payments are always `lost`, not `consumed`.",
}},
{"npc", `{"npcs":[{"name":"occurrence-registry-name"}]}`, []string{"Use the supplied canonical NPC", "noncombat_presence", "combat_opponent"}},
{"location", `{"locations":[{"name":"occurrence-registry-name","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry-context"}]}]}`,
[]string{"context supports that coreference", "must not create a registry location", "provenance must never replace current-chunk evidence", "complete ordered"}},

View File

@@ -902,6 +902,28 @@ Do not relax existing domain invariants merely to make the migration tests pass.
## Stage 20: Migrate Item Occurrence Extraction
**Status:** Complete. Item occurrences now use native initial/correction requests
through the shared typed producer helper with fixed transcript and names-only
registry materials. Added the same-schema/default-profile correction pair and
static dependencies; removed private completion, prompt/schema registration,
and manifest/fingerprint machinery. Existing private nullable-field mapping,
quantity copying, holder rules, canonical identity, source references, and event
ordering remain unchanged. Unknown selections provide module-authored guidance
before typed assessment, without partial output or invented items. Permissive
terminal output is an explicit codec-valid empty list, not a repaired holder or
ordinary acceptance. Profile/session/execution selections and incoming reuse
ineligibility are preserved. Migrated the holder regression to a native provider
fake with all four typed validators, exact defective-response history, contextual
guidance, final-turn acceptance, and no duplicate final assessment. Added terminal
mode, unknown-item, failed-continuation, structural-failure, cancellation, all-kind,
ordering, and durable nullable-holder cases. Native prompt-pair/holder-principle
and unchanged structural-schema tests run independently in the assets package.
Asset tests/vet/race checks, focused pipeline tests, and `git diff --check` pass.
The extractor and all five occurrence-validator package checks remain blocked at
setup by shared legacy helpers importing old ConfigKit's removed YAML
`plugin/limit`; the new native runtime tests are not claimed passing. Scheduled
dependency cleanup and final assembled verification must execute those tests.
**Goal:** retain the smaller-model holder reliability fixes in the native path.
**Inspect/change:** `internal/modules/dnd/extract/itemoccurrences`, associated

View File

@@ -3,8 +3,12 @@ package itemoccurrences
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -14,14 +18,14 @@ import (
)
const Key = "dnd/item-occurrences"
const PromptID = "notarius/dnd.item_occurrences"
const CorrectionPromptID = "notarius/dnd.item_occurrences.repair"
const (
ItemRegistryReferenceSlot = itemregistry.ReferenceSlot
ItemRegistryMaxBytes = itemregistry.MaxBytes
)
const mappingPolicy = "dnd.item_occurrences.extract_mapping.v3"
var requiredCapabilities = []string{
"chunks",
"source.transcript",
@@ -53,21 +57,17 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.ItemOccurrenceList] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
itemResolver *itemregistry.Resolver
promptSHA string
responseSchemaSHA string
engine *promptkit.Engine
itemResolver *itemregistry.Resolver
}
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
@@ -80,64 +80,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare item registry prompt input: %w", err)
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{llm: llmClient, itemResolver: itemResolver, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
return &Extractor{engine: engine, itemResolver: itemResolver}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
"mapping_policy": mappingPolicy,
"comparison_policy": shared.TextComparisonPolicy,
}
seeded := e.itemResolver.Seeded()
if seeded.Bound() {
metadata["item_registry_digest"] = seeded.Digest()
metadata["item_count"] = seeded.Count()
}
return metadata
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "comparison_policy", Value: shared.TextComparisonPolicy},
{Name: "item_registry", Value: e.itemResolver.Seeded().ProjectionDigest()},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("native engine must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
@@ -152,38 +107,50 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
if !registry.Bound() {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("item registry reference is required")
}
var response extractionResponse
inputs := shared.PromptInputs(sourceInput, req.References)
inputs[ItemRegistryReferenceSlot] = registry.PromptInput()
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction, Inputs: inputs,
}, &response)
if err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
sourceID := req.Source.ID
materials := shared.PromptInputs(sourceInput, req.References)
materials[ItemRegistryReferenceSlot] = registry.PromptInput()
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemOccurrenceList, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
value, err := canonicalItemOccurrenceList(response, order, sourceID, registry)
if err != nil {
return nil, []string{"Select the exact canonical item name from the supplied registry for every occurrence. Omit unsupported or ambiguous selections; do not invent an item. Return the complete replacement occurrence list, keeping current-transcript evidence separate from registry context."}, nil
}
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(context.Context, *dnd.ItemOccurrenceList) (dnd.ItemOccurrenceList, error) {
return dnd.ItemOccurrenceList{Occurrences: []dnd.ItemOccurrence{}}, nil
})
if err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err)
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err)
}
value, err := canonicalItemOccurrenceList(response, order, req.Source.ID, registry)
if err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("map item occurrence response: %w", err)
}
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, ModelCandidate: candidate}, nil
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, Outcome: outcome}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.ItemOccurrenceListKind,
ReferenceSlots: referenceSlots(),
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.ItemOccurrenceListKind,
ReferenceSlots: referenceSlots(),
}
}
@@ -193,7 +160,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -8,20 +8,14 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
itemidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
itemoccurrenceshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
)
func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
id := itemidentity.DeriveID("Torch")
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","from":"party","source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`)
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","quantity":null,"from":"party","to":null,"source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`)
client := &fakeItemOccurrencesLLMClient{content: append([]byte(nil), rawResponse...)}
req := extractionRequest()
req.References = itemRegistryReferences(t)
correction, err := contracts.NewSemanticCorrection([]byte(`{"occurrences":[]}`), "Keep the transcript-grounded item occurrence.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
req.Correction = correction
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
if err != nil {
t.Fatal(err)
@@ -33,28 +27,18 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
t.Fatalf("occurrence evidence = %#v, want current-source unit range", refs)
}
request := client.requests[0]
input := request.Inputs[ItemRegistryReferenceSlot]
if input.Name != ItemRegistryReferenceSlot || string(input.Content) != `{"items":[{"name":"Torch"}]}` || strings.Contains(string(input.Content), "item:sha256:") {
t.Fatalf("registry prompt input = %#v, want names-only projection", input)
input := renderedInput(t, request, ItemRegistryReferenceSlot)
if string(input) != `{"items":[{"name":"Torch"}]}` {
t.Fatalf("names-only registry = %s", input)
}
if request.Correction == nil || string(request.Correction.AssistantResponse) != `{"occurrences":[]}` || request.Correction.UserGuidance != "Keep the transcript-grounded item occurrence." {
t.Fatalf("correction = %#v, want exact request correction", request.Correction)
}
for _, material := range []string{string(request.Correction.AssistantResponse), request.Correction.UserGuidance} {
if strings.Contains(material, id) || strings.Contains(material, "item:sha256:") {
t.Fatalf("correction material leaked opaque item identity: %q", material)
for _, message := range request.Prompt.Messages {
if strings.Contains(message.Content, id) || strings.Contains(message.Content, "item:sha256:") {
t.Fatal("model context exposed durable item identity")
}
}
correction.AssistantResponse[0] = '['
if got := string(request.Correction.AssistantResponse); got != `{"occurrences":[]}` {
t.Fatalf("captured correction changed after caller mutation: %q", got)
}
if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || string(result.ModelCandidate.Response) != string(rawResponse) {
t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate)
}
client.content[0] = '['
if got := string(result.ModelCandidate.Response); got != string(rawResponse) {
t.Fatalf("model candidate changed after provider buffer mutation: %q", got)
if request.Prompt.SessionID != req.SessionID || result.Outcome.Disposition != contracts.ProducerAccepted {
t.Fatal("native selection or acceptance lost")
}
}
@@ -83,8 +67,8 @@ func TestExtractRejectsUnknownNameAfterValidOccurrence(t *testing.T) {
req := extractionRequest()
req.References = itemRegistryReferences(t)
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "occurrences[1].name is not in the item registry") {
t.Fatalf("Extract() error = %v, want occurrence index and unknown name", err)
if err == nil || result.Outcome.Disposition != "" {
t.Fatalf("Extract() error = %v, want atomic unknown-name failure", err)
}
if len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences)
@@ -114,12 +98,12 @@ func TestExtractAcceptsOnlyEmptyResponseForEmptyRegistry(t *testing.T) {
req.References = references
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
if test.wantErr {
if err == nil || !strings.Contains(err.Error(), "name is not in the item registry") || len(result.Value.Occurrences) != 0 {
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err)
}
return
}
if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible {
t.Fatalf("Extract() = %#v, %v; want present empty occurrences", result, err)
}
})
@@ -144,50 +128,9 @@ func TestExtractUsesOnlySupportedPromptInputs(t *testing.T) {
if _, err := newExtractor(t, client, req.References).Extract(context.Background(), req); err != nil {
t.Fatal(err)
}
if _, ok := client.requests[0].Inputs["unrelated"]; ok {
t.Fatalf("unexpected prompt input: %#v", client.requests[0].Inputs)
}
}
func TestExtractCarriesRejectedHolderGuidanceIntoCorrectedAttempt(t *testing.T) {
references := itemRegistryReferences(t)
req := extractionRequest()
req.References = references
defectiveResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
defective, err := newExtractor(t, &fakeItemOccurrencesLLMClient{content: defectiveResponse}, references).Extract(context.Background(), req)
if err != nil {
t.Fatal(err)
}
validation, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: defective.Value})
if err != nil || validation.Approved {
t.Fatalf("holder validation = %#v, %v; want rejection", validation, err)
}
for _, fragment := range []string{"For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
if !strings.Contains(validation.CorrectionGuidance, fragment) {
t.Fatalf("CorrectionGuidance = %q, want %q", validation.CorrectionGuidance, fragment)
for _, message := range client.requests[0].Prompt.Messages {
if strings.Contains(message.Content, "ignored") {
t.Fatal("unsupported reference became a prompt input")
}
}
for _, forbidden := range []string{defective.Value.Occurrences[0].ItemID, "item_id", "sha256", itemoccurrenceshape.Key, itemoccurrenceshape.ReasonCode} {
if strings.Contains(validation.CorrectionGuidance, forbidden) {
t.Fatalf("CorrectionGuidance leaked implementation identifier %q: %q", forbidden, validation.CorrectionGuidance)
}
}
correction, err := contracts.NewSemanticCorrection(defective.ModelCandidate.Response, validation.CorrectionGuidance)
if err != nil {
t.Fatal(err)
}
correctedResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
client := &fakeItemOccurrencesLLMClient{content: correctedResponse}
req.Correction = correction
corrected, err := newExtractor(t, client, references).Extract(context.Background(), req)
if err != nil {
t.Fatal(err)
}
accepted, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: corrected.Value})
if err != nil || !accepted.Approved {
t.Fatalf("corrected holder validation = %#v, %v; want approval", accepted, err)
}
if len(client.requests) != 1 || client.requests[0].Correction == nil || string(client.requests[0].Correction.AssistantResponse) != string(defectiveResponse) || client.requests[0].Correction.UserGuidance != validation.CorrectionGuidance {
t.Fatalf("corrected request = %#v, want byte-faithful defective response and semantic guidance", client.requests)
}
}

View File

@@ -0,0 +1,213 @@
package itemoccurrences
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
registryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/registry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const rejectedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
const correctedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
const unknownItem = `{"occurrences":[{"name":"Unknown Item","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
func bindAssessment(t *testing.T, request *contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (*[]string, *[]string) {
t.Helper()
registry, err := registryvalidator.New(registryvalidator.Options{}, request.References)
if err != nil {
t.Fatal(err)
}
validators := []contracts.TypedValidator[dnd.ItemOccurrenceList]{
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
relatedness.New(relatedness.Options{}), registry,
}
var calls, guidance []string
request.Operation.Assess = func(ctx context.Context, value dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
var assessment contracts.CandidateAssessment
for _, validator := range validators {
calls = append(calls, validator.Name())
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{
Stage: "extract", Source: request.Source, Chunk: request.Chunk, References: request.References, Value: value,
})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
if result.CorrectionGuidance != "" {
guidance = append(guidance, result.CorrectionGuidance)
}
}
return assessment, nil
}
return &calls, &guidance
}
func TestExtractCarriesRejectedHolderGuidanceIntoNativeCorrection(t *testing.T) {
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "item-occurrence-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, run promptkit.PreparedRun) { prepared = append(prepared, run) }
calls, guidance := bindAssessment(t, &request)
client := &fakeItemOccurrencesLLMClient{responses: []string{rejectedHolder, correctedHolder}}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, registryvalidator.Key}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Occurrences) != 1 || len(client.requests) != 2 || len(prepared) != 2 || len(*guidance) != 1 {
t.Fatalf("correction = %#v %v, validators=%#v guidance=%#v native=%d/%d", result, err, *calls, *guidance, len(client.requests), len(prepared))
}
occurrence := result.Value.Occurrences[0]
if occurrence.ItemID != identity.DeriveID("Torch") || occurrence.Name != "Torch" || occurrence.Kind != dnd.ItemOccurrenceKindAcquired ||
occurrence.From != "" || occurrence.To != "party" || occurrence.Quantity != nil || occurrence.SourceRefs[0].SourceID != request.Source.ID {
t.Fatalf("holder mapping = %#v", occurrence)
}
first, next := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(next) != len(first)+2 || !reflect.DeepEqual(first, next[:len(first)]) || next[len(first)].Role != "assistant" || next[len(first)].Content != rejectedHolder {
t.Fatal("native continuation did not preserve the exact defective response and fixed context")
}
correction := next[len(first)+1].Content
for _, fragment := range []string{(*guidance)[0], "For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
if !strings.Contains(correction, fragment) {
t.Fatalf("missing holder guidance %q", fragment)
}
}
for _, forbidden := range []string{occurrence.ItemID, "item_id", "sha256", shape.Key, shape.ReasonCode} {
if strings.Contains(correction, forbidden) {
t.Fatalf("operator identifier became guidance: %q", forbidden)
}
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "item-occurrence-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native profile/session/override lost")
}
if string(renderedInput(t, call, ItemRegistryReferenceSlot)) != `{"items":[{"name":"Torch"}]}` {
t.Fatal("item registry was not fixed and names-only")
}
}
content, err := occurrencecodec.New().Encode(result.Value)
if err != nil {
t.Fatal(err)
}
var wire struct {
Occurrences []map[string]json.RawMessage `json:"occurrences"`
}
if err := json.Unmarshal(content, &wire); err != nil {
t.Fatal(err)
}
if _, present := wire.Occurrences[0]["from"]; present {
t.Fatal("private null holder became a durable field")
}
}
func TestItemOccurrenceTerminalModesDoNotSynthesizeHolders(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
failures []error
assessments, runs int
wantError bool
}{
{"strict holder rejection", contracts.ModeStrict, rejectedHolder, nil, 2, 2, true},
{"permissive holder rejection", contracts.ModePermissive, rejectedHolder, nil, 2, 2, false},
{"strict unknown item", contracts.ModeStrict, unknownItem, nil, 0, 2, true},
{"permissive unknown item", contracts.ModePermissive, unknownItem, nil, 0, 2, false},
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed correction", contracts.ModePermissive, rejectedHolder, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
calls, _ := bindAssessment(t, &request)
runs := 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
client := &fakeItemOccurrencesLLMClient{content: []byte(test.response), errors: test.failures}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
t.Fatalf("terminal = %#v %v, validators=%#v runs=%d", result, err, *calls, runs)
}
if test.wantError {
if result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
t.Fatal("strict failure returned output")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
t.Fatalf("fallback invented an item or holder: %#v", result)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
})
}
}
func TestItemOccurrenceCancellationCannotPublishFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeItemOccurrencesLLMClient{content: []byte(correctedHolder), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("canceled delivery = %#v %v", result, err)
}
}
func TestItemOccurrenceKindsHoldersQuantityAndDocumentOrderSurvive(t *testing.T) {
quantity := 3
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
{Name: "Torch", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(5, 5)},
{Name: "Torch", Kind: "consumed", From: "Aria", SourceRefs: responseRefs(4, 4)},
{Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)},
{Name: "Torch", Kind: "acquired", To: "party", Quantity: &quantity, SourceRefs: responseRefs(2, 2)},
{Name: "Torch", Kind: "discovered", SourceRefs: responseRefs(1, 1)},
}}}
request := extractionRequest()
request.References = itemRegistryReferences(t)
bindAssessment(t, &request)
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Occurrences) != 5 {
t.Fatalf("kinds = %#v %v", result, err)
}
kinds := []dnd.ItemOccurrenceKind{dnd.ItemOccurrenceKindDiscovered, dnd.ItemOccurrenceKindAcquired, dnd.ItemOccurrenceKindLost, dnd.ItemOccurrenceKindConsumed, dnd.ItemOccurrenceKindTransferred}
for i, occurrence := range result.Value.Occurrences {
if occurrence.Kind != kinds[i] || occurrence.SourceRefs[0].StartUnitID != i+1 {
t.Fatalf("ordered occurrence = %#v", occurrence)
}
}
quantity = 9
if *result.Value.Occurrences[1].Quantity != 3 || result.Value.Occurrences[4].From != "Aria" || result.Value.Occurrences[4].To != "Borin" {
t.Fatal("holder or quantity interpretation changed")
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}

View File

@@ -1,75 +0,0 @@
package itemoccurrences
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: "dnd.item_occurrences",
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
{Name: "item-registry.md", Path: "prompts/item-registry.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-references.md",
"common-dnd-transcript-chunk.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/item-occurrences")
if err != nil {
return nil, fmt.Errorf("scope item-occurrence assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare item occurrence prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope item-occurrence schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,82 +0,0 @@
package itemoccurrences
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"item-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-occurrences-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[{"sentinel":"item-occurrence-transcript"}]}`),
"players": promptkit.Inline("item-occurrence-player"),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
"item_registry": promptkit.Inline(`{"items":[{"name":"Torch"}]}`),
},
})
if err != nil {
t.Fatal(err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_occurrences_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
rendered := make([]string, len(prepared.Messages))
for index, message := range prepared.Messages {
rendered[index] = message.Content
}
content := strings.Join(rendered, "\n")
for _, field := range []string{"start_unit_id", "end_unit_id"} {
if !strings.Contains(content, field) {
t.Fatalf("prepared prompt does not include shared evidence field %q", field)
}
}
for _, obsolete := range []string{"source_id", "start_segment", "end_segment"} {
if strings.Contains(content, obsolete) {
t.Fatalf("prepared prompt contains obsolete evidence field %q: %s", obsolete, content)
}
}
for _, holderRule := range []string{
"| `discovered` | `null` | `null` |",
"| `acquired` | `null` | `party` or the named party member gaining possession |",
"| `lost` | `party` or the named party member losing possession | `null` |",
"| `consumed` | `party` or the named party member consuming the item | `null` |",
"| `transferred` | one named party member | a different named party member |",
} {
if !strings.Contains(content, holderRule) {
t.Fatalf("prepared prompt does not include holder rule %q", holderRule)
}
}
}
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
hash, err := promptAssetMetadata()
if err != nil || !strings.HasPrefix(hash, "sha256:") {
t.Fatalf("promptAssetMetadata() = %q, %v", hash, err)
}
metadata := newExtractor(t, &fakeItemOccurrencesLLMClient{}).ManifestMetadata()
payload, err := json.Marshal(metadata)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"common-dnd-system", "dnd_item_occurrences_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
}
}
}

View File

@@ -36,8 +36,8 @@ func testSourceRefs() []source.SourceRef {
func TestModuleSpecDeclaresRequiredRegistry(t *testing.T) {
spec := ModuleSpec()
if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 {
t.Fatalf("correction protocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1)
if !reflect.DeepEqual(spec.Prompts, []string{PromptID, CorrectionPromptID}) {
t.Fatalf("prompt dependencies = %#v", spec.Prompts)
}
var slot contracts.ReferenceSlot
for _, candidate := range spec.ReferenceSlots {

View File

@@ -1,25 +0,0 @@
package itemoccurrences
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.item_occurrences"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_occurrences_llm")
ResponseSchemaID = "notarius.dnd.item_occurrences.llm"
ResponseSchemaName = "notarius_dnd_item_occurrences_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_item_occurrences_llm.v1.json",
})
}

View File

@@ -1,107 +0,0 @@
package itemoccurrences
import (
"bytes"
"encoding/json"
"strings"
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
)
func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v", schema)
}
valid := map[string]any{"occurrences": []any{
map[string]any{
"name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party",
"source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}},
},
map[string]any{
"name": "Hidden Cache", "kind": "discovered", "quantity": nil, "from": nil, "to": nil,
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
},
}}
content, err := json.Marshal(valid)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("private schema rejected validator-owned values: %v", err)
}
for _, test := range []struct {
name string
value map[string]any
}{
{"missing occurrences", map[string]any{}},
{"missing occurrence name", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "name")}}},
{"missing nullable field", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "quantity")}}},
{"unsupported kind", map[string]any{"occurrences": []any{withField(responseOccurrence(), "kind", "unsupported")}}},
{"opaque item identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "item_id", "item:sha256:opaque")}}},
{"unknown occurrence field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "extra", true)}}},
{"segment-named range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}},
{"opaque source identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "source_id": "session-alpha"}})}}},
{"unknown reference field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "extra": true}})}}},
{"noninteger range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 1}})}}},
} {
t.Run(test.name, func(t *testing.T) {
content, err := json.Marshal(test.value)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
t.Fatal("private schema accepted invalid structure")
}
})
}
first := schema.JSONSchema
first[0] = '['
second, err := loadResponseSchema()
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) {
t.Fatalf("defensive schema copy = %s, %v", second.JSONSchema, err)
}
if _, ok := second.DiagnosticsMap()["json_schema"]; ok {
t.Fatalf("schema diagnostics leaked content: %#v", second.DiagnosticsMap())
}
}
func responseOccurrence() map[string]any {
return map[string]any{
"name": "Ring", "kind": "acquired", "quantity": nil, "from": nil, "to": "party", "source_refs": []any{},
}
}
func withoutField(value map[string]any, name string) map[string]any {
delete(value, name)
return value
}
func withField(value map[string]any, name string, fieldValue any) map[string]any {
value[name] = fieldValue
return value
}
func validateJSONSchema(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return err
}
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return err
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", document); err != nil {
return err
}
compiled, err := compiler.Compile("schema.json")
if err != nil {
return err
}
return compiled.Validate(instance)
}

View File

@@ -3,14 +3,19 @@ package itemoccurrences
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func extractionRequest() contracts.TypedExtractionRequest {
func extractionRequest() contracts.TypedExtractionRequest[dnd.ItemOccurrenceList] {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-alpha:chunk:0",
@@ -21,12 +26,15 @@ func extractionRequest() contracts.TypedExtractionRequest {
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest{
return contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]{
Source: doc,
Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
SessionID: "session-123",
LLMProfile: "profile-item-occurrences",
Operation: contracts.ProducerOperation[dnd.ItemOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}},
}
}
@@ -47,51 +55,111 @@ func responseRefs(start, end int) []itemOccurrenceSourceRefResponse {
return []itemOccurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
profiles:
- id: profile-item-occurrences
endpoint: https://example.invalid/v1
model: occurrence-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatalf("New() error = %v", err)
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
extractor, err := New(engine, Options{}, references...)
if err != nil {
t.Fatal(err)
}
return extractor
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
t.Helper()
key := map[string]string{"transcript": "segments", "item_registry": "items"}[name]
for _, message := range request.Prompt.Messages {
start := strings.Index(message.Content, "{")
if start < 0 {
continue
}
content := []byte(message.Content[start:])
var fields map[string]json.RawMessage
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
return content
}
}
req.Correction = correction
return req
t.Fatalf("missing rendered %s input", name)
return nil
}
type fakeItemOccurrencesLLMClient struct {
response extractionResponse
content []byte
err error
requests []contracts.StructuredCompletionRequest
response extractionResponse
responses []string
content []byte
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeItemOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
func (client *fakeItemOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
return nil, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
content := append([]byte(nil), client.content...)
if len(content) == 0 {
var err error
content, err = json.Marshal(client.response)
content := string(client.content)
if index < len(client.responses) {
content = client.responses[index]
}
if content == "" {
encoded, err := completeResponse(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
content = string(encoded)
}
return &promptkit.GenerateResponse{Content: content}, nil
}
func completeResponse(response extractionResponse) ([]byte, error) {
content, err := json.Marshal(response)
if err != nil {
return nil, err
}
var wire struct {
Occurrences []map[string]any `json:"occurrences"`
}
if err := json.Unmarshal(content, &wire); err != nil {
return nil, err
}
for _, occurrence := range wire.Occurrences {
for _, name := range []string{"quantity", "from", "to"} {
if _, present := occurrence[name]; !present {
occurrence[name] = nil
}
}
}
if err := json.Unmarshal(content, target); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
return json.Marshal(wire)
}