Migrate item occurrences to native correction while preserving holder semantics
This commit is contained in:
7
assets/dnd/item-occurrences/prompts/correction.md
Normal file
7
assets/dnd/item-occurrences/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous item occurrence list using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return the complete replacement list using the original transcript and registry
|
||||
context. Preserve stated quantities and possession changes without inventing an
|
||||
item or holder. Return only JSON matching the original response contract.
|
||||
@@ -20,7 +20,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -30,11 +30,11 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./item-registry.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -42,3 +42,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json
|
||||
|
||||
- id: dnd.item_occurrences.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json
|
||||
|
||||
79
assets/item_occurrence_test.go
Normal file
79
assets/item_occurrence_test.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
)
|
||||
|
||||
func TestItemOccurrenceSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
valid := map[string]any{"occurrences": []any{
|
||||
map[string]any{
|
||||
"name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}},
|
||||
},
|
||||
map[string]any{
|
||||
"name": "Hidden Cache", "kind": "discovered", "quantity": nil, "from": nil, "to": nil,
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
},
|
||||
}}
|
||||
content, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err != nil {
|
||||
t.Fatalf("private schema rejected validator-owned values: %v", err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
value map[string]any
|
||||
}{
|
||||
{"missing occurrences", map[string]any{}},
|
||||
{"missing occurrence name", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "name")}}},
|
||||
{"missing nullable field", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "quantity")}}},
|
||||
{"missing from", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "from")}}},
|
||||
{"missing to", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "to")}}},
|
||||
{"noninteger quantity", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "quantity", 1.5)}}},
|
||||
{"unsupported kind", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "kind", "unsupported")}}},
|
||||
{"opaque item identifier", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "item_id", "item:sha256:opaque")}}},
|
||||
{"unknown occurrence field", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "extra", true)}}},
|
||||
{"segment-named range", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}},
|
||||
{"opaque source identifier", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "source_id": "session-alpha"}})}}},
|
||||
{"unknown reference field", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "extra": true}})}}},
|
||||
{"noninteger range", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 1}})}}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
content, err := json.Marshal(test.value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); err == nil {
|
||||
t.Fatal("private schema accepted invalid structure")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func itemOccurrenceRecord() map[string]any {
|
||||
return map[string]any{
|
||||
"name": "Ring", "kind": "acquired", "quantity": nil, "from": nil, "to": "party", "source_refs": []any{},
|
||||
}
|
||||
}
|
||||
|
||||
func itemOccurrenceWithoutField(value map[string]any, name string) map[string]any {
|
||||
delete(value, name)
|
||||
return value
|
||||
}
|
||||
|
||||
func itemOccurrenceWithField(value map[string]any, name string, fieldValue any) map[string]any {
|
||||
value[name] = fieldValue
|
||||
return value
|
||||
}
|
||||
@@ -18,6 +18,15 @@ func TestOccurrencePromptPairsPreserveRegistryAndTranscriptBoundaries(t *testing
|
||||
family, registry string
|
||||
rules []string
|
||||
}{
|
||||
{"item", `{"items":[{"name":"occurrence-registry-name"}]}`, []string{
|
||||
"Return both `from` and `to` for every occurrence.",
|
||||
"| `discovered` | `null` | `null` |",
|
||||
"| `acquired` | `null` | `party` or the named party member gaining possession |",
|
||||
"| `lost` | `party` or the named party member losing possession | `null` |",
|
||||
"| `consumed` | `party` or the named party member consuming the item | `null` |",
|
||||
"| `transferred` | one named party member | a different named party member |",
|
||||
"payments are always `lost`, not `consumed`.",
|
||||
}},
|
||||
{"npc", `{"npcs":[{"name":"occurrence-registry-name"}]}`, []string{"Use the supplied canonical NPC", "noncombat_presence", "combat_opponent"}},
|
||||
{"location", `{"locations":[{"name":"occurrence-registry-name","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry-context"}]}]}`,
|
||||
[]string{"context supports that coreference", "must not create a registry location", "provenance must never replace current-chunk evidence", "complete ordered"}},
|
||||
|
||||
@@ -902,6 +902,28 @@ Do not relax existing domain invariants merely to make the migration tests pass.
|
||||
|
||||
## Stage 20: Migrate Item Occurrence Extraction
|
||||
|
||||
**Status:** Complete. Item occurrences now use native initial/correction requests
|
||||
through the shared typed producer helper with fixed transcript and names-only
|
||||
registry materials. Added the same-schema/default-profile correction pair and
|
||||
static dependencies; removed private completion, prompt/schema registration,
|
||||
and manifest/fingerprint machinery. Existing private nullable-field mapping,
|
||||
quantity copying, holder rules, canonical identity, source references, and event
|
||||
ordering remain unchanged. Unknown selections provide module-authored guidance
|
||||
before typed assessment, without partial output or invented items. Permissive
|
||||
terminal output is an explicit codec-valid empty list, not a repaired holder or
|
||||
ordinary acceptance. Profile/session/execution selections and incoming reuse
|
||||
ineligibility are preserved. Migrated the holder regression to a native provider
|
||||
fake with all four typed validators, exact defective-response history, contextual
|
||||
guidance, final-turn acceptance, and no duplicate final assessment. Added terminal
|
||||
mode, unknown-item, failed-continuation, structural-failure, cancellation, all-kind,
|
||||
ordering, and durable nullable-holder cases. Native prompt-pair/holder-principle
|
||||
and unchanged structural-schema tests run independently in the assets package.
|
||||
Asset tests/vet/race checks, focused pipeline tests, and `git diff --check` pass.
|
||||
The extractor and all five occurrence-validator package checks remain blocked at
|
||||
setup by shared legacy helpers importing old ConfigKit's removed YAML
|
||||
`plugin/limit`; the new native runtime tests are not claimed passing. Scheduled
|
||||
dependency cleanup and final assembled verification must execute those tests.
|
||||
|
||||
**Goal:** retain the smaller-model holder reliability fixes in the native path.
|
||||
|
||||
**Inspect/change:** `internal/modules/dnd/extract/itemoccurrences`, associated
|
||||
|
||||
@@ -3,8 +3,12 @@ package itemoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
@@ -14,14 +18,14 @@ import (
|
||||
)
|
||||
|
||||
const Key = "dnd/item-occurrences"
|
||||
const PromptID = "notarius/dnd.item_occurrences"
|
||||
const CorrectionPromptID = "notarius/dnd.item_occurrences.repair"
|
||||
|
||||
const (
|
||||
ItemRegistryReferenceSlot = itemregistry.ReferenceSlot
|
||||
ItemRegistryMaxBytes = itemregistry.MaxBytes
|
||||
)
|
||||
|
||||
const mappingPolicy = "dnd.item_occurrences.extract_mapping.v3"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
"source.transcript",
|
||||
@@ -53,21 +57,17 @@ func referenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.ItemOccurrenceList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
itemResolver *itemregistry.Resolver
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
engine *promptkit.Engine
|
||||
itemResolver *itemregistry.Resolver
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
@@ -80,64 +80,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("prepare item registry prompt input: %w", err)
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{llm: llmClient, itemResolver: itemResolver, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||
return &Extractor{engine: engine, itemResolver: itemResolver}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
"mapping_policy": mappingPolicy,
|
||||
"comparison_policy": shared.TextComparisonPolicy,
|
||||
}
|
||||
seeded := e.itemResolver.Seeded()
|
||||
if seeded.Bound() {
|
||||
metadata["item_registry_digest"] = seeded.Digest()
|
||||
metadata["item_count"] = seeded.Count()
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
{Name: "comparison_policy", Value: shared.TextComparisonPolicy},
|
||||
{Name: "item_registry", Value: e.itemResolver.Seeded().ProjectionDigest()},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||
if err != nil {
|
||||
@@ -152,38 +107,50 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
if !registry.Bound() {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("item registry reference is required")
|
||||
}
|
||||
var response extractionResponse
|
||||
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs[ItemRegistryReferenceSlot] = registry.PromptInput()
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction, Inputs: inputs,
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
|
||||
sourceID := req.Source.ID
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
materials[ItemRegistryReferenceSlot] = registry.PromptInput()
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemOccurrenceList, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
value, err := canonicalItemOccurrenceList(response, order, sourceID, registry)
|
||||
if err != nil {
|
||||
return nil, []string{"Select the exact canonical item name from the supplied registry for every occurrence. Omit unsupported or ambiguous selections; do not invent an item. Return the complete replacement occurrence list, keeping current-transcript evidence separate from registry context."}, nil
|
||||
}
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *dnd.ItemOccurrenceList) (dnd.ItemOccurrenceList, error) {
|
||||
return dnd.ItemOccurrenceList{Occurrences: []dnd.ItemOccurrence{}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err)
|
||||
}
|
||||
value, err := canonicalItemOccurrenceList(response, order, req.Source.ID, registry)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("map item occurrence response: %w", err)
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, ModelCandidate: candidate}, nil
|
||||
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.ItemOccurrenceListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.ItemOccurrenceListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,7 +160,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -8,20 +8,14 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
itemidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
itemoccurrenceshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
|
||||
)
|
||||
|
||||
func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
|
||||
id := itemidentity.DeriveID("Torch")
|
||||
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","from":"party","source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`)
|
||||
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","quantity":null,"from":"party","to":null,"source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`)
|
||||
client := &fakeItemOccurrencesLLMClient{content: append([]byte(nil), rawResponse...)}
|
||||
req := extractionRequest()
|
||||
req.References = itemRegistryReferences(t)
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(`{"occurrences":[]}`), "Keep the transcript-grounded item occurrence.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
}
|
||||
req.Correction = correction
|
||||
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -33,28 +27,18 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
|
||||
t.Fatalf("occurrence evidence = %#v, want current-source unit range", refs)
|
||||
}
|
||||
request := client.requests[0]
|
||||
input := request.Inputs[ItemRegistryReferenceSlot]
|
||||
if input.Name != ItemRegistryReferenceSlot || string(input.Content) != `{"items":[{"name":"Torch"}]}` || strings.Contains(string(input.Content), "item:sha256:") {
|
||||
t.Fatalf("registry prompt input = %#v, want names-only projection", input)
|
||||
|
||||
input := renderedInput(t, request, ItemRegistryReferenceSlot)
|
||||
if string(input) != `{"items":[{"name":"Torch"}]}` {
|
||||
t.Fatalf("names-only registry = %s", input)
|
||||
}
|
||||
if request.Correction == nil || string(request.Correction.AssistantResponse) != `{"occurrences":[]}` || request.Correction.UserGuidance != "Keep the transcript-grounded item occurrence." {
|
||||
t.Fatalf("correction = %#v, want exact request correction", request.Correction)
|
||||
}
|
||||
for _, material := range []string{string(request.Correction.AssistantResponse), request.Correction.UserGuidance} {
|
||||
if strings.Contains(material, id) || strings.Contains(material, "item:sha256:") {
|
||||
t.Fatalf("correction material leaked opaque item identity: %q", material)
|
||||
for _, message := range request.Prompt.Messages {
|
||||
if strings.Contains(message.Content, id) || strings.Contains(message.Content, "item:sha256:") {
|
||||
t.Fatal("model context exposed durable item identity")
|
||||
}
|
||||
}
|
||||
correction.AssistantResponse[0] = '['
|
||||
if got := string(request.Correction.AssistantResponse); got != `{"occurrences":[]}` {
|
||||
t.Fatalf("captured correction changed after caller mutation: %q", got)
|
||||
}
|
||||
if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || string(result.ModelCandidate.Response) != string(rawResponse) {
|
||||
t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate)
|
||||
}
|
||||
client.content[0] = '['
|
||||
if got := string(result.ModelCandidate.Response); got != string(rawResponse) {
|
||||
t.Fatalf("model candidate changed after provider buffer mutation: %q", got)
|
||||
if request.Prompt.SessionID != req.SessionID || result.Outcome.Disposition != contracts.ProducerAccepted {
|
||||
t.Fatal("native selection or acceptance lost")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,8 +67,8 @@ func TestExtractRejectsUnknownNameAfterValidOccurrence(t *testing.T) {
|
||||
req := extractionRequest()
|
||||
req.References = itemRegistryReferences(t)
|
||||
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "occurrences[1].name is not in the item registry") {
|
||||
t.Fatalf("Extract() error = %v, want occurrence index and unknown name", err)
|
||||
if err == nil || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("Extract() error = %v, want atomic unknown-name failure", err)
|
||||
}
|
||||
if len(result.Value.Occurrences) != 0 {
|
||||
t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences)
|
||||
@@ -114,12 +98,12 @@ func TestExtractAcceptsOnlyEmptyResponseForEmptyRegistry(t *testing.T) {
|
||||
req.References = references
|
||||
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
||||
if test.wantErr {
|
||||
if err == nil || !strings.Contains(err.Error(), "name is not in the item registry") || len(result.Value.Occurrences) != 0 {
|
||||
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
|
||||
t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
|
||||
if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible {
|
||||
t.Fatalf("Extract() = %#v, %v; want present empty occurrences", result, err)
|
||||
}
|
||||
})
|
||||
@@ -144,50 +128,9 @@ func TestExtractUsesOnlySupportedPromptInputs(t *testing.T) {
|
||||
if _, err := newExtractor(t, client, req.References).Extract(context.Background(), req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := client.requests[0].Inputs["unrelated"]; ok {
|
||||
t.Fatalf("unexpected prompt input: %#v", client.requests[0].Inputs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCarriesRejectedHolderGuidanceIntoCorrectedAttempt(t *testing.T) {
|
||||
references := itemRegistryReferences(t)
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
defectiveResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
|
||||
defective, err := newExtractor(t, &fakeItemOccurrencesLLMClient{content: defectiveResponse}, references).Extract(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validation, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: defective.Value})
|
||||
if err != nil || validation.Approved {
|
||||
t.Fatalf("holder validation = %#v, %v; want rejection", validation, err)
|
||||
}
|
||||
for _, fragment := range []string{"For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
|
||||
if !strings.Contains(validation.CorrectionGuidance, fragment) {
|
||||
t.Fatalf("CorrectionGuidance = %q, want %q", validation.CorrectionGuidance, fragment)
|
||||
for _, message := range client.requests[0].Prompt.Messages {
|
||||
if strings.Contains(message.Content, "ignored") {
|
||||
t.Fatal("unsupported reference became a prompt input")
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{defective.Value.Occurrences[0].ItemID, "item_id", "sha256", itemoccurrenceshape.Key, itemoccurrenceshape.ReasonCode} {
|
||||
if strings.Contains(validation.CorrectionGuidance, forbidden) {
|
||||
t.Fatalf("CorrectionGuidance leaked implementation identifier %q: %q", forbidden, validation.CorrectionGuidance)
|
||||
}
|
||||
}
|
||||
correction, err := contracts.NewSemanticCorrection(defective.ModelCandidate.Response, validation.CorrectionGuidance)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
correctedResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
|
||||
client := &fakeItemOccurrencesLLMClient{content: correctedResponse}
|
||||
req.Correction = correction
|
||||
corrected, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
accepted, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: corrected.Value})
|
||||
if err != nil || !accepted.Approved {
|
||||
t.Fatalf("corrected holder validation = %#v, %v; want approval", accepted, err)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Correction == nil || string(client.requests[0].Correction.AssistantResponse) != string(defectiveResponse) || client.requests[0].Correction.UserGuidance != validation.CorrectionGuidance {
|
||||
t.Fatalf("corrected request = %#v, want byte-faithful defective response and semantic guidance", client.requests)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
package itemoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
|
||||
registryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/registry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
|
||||
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_relatedness"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const rejectedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
|
||||
const correctedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
|
||||
const unknownItem = `{"occurrences":[{"name":"Unknown Item","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||||
|
||||
func bindAssessment(t *testing.T, request *contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (*[]string, *[]string) {
|
||||
t.Helper()
|
||||
registry, err := registryvalidator.New(registryvalidator.Options{}, request.References)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validators := []contracts.TypedValidator[dnd.ItemOccurrenceList]{
|
||||
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
|
||||
relatedness.New(relatedness.Options{}), registry,
|
||||
}
|
||||
var calls, guidance []string
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
var assessment contracts.CandidateAssessment
|
||||
for _, validator := range validators {
|
||||
calls = append(calls, validator.Name())
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{
|
||||
Stage: "extract", Source: request.Source, Chunk: request.Chunk, References: request.References, Value: value,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||||
if result.CorrectionGuidance != "" {
|
||||
guidance = append(guidance, result.CorrectionGuidance)
|
||||
}
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
return &calls, &guidance
|
||||
}
|
||||
|
||||
func TestExtractCarriesRejectedHolderGuidanceIntoNativeCorrection(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.References = itemRegistryReferences(t)
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.ReuseEligible = false
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "item-occurrence-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, run promptkit.PreparedRun) { prepared = append(prepared, run) }
|
||||
calls, guidance := bindAssessment(t, &request)
|
||||
client := &fakeItemOccurrencesLLMClient{responses: []string{rejectedHolder, correctedHolder}}
|
||||
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, registryvalidator.Key}
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Occurrences) != 1 || len(client.requests) != 2 || len(prepared) != 2 || len(*guidance) != 1 {
|
||||
t.Fatalf("correction = %#v %v, validators=%#v guidance=%#v native=%d/%d", result, err, *calls, *guidance, len(client.requests), len(prepared))
|
||||
}
|
||||
occurrence := result.Value.Occurrences[0]
|
||||
if occurrence.ItemID != identity.DeriveID("Torch") || occurrence.Name != "Torch" || occurrence.Kind != dnd.ItemOccurrenceKindAcquired ||
|
||||
occurrence.From != "" || occurrence.To != "party" || occurrence.Quantity != nil || occurrence.SourceRefs[0].SourceID != request.Source.ID {
|
||||
t.Fatalf("holder mapping = %#v", occurrence)
|
||||
}
|
||||
first, next := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(next) != len(first)+2 || !reflect.DeepEqual(first, next[:len(first)]) || next[len(first)].Role != "assistant" || next[len(first)].Content != rejectedHolder {
|
||||
t.Fatal("native continuation did not preserve the exact defective response and fixed context")
|
||||
}
|
||||
correction := next[len(first)+1].Content
|
||||
for _, fragment := range []string{(*guidance)[0], "For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
|
||||
if !strings.Contains(correction, fragment) {
|
||||
t.Fatalf("missing holder guidance %q", fragment)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{occurrence.ItemID, "item_id", "sha256", shape.Key, shape.ReasonCode} {
|
||||
if strings.Contains(correction, forbidden) {
|
||||
t.Fatalf("operator identifier became guidance: %q", forbidden)
|
||||
}
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "item-occurrence-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native profile/session/override lost")
|
||||
}
|
||||
if string(renderedInput(t, call, ItemRegistryReferenceSlot)) != `{"items":[{"name":"Torch"}]}` {
|
||||
t.Fatal("item registry was not fixed and names-only")
|
||||
}
|
||||
}
|
||||
content, err := occurrencecodec.New().Encode(result.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wire struct {
|
||||
Occurrences []map[string]json.RawMessage `json:"occurrences"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &wire); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, present := wire.Occurrences[0]["from"]; present {
|
||||
t.Fatal("private null holder became a durable field")
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemOccurrenceTerminalModesDoNotSynthesizeHolders(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
response string
|
||||
failures []error
|
||||
assessments, runs int
|
||||
wantError bool
|
||||
}{
|
||||
{"strict holder rejection", contracts.ModeStrict, rejectedHolder, nil, 2, 2, true},
|
||||
{"permissive holder rejection", contracts.ModePermissive, rejectedHolder, nil, 2, 2, false},
|
||||
{"strict unknown item", contracts.ModeStrict, unknownItem, nil, 0, 2, true},
|
||||
{"permissive unknown item", contracts.ModePermissive, unknownItem, nil, 0, 2, false},
|
||||
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
|
||||
{"failed correction", contracts.ModePermissive, rejectedHolder, []error{nil, failure}, 1, 2, false},
|
||||
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.References = itemRegistryReferences(t)
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
calls, _ := bindAssessment(t, &request)
|
||||
runs := 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
|
||||
client := &fakeItemOccurrencesLLMClient{content: []byte(test.response), errors: test.failures}
|
||||
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
|
||||
t.Fatalf("terminal = %#v %v, validators=%#v runs=%d", result, err, *calls, runs)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
|
||||
t.Fatal("strict failure returned output")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
|
||||
t.Fatalf("fallback invented an item or holder: %#v", result)
|
||||
}
|
||||
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemOccurrenceCancellationCannotPublishFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
request := extractionRequest()
|
||||
request.References = itemRegistryReferences(t)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
client := &fakeItemOccurrencesLLMClient{content: []byte(correctedHolder), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("canceled delivery = %#v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemOccurrenceKindsHoldersQuantityAndDocumentOrderSurvive(t *testing.T) {
|
||||
quantity := 3
|
||||
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
|
||||
{Name: "Torch", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(5, 5)},
|
||||
{Name: "Torch", Kind: "consumed", From: "Aria", SourceRefs: responseRefs(4, 4)},
|
||||
{Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)},
|
||||
{Name: "Torch", Kind: "acquired", To: "party", Quantity: &quantity, SourceRefs: responseRefs(2, 2)},
|
||||
{Name: "Torch", Kind: "discovered", SourceRefs: responseRefs(1, 1)},
|
||||
}}}
|
||||
request := extractionRequest()
|
||||
request.References = itemRegistryReferences(t)
|
||||
bindAssessment(t, &request)
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Occurrences) != 5 {
|
||||
t.Fatalf("kinds = %#v %v", result, err)
|
||||
}
|
||||
kinds := []dnd.ItemOccurrenceKind{dnd.ItemOccurrenceKindDiscovered, dnd.ItemOccurrenceKindAcquired, dnd.ItemOccurrenceKindLost, dnd.ItemOccurrenceKindConsumed, dnd.ItemOccurrenceKindTransferred}
|
||||
for i, occurrence := range result.Value.Occurrences {
|
||||
if occurrence.Kind != kinds[i] || occurrence.SourceRefs[0].StartUnitID != i+1 {
|
||||
t.Fatalf("ordered occurrence = %#v", occurrence)
|
||||
}
|
||||
}
|
||||
quantity = 9
|
||||
if *result.Value.Occurrences[1].Quantity != 3 || result.Value.Occurrences[4].From != "Aria" || result.Value.Occurrences[4].To != "Borin" {
|
||||
t.Fatal("holder or quantity interpretation changed")
|
||||
}
|
||||
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package itemoccurrences
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.item_occurrences",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
{Name: "item-registry.md", Path: "prompts/item-registry.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/item-occurrences")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope item-occurrence assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare item occurrence prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope item-occurrence schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,82 +0,0 @@
|
||||
package itemoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"item-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-occurrences-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[{"sentinel":"item-occurrence-transcript"}]}`),
|
||||
"players": promptkit.Inline("item-occurrence-player"),
|
||||
"party": promptkit.Inline(" "),
|
||||
"glossary": promptkit.Inline(" "),
|
||||
"item_registry": promptkit.Inline(`{"items":[{"name":"Torch"}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_occurrences_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
rendered := make([]string, len(prepared.Messages))
|
||||
for index, message := range prepared.Messages {
|
||||
rendered[index] = message.Content
|
||||
}
|
||||
content := strings.Join(rendered, "\n")
|
||||
for _, field := range []string{"start_unit_id", "end_unit_id"} {
|
||||
if !strings.Contains(content, field) {
|
||||
t.Fatalf("prepared prompt does not include shared evidence field %q", field)
|
||||
}
|
||||
}
|
||||
for _, obsolete := range []string{"source_id", "start_segment", "end_segment"} {
|
||||
if strings.Contains(content, obsolete) {
|
||||
t.Fatalf("prepared prompt contains obsolete evidence field %q: %s", obsolete, content)
|
||||
}
|
||||
}
|
||||
for _, holderRule := range []string{
|
||||
"| `discovered` | `null` | `null` |",
|
||||
"| `acquired` | `null` | `party` or the named party member gaining possession |",
|
||||
"| `lost` | `party` or the named party member losing possession | `null` |",
|
||||
"| `consumed` | `party` or the named party member consuming the item | `null` |",
|
||||
"| `transferred` | one named party member | a different named party member |",
|
||||
} {
|
||||
if !strings.Contains(content, holderRule) {
|
||||
t.Fatalf("prepared prompt does not include holder rule %q", holderRule)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
|
||||
hash, err := promptAssetMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("promptAssetMetadata() = %q, %v", hash, err)
|
||||
}
|
||||
metadata := newExtractor(t, &fakeItemOccurrencesLLMClient{}).ManifestMetadata()
|
||||
payload, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_item_occurrences_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,8 +36,8 @@ func testSourceRefs() []source.SourceRef {
|
||||
|
||||
func TestModuleSpecDeclaresRequiredRegistry(t *testing.T) {
|
||||
spec := ModuleSpec()
|
||||
if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 {
|
||||
t.Fatalf("correction protocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1)
|
||||
if !reflect.DeepEqual(spec.Prompts, []string{PromptID, CorrectionPromptID}) {
|
||||
t.Fatalf("prompt dependencies = %#v", spec.Prompts)
|
||||
}
|
||||
var slot contracts.ReferenceSlot
|
||||
for _, candidate := range spec.ReferenceSlots {
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package itemoccurrences
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.item_occurrences"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_occurrences_llm")
|
||||
ResponseSchemaID = "notarius.dnd.item_occurrences.llm"
|
||||
ResponseSchemaName = "notarius_dnd_item_occurrences_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_item_occurrences_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
package itemoccurrences
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v", schema)
|
||||
}
|
||||
valid := map[string]any{"occurrences": []any{
|
||||
map[string]any{
|
||||
"name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}},
|
||||
},
|
||||
map[string]any{
|
||||
"name": "Hidden Cache", "kind": "discovered", "quantity": nil, "from": nil, "to": nil,
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
},
|
||||
}}
|
||||
content, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("private schema rejected validator-owned values: %v", err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
value map[string]any
|
||||
}{
|
||||
{"missing occurrences", map[string]any{}},
|
||||
{"missing occurrence name", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "name")}}},
|
||||
{"missing nullable field", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "quantity")}}},
|
||||
{"unsupported kind", map[string]any{"occurrences": []any{withField(responseOccurrence(), "kind", "unsupported")}}},
|
||||
{"opaque item identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "item_id", "item:sha256:opaque")}}},
|
||||
{"unknown occurrence field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "extra", true)}}},
|
||||
{"segment-named range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}},
|
||||
{"opaque source identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "source_id": "session-alpha"}})}}},
|
||||
{"unknown reference field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "extra": true}})}}},
|
||||
{"noninteger range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 1}})}}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
content, err := json.Marshal(test.value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
|
||||
t.Fatal("private schema accepted invalid structure")
|
||||
}
|
||||
})
|
||||
}
|
||||
first := schema.JSONSchema
|
||||
first[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) {
|
||||
t.Fatalf("defensive schema copy = %s, %v", second.JSONSchema, err)
|
||||
}
|
||||
if _, ok := second.DiagnosticsMap()["json_schema"]; ok {
|
||||
t.Fatalf("schema diagnostics leaked content: %#v", second.DiagnosticsMap())
|
||||
}
|
||||
}
|
||||
|
||||
func responseOccurrence() map[string]any {
|
||||
return map[string]any{
|
||||
"name": "Ring", "kind": "acquired", "quantity": nil, "from": nil, "to": "party", "source_refs": []any{},
|
||||
}
|
||||
}
|
||||
|
||||
func withoutField(value map[string]any, name string) map[string]any {
|
||||
delete(value, name)
|
||||
return value
|
||||
}
|
||||
|
||||
func withField(value map[string]any, name string, fieldValue any) map[string]any {
|
||||
value[name] = fieldValue
|
||||
return value
|
||||
}
|
||||
|
||||
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", document); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(instance)
|
||||
}
|
||||
@@ -3,14 +3,19 @@ package itemoccurrences
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
func extractionRequest() contracts.TypedExtractionRequest[dnd.ItemOccurrenceList] {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0",
|
||||
@@ -21,12 +26,15 @@ func extractionRequest() contracts.TypedExtractionRequest {
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
return contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]{
|
||||
Source: doc,
|
||||
Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123",
|
||||
LLMProfile: "profile-item-occurrences",
|
||||
Operation: contracts.ProducerOperation[dnd.ItemOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,51 +55,111 @@ func responseRefs(start, end int) []itemOccurrenceSourceRefResponse {
|
||||
return []itemOccurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: profile-item-occurrences
|
||||
endpoint: https://example.invalid/v1
|
||||
model: occurrence-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := map[string]string{"transcript": "segments", "item_registry": "items"}[name]
|
||||
for _, message := range request.Prompt.Messages {
|
||||
start := strings.Index(message.Content, "{")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
content := []byte(message.Content[start:])
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
||||
return content
|
||||
}
|
||||
}
|
||||
req.Correction = correction
|
||||
return req
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeItemOccurrencesLLMClient struct {
|
||||
response extractionResponse
|
||||
content []byte
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response extractionResponse
|
||||
responses []string
|
||||
content []byte
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeItemOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
func (client *fakeItemOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
return nil, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
content := append([]byte(nil), client.content...)
|
||||
if len(content) == 0 {
|
||||
var err error
|
||||
content, err = json.Marshal(client.response)
|
||||
content := string(client.content)
|
||||
if index < len(client.responses) {
|
||||
content = client.responses[index]
|
||||
}
|
||||
if content == "" {
|
||||
encoded, err := completeResponse(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
content = string(encoded)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func completeResponse(response extractionResponse) ([]byte, error) {
|
||||
content, err := json.Marshal(response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var wire struct {
|
||||
Occurrences []map[string]any `json:"occurrences"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &wire); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, occurrence := range wire.Occurrences {
|
||||
for _, name := range []string{"quantity", "from", "to"} {
|
||||
if _, present := occurrence[name]; !present {
|
||||
occurrence[name] = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := json.Unmarshal(content, target); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return json.Marshal(wire)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user