Migrate item occurrences to native correction while preserving holder semantics

This commit is contained in:
2026-11-22 01:08:02 +00:00
parent 2c1d727232
commit 1fe231b6fb
14 changed files with 519 additions and 486 deletions

View File

@@ -3,8 +3,12 @@ package itemoccurrences
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -14,14 +18,14 @@ import (
)
const Key = "dnd/item-occurrences"
const PromptID = "notarius/dnd.item_occurrences"
const CorrectionPromptID = "notarius/dnd.item_occurrences.repair"
const (
ItemRegistryReferenceSlot = itemregistry.ReferenceSlot
ItemRegistryMaxBytes = itemregistry.MaxBytes
)
const mappingPolicy = "dnd.item_occurrences.extract_mapping.v3"
var requiredCapabilities = []string{
"chunks",
"source.transcript",
@@ -53,21 +57,17 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.ItemOccurrenceList] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
itemResolver *itemregistry.Resolver
promptSHA string
responseSchemaSHA string
engine *promptkit.Engine
itemResolver *itemregistry.Resolver
}
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
@@ -80,64 +80,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare item registry prompt input: %w", err)
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{llm: llmClient, itemResolver: itemResolver, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
return &Extractor{engine: engine, itemResolver: itemResolver}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
"mapping_policy": mappingPolicy,
"comparison_policy": shared.TextComparisonPolicy,
}
seeded := e.itemResolver.Seeded()
if seeded.Bound() {
metadata["item_registry_digest"] = seeded.Digest()
metadata["item_count"] = seeded.Count()
}
return metadata
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "comparison_policy", Value: shared.TextComparisonPolicy},
{Name: "item_registry", Value: e.itemResolver.Seeded().ProjectionDigest()},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("native engine must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
@@ -152,38 +107,50 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
if !registry.Bound() {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("item registry reference is required")
}
var response extractionResponse
inputs := shared.PromptInputs(sourceInput, req.References)
inputs[ItemRegistryReferenceSlot] = registry.PromptInput()
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction, Inputs: inputs,
}, &response)
if err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
sourceID := req.Source.ID
materials := shared.PromptInputs(sourceInput, req.References)
materials[ItemRegistryReferenceSlot] = registry.PromptInput()
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemOccurrenceList, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
value, err := canonicalItemOccurrenceList(response, order, sourceID, registry)
if err != nil {
return nil, []string{"Select the exact canonical item name from the supplied registry for every occurrence. Omit unsupported or ambiguous selections; do not invent an item. Return the complete replacement occurrence list, keeping current-transcript evidence separate from registry context."}, nil
}
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(context.Context, *dnd.ItemOccurrenceList) (dnd.ItemOccurrenceList, error) {
return dnd.ItemOccurrenceList{Occurrences: []dnd.ItemOccurrence{}}, nil
})
if err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err)
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err)
}
value, err := canonicalItemOccurrenceList(response, order, req.Source.ID, registry)
if err != nil {
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("map item occurrence response: %w", err)
}
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, ModelCandidate: candidate}, nil
return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, Outcome: outcome}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.ItemOccurrenceListKind,
ReferenceSlots: referenceSlots(),
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.ItemOccurrenceListKind,
ReferenceSlots: referenceSlots(),
}
}
@@ -193,7 +160,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -8,20 +8,14 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
itemidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
itemoccurrenceshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
)
func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
id := itemidentity.DeriveID("Torch")
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","from":"party","source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`)
rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","quantity":null,"from":"party","to":null,"source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`)
client := &fakeItemOccurrencesLLMClient{content: append([]byte(nil), rawResponse...)}
req := extractionRequest()
req.References = itemRegistryReferences(t)
correction, err := contracts.NewSemanticCorrection([]byte(`{"occurrences":[]}`), "Keep the transcript-grounded item occurrence.")
if err != nil {
t.Fatalf("NewSemanticCorrection() error = %v", err)
}
req.Correction = correction
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
if err != nil {
t.Fatal(err)
@@ -33,28 +27,18 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) {
t.Fatalf("occurrence evidence = %#v, want current-source unit range", refs)
}
request := client.requests[0]
input := request.Inputs[ItemRegistryReferenceSlot]
if input.Name != ItemRegistryReferenceSlot || string(input.Content) != `{"items":[{"name":"Torch"}]}` || strings.Contains(string(input.Content), "item:sha256:") {
t.Fatalf("registry prompt input = %#v, want names-only projection", input)
input := renderedInput(t, request, ItemRegistryReferenceSlot)
if string(input) != `{"items":[{"name":"Torch"}]}` {
t.Fatalf("names-only registry = %s", input)
}
if request.Correction == nil || string(request.Correction.AssistantResponse) != `{"occurrences":[]}` || request.Correction.UserGuidance != "Keep the transcript-grounded item occurrence." {
t.Fatalf("correction = %#v, want exact request correction", request.Correction)
}
for _, material := range []string{string(request.Correction.AssistantResponse), request.Correction.UserGuidance} {
if strings.Contains(material, id) || strings.Contains(material, "item:sha256:") {
t.Fatalf("correction material leaked opaque item identity: %q", material)
for _, message := range request.Prompt.Messages {
if strings.Contains(message.Content, id) || strings.Contains(message.Content, "item:sha256:") {
t.Fatal("model context exposed durable item identity")
}
}
correction.AssistantResponse[0] = '['
if got := string(request.Correction.AssistantResponse); got != `{"occurrences":[]}` {
t.Fatalf("captured correction changed after caller mutation: %q", got)
}
if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || string(result.ModelCandidate.Response) != string(rawResponse) {
t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate)
}
client.content[0] = '['
if got := string(result.ModelCandidate.Response); got != string(rawResponse) {
t.Fatalf("model candidate changed after provider buffer mutation: %q", got)
if request.Prompt.SessionID != req.SessionID || result.Outcome.Disposition != contracts.ProducerAccepted {
t.Fatal("native selection or acceptance lost")
}
}
@@ -83,8 +67,8 @@ func TestExtractRejectsUnknownNameAfterValidOccurrence(t *testing.T) {
req := extractionRequest()
req.References = itemRegistryReferences(t)
result, err := newExtractor(t, client, req.References).Extract(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "occurrences[1].name is not in the item registry") {
t.Fatalf("Extract() error = %v, want occurrence index and unknown name", err)
if err == nil || result.Outcome.Disposition != "" {
t.Fatalf("Extract() error = %v, want atomic unknown-name failure", err)
}
if len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences)
@@ -114,12 +98,12 @@ func TestExtractAcceptsOnlyEmptyResponseForEmptyRegistry(t *testing.T) {
req.References = references
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
if test.wantErr {
if err == nil || !strings.Contains(err.Error(), "name is not in the item registry") || len(result.Value.Occurrences) != 0 {
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err)
}
return
}
if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible {
t.Fatalf("Extract() = %#v, %v; want present empty occurrences", result, err)
}
})
@@ -144,50 +128,9 @@ func TestExtractUsesOnlySupportedPromptInputs(t *testing.T) {
if _, err := newExtractor(t, client, req.References).Extract(context.Background(), req); err != nil {
t.Fatal(err)
}
if _, ok := client.requests[0].Inputs["unrelated"]; ok {
t.Fatalf("unexpected prompt input: %#v", client.requests[0].Inputs)
}
}
func TestExtractCarriesRejectedHolderGuidanceIntoCorrectedAttempt(t *testing.T) {
references := itemRegistryReferences(t)
req := extractionRequest()
req.References = references
defectiveResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
defective, err := newExtractor(t, &fakeItemOccurrencesLLMClient{content: defectiveResponse}, references).Extract(context.Background(), req)
if err != nil {
t.Fatal(err)
}
validation, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: defective.Value})
if err != nil || validation.Approved {
t.Fatalf("holder validation = %#v, %v; want rejection", validation, err)
}
for _, fragment := range []string{"For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
if !strings.Contains(validation.CorrectionGuidance, fragment) {
t.Fatalf("CorrectionGuidance = %q, want %q", validation.CorrectionGuidance, fragment)
for _, message := range client.requests[0].Prompt.Messages {
if strings.Contains(message.Content, "ignored") {
t.Fatal("unsupported reference became a prompt input")
}
}
for _, forbidden := range []string{defective.Value.Occurrences[0].ItemID, "item_id", "sha256", itemoccurrenceshape.Key, itemoccurrenceshape.ReasonCode} {
if strings.Contains(validation.CorrectionGuidance, forbidden) {
t.Fatalf("CorrectionGuidance leaked implementation identifier %q: %q", forbidden, validation.CorrectionGuidance)
}
}
correction, err := contracts.NewSemanticCorrection(defective.ModelCandidate.Response, validation.CorrectionGuidance)
if err != nil {
t.Fatal(err)
}
correctedResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`)
client := &fakeItemOccurrencesLLMClient{content: correctedResponse}
req.Correction = correction
corrected, err := newExtractor(t, client, references).Extract(context.Background(), req)
if err != nil {
t.Fatal(err)
}
accepted, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: corrected.Value})
if err != nil || !accepted.Approved {
t.Fatalf("corrected holder validation = %#v, %v; want approval", accepted, err)
}
if len(client.requests) != 1 || client.requests[0].Correction == nil || string(client.requests[0].Correction.AssistantResponse) != string(defectiveResponse) || client.requests[0].Correction.UserGuidance != validation.CorrectionGuidance {
t.Fatalf("corrected request = %#v, want byte-faithful defective response and semantic guidance", client.requests)
}
}

View File

@@ -0,0 +1,213 @@
package itemoccurrences
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
registryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/registry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const rejectedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
const correctedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`
const unknownItem = `{"occurrences":[{"name":"Unknown Item","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
func bindAssessment(t *testing.T, request *contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (*[]string, *[]string) {
t.Helper()
registry, err := registryvalidator.New(registryvalidator.Options{}, request.References)
if err != nil {
t.Fatal(err)
}
validators := []contracts.TypedValidator[dnd.ItemOccurrenceList]{
shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}),
relatedness.New(relatedness.Options{}), registry,
}
var calls, guidance []string
request.Operation.Assess = func(ctx context.Context, value dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
var assessment contracts.CandidateAssessment
for _, validator := range validators {
calls = append(calls, validator.Name())
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{
Stage: "extract", Source: request.Source, Chunk: request.Chunk, References: request.References, Value: value,
})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
if result.CorrectionGuidance != "" {
guidance = append(guidance, result.CorrectionGuidance)
}
}
return assessment, nil
}
return &calls, &guidance
}
func TestExtractCarriesRejectedHolderGuidanceIntoNativeCorrection(t *testing.T) {
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "item-occurrence-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, run promptkit.PreparedRun) { prepared = append(prepared, run) }
calls, guidance := bindAssessment(t, &request)
client := &fakeItemOccurrencesLLMClient{responses: []string{rejectedHolder, correctedHolder}}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, registryvalidator.Key}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Occurrences) != 1 || len(client.requests) != 2 || len(prepared) != 2 || len(*guidance) != 1 {
t.Fatalf("correction = %#v %v, validators=%#v guidance=%#v native=%d/%d", result, err, *calls, *guidance, len(client.requests), len(prepared))
}
occurrence := result.Value.Occurrences[0]
if occurrence.ItemID != identity.DeriveID("Torch") || occurrence.Name != "Torch" || occurrence.Kind != dnd.ItemOccurrenceKindAcquired ||
occurrence.From != "" || occurrence.To != "party" || occurrence.Quantity != nil || occurrence.SourceRefs[0].SourceID != request.Source.ID {
t.Fatalf("holder mapping = %#v", occurrence)
}
first, next := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(next) != len(first)+2 || !reflect.DeepEqual(first, next[:len(first)]) || next[len(first)].Role != "assistant" || next[len(first)].Content != rejectedHolder {
t.Fatal("native continuation did not preserve the exact defective response and fixed context")
}
correction := next[len(first)+1].Content
for _, fragment := range []string{(*guidance)[0], "For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} {
if !strings.Contains(correction, fragment) {
t.Fatalf("missing holder guidance %q", fragment)
}
}
for _, forbidden := range []string{occurrence.ItemID, "item_id", "sha256", shape.Key, shape.ReasonCode} {
if strings.Contains(correction, forbidden) {
t.Fatalf("operator identifier became guidance: %q", forbidden)
}
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "item-occurrence-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native profile/session/override lost")
}
if string(renderedInput(t, call, ItemRegistryReferenceSlot)) != `{"items":[{"name":"Torch"}]}` {
t.Fatal("item registry was not fixed and names-only")
}
}
content, err := occurrencecodec.New().Encode(result.Value)
if err != nil {
t.Fatal(err)
}
var wire struct {
Occurrences []map[string]json.RawMessage `json:"occurrences"`
}
if err := json.Unmarshal(content, &wire); err != nil {
t.Fatal(err)
}
if _, present := wire.Occurrences[0]["from"]; present {
t.Fatal("private null holder became a durable field")
}
}
func TestItemOccurrenceTerminalModesDoNotSynthesizeHolders(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
failures []error
assessments, runs int
wantError bool
}{
{"strict holder rejection", contracts.ModeStrict, rejectedHolder, nil, 2, 2, true},
{"permissive holder rejection", contracts.ModePermissive, rejectedHolder, nil, 2, 2, false},
{"strict unknown item", contracts.ModeStrict, unknownItem, nil, 0, 2, true},
{"permissive unknown item", contracts.ModePermissive, unknownItem, nil, 0, 2, false},
{"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed correction", contracts.ModePermissive, rejectedHolder, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
calls, _ := bindAssessment(t, &request)
runs := 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ }
client := &fakeItemOccurrencesLLMClient{content: []byte(test.response), errors: test.failures}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs {
t.Fatalf("terminal = %#v %v, validators=%#v runs=%d", result, err, *calls, runs)
}
if test.wantError {
if result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
t.Fatal("strict failure returned output")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
t.Fatalf("fallback invented an item or holder: %#v", result)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
})
}
}
func TestItemOccurrenceCancellationCannotPublishFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request := extractionRequest()
request.References = itemRegistryReferences(t)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeItemOccurrencesLLMClient{content: []byte(correctedHolder), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("canceled delivery = %#v %v", result, err)
}
}
func TestItemOccurrenceKindsHoldersQuantityAndDocumentOrderSurvive(t *testing.T) {
quantity := 3
client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{
{Name: "Torch", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(5, 5)},
{Name: "Torch", Kind: "consumed", From: "Aria", SourceRefs: responseRefs(4, 4)},
{Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)},
{Name: "Torch", Kind: "acquired", To: "party", Quantity: &quantity, SourceRefs: responseRefs(2, 2)},
{Name: "Torch", Kind: "discovered", SourceRefs: responseRefs(1, 1)},
}}}
request := extractionRequest()
request.References = itemRegistryReferences(t)
bindAssessment(t, &request)
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Occurrences) != 5 {
t.Fatalf("kinds = %#v %v", result, err)
}
kinds := []dnd.ItemOccurrenceKind{dnd.ItemOccurrenceKindDiscovered, dnd.ItemOccurrenceKindAcquired, dnd.ItemOccurrenceKindLost, dnd.ItemOccurrenceKindConsumed, dnd.ItemOccurrenceKindTransferred}
for i, occurrence := range result.Value.Occurrences {
if occurrence.Kind != kinds[i] || occurrence.SourceRefs[0].StartUnitID != i+1 {
t.Fatalf("ordered occurrence = %#v", occurrence)
}
}
quantity = 9
if *result.Value.Occurrences[1].Quantity != 3 || result.Value.Occurrences[4].From != "Aria" || result.Value.Occurrences[4].To != "Borin" {
t.Fatal("holder or quantity interpretation changed")
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatal(err)
}
}

View File

@@ -1,75 +0,0 @@
package itemoccurrences
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: "dnd.item_occurrences",
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
{Name: "item-registry.md", Path: "prompts/item-registry.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-references.md",
"common-dnd-transcript-chunk.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/item-occurrences")
if err != nil {
return nil, fmt.Errorf("scope item-occurrence assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare item occurrence prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope item-occurrence schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,82 +0,0 @@
package itemoccurrences
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"item-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-occurrences-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[{"sentinel":"item-occurrence-transcript"}]}`),
"players": promptkit.Inline("item-occurrence-player"),
"party": promptkit.Inline(" "),
"glossary": promptkit.Inline(" "),
"item_registry": promptkit.Inline(`{"items":[{"name":"Torch"}]}`),
},
})
if err != nil {
t.Fatal(err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_occurrences_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
rendered := make([]string, len(prepared.Messages))
for index, message := range prepared.Messages {
rendered[index] = message.Content
}
content := strings.Join(rendered, "\n")
for _, field := range []string{"start_unit_id", "end_unit_id"} {
if !strings.Contains(content, field) {
t.Fatalf("prepared prompt does not include shared evidence field %q", field)
}
}
for _, obsolete := range []string{"source_id", "start_segment", "end_segment"} {
if strings.Contains(content, obsolete) {
t.Fatalf("prepared prompt contains obsolete evidence field %q: %s", obsolete, content)
}
}
for _, holderRule := range []string{
"| `discovered` | `null` | `null` |",
"| `acquired` | `null` | `party` or the named party member gaining possession |",
"| `lost` | `party` or the named party member losing possession | `null` |",
"| `consumed` | `party` or the named party member consuming the item | `null` |",
"| `transferred` | one named party member | a different named party member |",
} {
if !strings.Contains(content, holderRule) {
t.Fatalf("prepared prompt does not include holder rule %q", holderRule)
}
}
}
func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) {
hash, err := promptAssetMetadata()
if err != nil || !strings.HasPrefix(hash, "sha256:") {
t.Fatalf("promptAssetMetadata() = %q, %v", hash, err)
}
metadata := newExtractor(t, &fakeItemOccurrencesLLMClient{}).ManifestMetadata()
payload, err := json.Marshal(metadata)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"common-dnd-system", "dnd_item_occurrences_llm.v1.json"} {
if strings.Contains(string(payload), forbidden) {
t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload)
}
}
}

View File

@@ -36,8 +36,8 @@ func testSourceRefs() []source.SourceRef {
func TestModuleSpecDeclaresRequiredRegistry(t *testing.T) {
spec := ModuleSpec()
if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 {
t.Fatalf("correction protocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1)
if !reflect.DeepEqual(spec.Prompts, []string{PromptID, CorrectionPromptID}) {
t.Fatalf("prompt dependencies = %#v", spec.Prompts)
}
var slot contracts.ReferenceSlot
for _, candidate := range spec.ReferenceSlots {

View File

@@ -1,25 +0,0 @@
package itemoccurrences
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.item_occurrences"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_occurrences_llm")
ResponseSchemaID = "notarius.dnd.item_occurrences.llm"
ResponseSchemaName = "notarius_dnd_item_occurrences_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_item_occurrences_llm.v1.json",
})
}

View File

@@ -1,107 +0,0 @@
package itemoccurrences
import (
"bytes"
"encoding/json"
"strings"
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
)
func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v", schema)
}
valid := map[string]any{"occurrences": []any{
map[string]any{
"name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party",
"source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}},
},
map[string]any{
"name": "Hidden Cache", "kind": "discovered", "quantity": nil, "from": nil, "to": nil,
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}},
},
}}
content, err := json.Marshal(valid)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("private schema rejected validator-owned values: %v", err)
}
for _, test := range []struct {
name string
value map[string]any
}{
{"missing occurrences", map[string]any{}},
{"missing occurrence name", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "name")}}},
{"missing nullable field", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "quantity")}}},
{"unsupported kind", map[string]any{"occurrences": []any{withField(responseOccurrence(), "kind", "unsupported")}}},
{"opaque item identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "item_id", "item:sha256:opaque")}}},
{"unknown occurrence field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "extra", true)}}},
{"segment-named range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}},
{"opaque source identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "source_id": "session-alpha"}})}}},
{"unknown reference field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "extra": true}})}}},
{"noninteger range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 1}})}}},
} {
t.Run(test.name, func(t *testing.T) {
content, err := json.Marshal(test.value)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
t.Fatal("private schema accepted invalid structure")
}
})
}
first := schema.JSONSchema
first[0] = '['
second, err := loadResponseSchema()
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) {
t.Fatalf("defensive schema copy = %s, %v", second.JSONSchema, err)
}
if _, ok := second.DiagnosticsMap()["json_schema"]; ok {
t.Fatalf("schema diagnostics leaked content: %#v", second.DiagnosticsMap())
}
}
func responseOccurrence() map[string]any {
return map[string]any{
"name": "Ring", "kind": "acquired", "quantity": nil, "from": nil, "to": "party", "source_refs": []any{},
}
}
func withoutField(value map[string]any, name string) map[string]any {
delete(value, name)
return value
}
func withField(value map[string]any, name string, fieldValue any) map[string]any {
value[name] = fieldValue
return value
}
func validateJSONSchema(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return err
}
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return err
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", document); err != nil {
return err
}
compiled, err := compiler.Compile("schema.json")
if err != nil {
return err
}
return compiled.Validate(instance)
}

View File

@@ -3,14 +3,19 @@ package itemoccurrences
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func extractionRequest() contracts.TypedExtractionRequest {
func extractionRequest() contracts.TypedExtractionRequest[dnd.ItemOccurrenceList] {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-alpha:chunk:0",
@@ -21,12 +26,15 @@ func extractionRequest() contracts.TypedExtractionRequest {
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest{
return contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]{
Source: doc,
Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
SessionID: "session-123",
LLMProfile: "profile-item-occurrences",
Operation: contracts.ProducerOperation[dnd.ItemOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}},
}
}
@@ -47,51 +55,111 @@ func responseRefs(start, end int) []itemOccurrenceSourceRefResponse {
return []itemOccurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
profiles:
- id: profile-item-occurrences
endpoint: https://example.invalid/v1
model: occurrence-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatalf("New() error = %v", err)
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
extractor, err := New(engine, Options{}, references...)
if err != nil {
t.Fatal(err)
}
return extractor
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
t.Helper()
key := map[string]string{"transcript": "segments", "item_registry": "items"}[name]
for _, message := range request.Prompt.Messages {
start := strings.Index(message.Content, "{")
if start < 0 {
continue
}
content := []byte(message.Content[start:])
var fields map[string]json.RawMessage
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
return content
}
}
req.Correction = correction
return req
t.Fatalf("missing rendered %s input", name)
return nil
}
type fakeItemOccurrencesLLMClient struct {
response extractionResponse
content []byte
err error
requests []contracts.StructuredCompletionRequest
response extractionResponse
responses []string
content []byte
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeItemOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
func (client *fakeItemOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
return nil, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
content := append([]byte(nil), client.content...)
if len(content) == 0 {
var err error
content, err = json.Marshal(client.response)
content := string(client.content)
if index < len(client.responses) {
content = client.responses[index]
}
if content == "" {
encoded, err := completeResponse(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
content = string(encoded)
}
return &promptkit.GenerateResponse{Content: content}, nil
}
func completeResponse(response extractionResponse) ([]byte, error) {
content, err := json.Marshal(response)
if err != nil {
return nil, err
}
var wire struct {
Occurrences []map[string]any `json:"occurrences"`
}
if err := json.Unmarshal(content, &wire); err != nil {
return nil, err
}
for _, occurrence := range wire.Occurrences {
for _, name := range []string{"quantity", "from", "to"} {
if _, present := occurrence[name]; !present {
occurrence[name] = nil
}
}
}
if err := json.Unmarshal(content, target); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
return json.Marshal(wire)
}