diff --git a/assets/dnd/item-occurrences/prompts/correction.md b/assets/dnd/item-occurrences/prompts/correction.md new file mode 100644 index 00000000..076a0d4a --- /dev/null +++ b/assets/dnd/item-occurrences/prompts/correction.md @@ -0,0 +1,7 @@ +Revise the previous item occurrence list using these findings: + +{{ input "guidance" }} + +Return the complete replacement list using the original transcript and registry +context. Preserve stated quantities and possession changes without inventing an +item or holder. Return only JSON matching the original response contract. diff --git a/assets/dnd/item-occurrences/prompts/prompt.yaml b/assets/dnd/item-occurrences/prompts/prompt.yaml index 93a13b5d..6d0529f1 100644 --- a/assets/dnd/item-occurrences/prompts/prompt.yaml +++ b/assets/dnd/item-occurrences/prompts/prompt.yaml @@ -20,7 +20,7 @@ prompts: messages: - role: system content_file: ../../shared/prompts/common-dnd-system.md - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-identity.md - role: user content_file: ../../shared/prompts/common-dnd-references.md @@ -30,11 +30,11 @@ prompts: content_file: ../../shared/prompts/common-dnd-transcript-chunk.md cache_control: type: ephemeral - - role: user + - role: system content_file: ../../shared/prompts/common-dnd-extraction-evidence.md - role: user content_file: ./item-registry.md - - role: user + - role: system content_file: ./instructions.md cache_control: type: ephemeral @@ -42,3 +42,17 @@ prompts: format: json validation_mode: json_schema schema_path: dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json + + - id: dnd.item_occurrences.repair + default_profile: notarius/dnd-extraction + inputs: + - name: guidance + required: true + content_type: text/plain + messages: + - role: user + content_file: ./correction.md + output: + format: json + validation_mode: json_schema + schema_path: dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json diff --git a/assets/item_occurrence_test.go b/assets/item_occurrence_test.go new file mode 100644 index 00000000..64701051 --- /dev/null +++ b/assets/item_occurrence_test.go @@ -0,0 +1,79 @@ +package assets_test + +import ( + "encoding/json" + "io/fs" + "testing" + + "gitea.maximumdirect.net/eric/notarius/assets" +) + +func TestItemOccurrenceSchemaIsStrictlyStructuralAndPrivate(t *testing.T) { + schema, err := fs.ReadFile(assets.FS(), "dnd/item-occurrences/schemas/dnd_item_occurrences_llm.v1.json") + if err != nil { + t.Fatal(err) + } + + valid := map[string]any{"occurrences": []any{ + map[string]any{ + "name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party", + "source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}}, + }, + map[string]any{ + "name": "Hidden Cache", "kind": "discovered", "quantity": nil, "from": nil, "to": nil, + "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}}, + }, + }} + content, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err != nil { + t.Fatalf("private schema rejected validator-owned values: %v", err) + } + for _, test := range []struct { + name string + value map[string]any + }{ + {"missing occurrences", map[string]any{}}, + {"missing occurrence name", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "name")}}}, + {"missing nullable field", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "quantity")}}}, + {"missing from", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "from")}}}, + {"missing to", map[string]any{"occurrences": []any{itemOccurrenceWithoutField(itemOccurrenceRecord(), "to")}}}, + {"noninteger quantity", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "quantity", 1.5)}}}, + {"unsupported kind", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "kind", "unsupported")}}}, + {"opaque item identifier", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "item_id", "item:sha256:opaque")}}}, + {"unknown occurrence field", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "extra", true)}}}, + {"segment-named range", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}}, + {"opaque source identifier", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "source_id": "session-alpha"}})}}}, + {"unknown reference field", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "extra": true}})}}}, + {"noninteger range", map[string]any{"occurrences": []any{itemOccurrenceWithField(itemOccurrenceRecord(), "source_refs", []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 1}})}}}, + } { + t.Run(test.name, func(t *testing.T) { + content, err := json.Marshal(test.value) + if err != nil { + t.Fatal(err) + } + if err := validateJSONSchema(content, schema); err == nil { + t.Fatal("private schema accepted invalid structure") + } + }) + } + +} + +func itemOccurrenceRecord() map[string]any { + return map[string]any{ + "name": "Ring", "kind": "acquired", "quantity": nil, "from": nil, "to": "party", "source_refs": []any{}, + } +} + +func itemOccurrenceWithoutField(value map[string]any, name string) map[string]any { + delete(value, name) + return value +} + +func itemOccurrenceWithField(value map[string]any, name string, fieldValue any) map[string]any { + value[name] = fieldValue + return value +} diff --git a/assets/occurrence_extraction_test.go b/assets/occurrence_extraction_test.go index e22b511c..fff16c65 100644 --- a/assets/occurrence_extraction_test.go +++ b/assets/occurrence_extraction_test.go @@ -18,6 +18,15 @@ func TestOccurrencePromptPairsPreserveRegistryAndTranscriptBoundaries(t *testing family, registry string rules []string }{ + {"item", `{"items":[{"name":"occurrence-registry-name"}]}`, []string{ + "Return both `from` and `to` for every occurrence.", + "| `discovered` | `null` | `null` |", + "| `acquired` | `null` | `party` or the named party member gaining possession |", + "| `lost` | `party` or the named party member losing possession | `null` |", + "| `consumed` | `party` or the named party member consuming the item | `null` |", + "| `transferred` | one named party member | a different named party member |", + "payments are always `lost`, not `consumed`.", + }}, {"npc", `{"npcs":[{"name":"occurrence-registry-name"}]}`, []string{"Use the supplied canonical NPC", "noncombat_presence", "combat_opponent"}}, {"location", `{"locations":[{"name":"occurrence-registry-name","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry-context"}]}]}`, []string{"context supports that coreference", "must not create a registry location", "provenance must never replace current-chunk evidence", "complete ordered"}}, diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 41c7c3af..6bbff124 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -902,6 +902,28 @@ Do not relax existing domain invariants merely to make the migration tests pass. ## Stage 20: Migrate Item Occurrence Extraction +**Status:** Complete. Item occurrences now use native initial/correction requests +through the shared typed producer helper with fixed transcript and names-only +registry materials. Added the same-schema/default-profile correction pair and +static dependencies; removed private completion, prompt/schema registration, +and manifest/fingerprint machinery. Existing private nullable-field mapping, +quantity copying, holder rules, canonical identity, source references, and event +ordering remain unchanged. Unknown selections provide module-authored guidance +before typed assessment, without partial output or invented items. Permissive +terminal output is an explicit codec-valid empty list, not a repaired holder or +ordinary acceptance. Profile/session/execution selections and incoming reuse +ineligibility are preserved. Migrated the holder regression to a native provider +fake with all four typed validators, exact defective-response history, contextual +guidance, final-turn acceptance, and no duplicate final assessment. Added terminal +mode, unknown-item, failed-continuation, structural-failure, cancellation, all-kind, +ordering, and durable nullable-holder cases. Native prompt-pair/holder-principle +and unchanged structural-schema tests run independently in the assets package. +Asset tests/vet/race checks, focused pipeline tests, and `git diff --check` pass. +The extractor and all five occurrence-validator package checks remain blocked at +setup by shared legacy helpers importing old ConfigKit's removed YAML +`plugin/limit`; the new native runtime tests are not claimed passing. Scheduled +dependency cleanup and final assembled verification must execute those tests. + **Goal:** retain the smaller-model holder reliability fixes in the native path. **Inspect/change:** `internal/modules/dnd/extract/itemoccurrences`, associated diff --git a/internal/modules/dnd/extract/itemoccurrences/extractor.go b/internal/modules/dnd/extract/itemoccurrences/extractor.go index f353efd0..7fc4305d 100644 --- a/internal/modules/dnd/extract/itemoccurrences/extractor.go +++ b/internal/modules/dnd/extract/itemoccurrences/extractor.go @@ -3,8 +3,12 @@ package itemoccurrences import ( "context" + "encoding/json" "fmt" "sort" + "strings" + + "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" @@ -14,14 +18,14 @@ import ( ) const Key = "dnd/item-occurrences" +const PromptID = "notarius/dnd.item_occurrences" +const CorrectionPromptID = "notarius/dnd.item_occurrences.repair" const ( ItemRegistryReferenceSlot = itemregistry.ReferenceSlot ItemRegistryMaxBytes = itemregistry.MaxBytes ) -const mappingPolicy = "dnd.item_occurrences.extract_mapping.v3" - var requiredCapabilities = []string{ "chunks", "source.transcript", @@ -53,21 +57,17 @@ func referenceSlots() []contracts.ReferenceSlot { } var _ contracts.Extractor[dnd.ItemOccurrenceList] = (*Extractor)(nil) -var _ contracts.ManifestMetadataProvider = (*Extractor)(nil) -var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil) type Options struct{} type Extractor struct { - llm contracts.StructuredLLMClient - itemResolver *itemregistry.Resolver - promptSHA string - responseSchemaSHA string + engine *promptkit.Engine + itemResolver *itemregistry.Resolver } -func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { - if llmClient == nil { - return nil, extractorErrorf("LLM client must not be nil") +func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) { + if engine == nil { + return nil, extractorErrorf("native engine must not be nil") } if len(references) > 1 { return nil, extractorErrorf("at most one reference set may be supplied") @@ -80,64 +80,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr if err != nil { return nil, extractorErrorf("prepare item registry prompt input: %w", err) } - promptSHA, err := promptAssetMetadata() - if err != nil { - return nil, extractorErrorf("load prompt metadata: %w", err) - } - responseSchema, err := loadResponseSchema() - if err != nil { - return nil, extractorErrorf("load response schema: %w", err) - } - return &Extractor{llm: llmClient, itemResolver: itemResolver, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil + return &Extractor{engine: engine, itemResolver: itemResolver}, nil } func (e *Extractor) Key() string { return Key } func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() } -func (e *Extractor) ManifestMetadata() map[string]any { - if e == nil { - return nil - } - metadata := map[string]any{ - "prompt_id": PromptID, - "prompt_version": SchemaVersion, - "prompt_sha256": e.promptSHA, - "response_schema_key": string(ResponseSchemaKey), - "response_schema_id": ResponseSchemaID, - "response_schema_name": ResponseSchemaName, - "response_schema_version": SchemaVersion, - "response_schema_sha256": e.responseSchemaSHA, - "mapping_policy": mappingPolicy, - "comparison_policy": shared.TextComparisonPolicy, - } - seeded := e.itemResolver.Seeded() - if seeded.Bound() { - metadata["item_registry_digest"] = seeded.Digest() - metadata["item_count"] = seeded.Count() - } - return metadata -} - -func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint { - if e == nil { - return nil - } - return []pipeline.CheckpointFingerprint{ - {Name: "prompt", Value: e.promptSHA}, - {Name: "response_schema", Value: e.responseSchemaSHA}, - {Name: "mapping_policy", Value: mappingPolicy}, - {Name: "comparison_policy", Value: shared.TextComparisonPolicy}, - {Name: "item_registry", Value: e.itemResolver.Seeded().ProjectionDigest()}, - } -} - -func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) { +func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (contracts.TypedExtractionResult[dnd.ItemOccurrenceList], error) { if e == nil { return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("extractor must not be nil") } - if e.llm == nil { - return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("LLM client must not be nil") + if e.engine == nil { + return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("native engine must not be nil") } sourceInput, err := shared.PrepareChunkExtraction(ctx, req) if err != nil { @@ -152,38 +107,50 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe if !registry.Bound() { return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("item registry reference is required") } - var response extractionResponse - inputs := shared.PromptInputs(sourceInput, req.References) - inputs[ItemRegistryReferenceSlot] = registry.PromptInput() - completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ - StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion, - ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, - Correction: req.Correction, Inputs: inputs, - }, &response) - if err != nil { - return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("complete structured output: %w", err) + + sourceID := req.Source.ID + materials := shared.PromptInputs(sourceInput, req.References) + materials[ItemRegistryReferenceSlot] = registry.PromptInput() + inputs := make(map[string]promptkit.ArtifactRef, len(materials)) + for name, material := range materials { + inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content)) } - candidate, err := shared.ModelCandidateFromResponse(completion) + initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs} + value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial, + func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemOccurrenceList, []string, error) { + var response extractionResponse + if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil { + return nil, nil, extractorErrorf("decode native output: %w", err) + } + value, err := canonicalItemOccurrenceList(response, order, sourceID, registry) + if err != nil { + return nil, []string{"Select the exact canonical item name from the supplied registry for every occurrence. Omit unsupported or ambiguous selections; do not invent an item. Return the complete replacement occurrence list, keeping current-transcript evidence separate from registry context."}, nil + } + return &value, nil, nil + }, + func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, + Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil + }, + func(context.Context, *dnd.ItemOccurrenceList) (dnd.ItemOccurrenceList, error) { + return dnd.ItemOccurrenceList{Occurrences: []dnd.ItemOccurrence{}}, nil + }) if err != nil { - return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err) + return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err) } - value, err := canonicalItemOccurrenceList(response, order, req.Source.ID, registry) - if err != nil { - return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{}, extractorErrorf("map item occurrence response: %w", err) - } - return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, ModelCandidate: candidate}, nil + return contracts.TypedExtractionResult[dnd.ItemOccurrenceList]{Value: value, Outcome: outcome}, nil } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{ - Key: Key, - Stage: pipeline.StageExtract, - ExecutionClass: contracts.ExecutionClassLLMBacked, - CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, - Requires: append([]string(nil), requiredCapabilities...), - Provides: append([]string(nil), providedCapabilities...), - ArtifactKind: dnd.ItemOccurrenceListKind, - ReferenceSlots: referenceSlots(), + Key: Key, + Stage: pipeline.StageExtract, + ExecutionClass: contracts.ExecutionClassLLMBacked, + Prompts: []string{PromptID, CorrectionPromptID}, + Requires: append([]string(nil), requiredCapabilities...), + Provides: append([]string(nil), providedCapabilities...), + ArtifactKind: dnd.ItemOccurrenceListKind, + ReferenceSlots: referenceSlots(), } } @@ -193,7 +160,7 @@ func Register(registry *pipeline.ExtractorRegistry) error { if err != nil { return nil, err } - return New(request.Dependencies.LLM, options, request.References) + return New(request.Dependencies.Engine, options, request.References) }) } diff --git a/internal/modules/dnd/extract/itemoccurrences/extractor_test.go b/internal/modules/dnd/extract/itemoccurrences/extractor_test.go index 81bd7a17..38eb60af 100644 --- a/internal/modules/dnd/extract/itemoccurrences/extractor_test.go +++ b/internal/modules/dnd/extract/itemoccurrences/extractor_test.go @@ -8,20 +8,14 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" itemidentity "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" - itemoccurrenceshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape" ) func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) { id := itemidentity.DeriveID("Torch") - rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","from":"party","source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`) + rawResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"lost","quantity":null,"from":"party","to":null,"source_refs":[{"start_unit_id":2,"end_unit_id":1}]}]}`) client := &fakeItemOccurrencesLLMClient{content: append([]byte(nil), rawResponse...)} req := extractionRequest() req.References = itemRegistryReferences(t) - correction, err := contracts.NewSemanticCorrection([]byte(`{"occurrences":[]}`), "Keep the transcript-grounded item occurrence.") - if err != nil { - t.Fatalf("NewSemanticCorrection() error = %v", err) - } - req.Correction = correction result, err := newExtractor(t, client, req.References).Extract(context.Background(), req) if err != nil { t.Fatal(err) @@ -33,28 +27,18 @@ func TestExtractGroundsOccurrencesInRequiredRegistry(t *testing.T) { t.Fatalf("occurrence evidence = %#v, want current-source unit range", refs) } request := client.requests[0] - input := request.Inputs[ItemRegistryReferenceSlot] - if input.Name != ItemRegistryReferenceSlot || string(input.Content) != `{"items":[{"name":"Torch"}]}` || strings.Contains(string(input.Content), "item:sha256:") { - t.Fatalf("registry prompt input = %#v, want names-only projection", input) + + input := renderedInput(t, request, ItemRegistryReferenceSlot) + if string(input) != `{"items":[{"name":"Torch"}]}` { + t.Fatalf("names-only registry = %s", input) } - if request.Correction == nil || string(request.Correction.AssistantResponse) != `{"occurrences":[]}` || request.Correction.UserGuidance != "Keep the transcript-grounded item occurrence." { - t.Fatalf("correction = %#v, want exact request correction", request.Correction) - } - for _, material := range []string{string(request.Correction.AssistantResponse), request.Correction.UserGuidance} { - if strings.Contains(material, id) || strings.Contains(material, "item:sha256:") { - t.Fatalf("correction material leaked opaque item identity: %q", material) + for _, message := range request.Prompt.Messages { + if strings.Contains(message.Content, id) || strings.Contains(message.Content, "item:sha256:") { + t.Fatal("model context exposed durable item identity") } } - correction.AssistantResponse[0] = '[' - if got := string(request.Correction.AssistantResponse); got != `{"occurrences":[]}` { - t.Fatalf("captured correction changed after caller mutation: %q", got) - } - if result.ModelCandidate == nil || result.ModelCandidate.Protocol != contracts.CorrectionProtocolSingleResponseV1 || string(result.ModelCandidate.Response) != string(rawResponse) { - t.Fatalf("model candidate = %#v, want exact validated response", result.ModelCandidate) - } - client.content[0] = '[' - if got := string(result.ModelCandidate.Response); got != string(rawResponse) { - t.Fatalf("model candidate changed after provider buffer mutation: %q", got) + if request.Prompt.SessionID != req.SessionID || result.Outcome.Disposition != contracts.ProducerAccepted { + t.Fatal("native selection or acceptance lost") } } @@ -83,8 +67,8 @@ func TestExtractRejectsUnknownNameAfterValidOccurrence(t *testing.T) { req := extractionRequest() req.References = itemRegistryReferences(t) result, err := newExtractor(t, client, req.References).Extract(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "occurrences[1].name is not in the item registry") { - t.Fatalf("Extract() error = %v, want occurrence index and unknown name", err) + if err == nil || result.Outcome.Disposition != "" { + t.Fatalf("Extract() error = %v, want atomic unknown-name failure", err) } if len(result.Value.Occurrences) != 0 { t.Fatalf("Extract() returned partial occurrences: %#v", result.Value.Occurrences) @@ -114,12 +98,12 @@ func TestExtractAcceptsOnlyEmptyResponseForEmptyRegistry(t *testing.T) { req.References = references result, err := newExtractor(t, client, references).Extract(context.Background(), req) if test.wantErr { - if err == nil || !strings.Contains(err.Error(), "name is not in the item registry") || len(result.Value.Occurrences) != 0 { + if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 { t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err) } return } - if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 { + if err != nil || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible { t.Fatalf("Extract() = %#v, %v; want present empty occurrences", result, err) } }) @@ -144,50 +128,9 @@ func TestExtractUsesOnlySupportedPromptInputs(t *testing.T) { if _, err := newExtractor(t, client, req.References).Extract(context.Background(), req); err != nil { t.Fatal(err) } - if _, ok := client.requests[0].Inputs["unrelated"]; ok { - t.Fatalf("unexpected prompt input: %#v", client.requests[0].Inputs) - } -} - -func TestExtractCarriesRejectedHolderGuidanceIntoCorrectedAttempt(t *testing.T) { - references := itemRegistryReferences(t) - req := extractionRequest() - req.References = references - defectiveResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`) - defective, err := newExtractor(t, &fakeItemOccurrencesLLMClient{content: defectiveResponse}, references).Extract(context.Background(), req) - if err != nil { - t.Fatal(err) - } - validation, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: defective.Value}) - if err != nil || validation.Approved { - t.Fatalf("holder validation = %#v, %v; want rejection", validation, err) - } - for _, fragment := range []string{"For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} { - if !strings.Contains(validation.CorrectionGuidance, fragment) { - t.Fatalf("CorrectionGuidance = %q, want %q", validation.CorrectionGuidance, fragment) + for _, message := range client.requests[0].Prompt.Messages { + if strings.Contains(message.Content, "ignored") { + t.Fatal("unsupported reference became a prompt input") } } - for _, forbidden := range []string{defective.Value.Occurrences[0].ItemID, "item_id", "sha256", itemoccurrenceshape.Key, itemoccurrenceshape.ReasonCode} { - if strings.Contains(validation.CorrectionGuidance, forbidden) { - t.Fatalf("CorrectionGuidance leaked implementation identifier %q: %q", forbidden, validation.CorrectionGuidance) - } - } - correction, err := contracts.NewSemanticCorrection(defective.ModelCandidate.Response, validation.CorrectionGuidance) - if err != nil { - t.Fatal(err) - } - correctedResponse := []byte(`{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}`) - client := &fakeItemOccurrencesLLMClient{content: correctedResponse} - req.Correction = correction - corrected, err := newExtractor(t, client, references).Extract(context.Background(), req) - if err != nil { - t.Fatal(err) - } - accepted, err := itemoccurrenceshape.New(itemoccurrenceshape.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{Value: corrected.Value}) - if err != nil || !accepted.Approved { - t.Fatalf("corrected holder validation = %#v, %v; want approval", accepted, err) - } - if len(client.requests) != 1 || client.requests[0].Correction == nil || string(client.requests[0].Correction.AssistantResponse) != string(defectiveResponse) || client.requests[0].Correction.UserGuidance != validation.CorrectionGuidance { - t.Fatalf("corrected request = %#v, want byte-faithful defective response and semantic guidance", client.requests) - } } diff --git a/internal/modules/dnd/extract/itemoccurrences/native_execution_test.go b/internal/modules/dnd/extract/itemoccurrences/native_execution_test.go new file mode 100644 index 00000000..c5416c4c --- /dev/null +++ b/internal/modules/dnd/extract/itemoccurrences/native_execution_test.go @@ -0,0 +1,213 @@ +package itemoccurrences + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" + occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemoccurrences" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" + registryvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/registry" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/shape" + sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_refs" + relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemoccurrences/source_relatedness" + "go.maximumdirect.net/promptkit" +) + +const rejectedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":"Chest","to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}` +const correctedHolder = `{"occurrences":[{"name":"Torch","kind":"acquired","quantity":null,"from":null,"to":"party","source_refs":[{"start_unit_id":2,"end_unit_id":2}]}]}` +const unknownItem = `{"occurrences":[{"name":"Unknown Item","kind":"discovered","quantity":null,"from":null,"to":null,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}` + +func bindAssessment(t *testing.T, request *contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]) (*[]string, *[]string) { + t.Helper() + registry, err := registryvalidator.New(registryvalidator.Options{}, request.References) + if err != nil { + t.Fatal(err) + } + validators := []contracts.TypedValidator[dnd.ItemOccurrenceList]{ + shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), + relatedness.New(relatedness.Options{}), registry, + } + var calls, guidance []string + request.Operation.Assess = func(ctx context.Context, value dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) { + var assessment contracts.CandidateAssessment + for _, validator := range validators { + calls = append(calls, validator.Name()) + result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.ItemOccurrenceList]{ + Stage: "extract", Source: request.Source, Chunk: request.Chunk, References: request.References, Value: value, + }) + if err != nil { + return contracts.CandidateAssessment{}, err + } + decision := contracts.AssessmentRejected + if result.Approved { + decision = contracts.AssessmentApproved + } + assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result}) + if result.CorrectionGuidance != "" { + guidance = append(guidance, result.CorrectionGuidance) + } + } + return assessment, nil + } + return &calls, &guidance +} + +func TestExtractCarriesRejectedHolderGuidanceIntoNativeCorrection(t *testing.T) { + request := extractionRequest() + request.References = itemRegistryReferences(t) + request.Operation.Controls.MaxAdditionalTurns = 1 + request.Operation.ReuseEligible = false + request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "item-occurrence-override"} + var prepared []promptkit.PreparedRun + request.Operation.Execution.OnPrepared = func(_ context.Context, run promptkit.PreparedRun) { prepared = append(prepared, run) } + calls, guidance := bindAssessment(t, &request) + client := &fakeItemOccurrencesLLMClient{responses: []string{rejectedHolder, correctedHolder}} + result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) + chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, registryvalidator.Key} + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible || + !reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(result.Value.Occurrences) != 1 || len(client.requests) != 2 || len(prepared) != 2 || len(*guidance) != 1 { + t.Fatalf("correction = %#v %v, validators=%#v guidance=%#v native=%d/%d", result, err, *calls, *guidance, len(client.requests), len(prepared)) + } + occurrence := result.Value.Occurrences[0] + if occurrence.ItemID != identity.DeriveID("Torch") || occurrence.Name != "Torch" || occurrence.Kind != dnd.ItemOccurrenceKindAcquired || + occurrence.From != "" || occurrence.To != "party" || occurrence.Quantity != nil || occurrence.SourceRefs[0].SourceID != request.Source.ID { + t.Fatalf("holder mapping = %#v", occurrence) + } + first, next := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages + if len(next) != len(first)+2 || !reflect.DeepEqual(first, next[:len(first)]) || next[len(first)].Role != "assistant" || next[len(first)].Content != rejectedHolder { + t.Fatal("native continuation did not preserve the exact defective response and fixed context") + } + correction := next[len(first)+1].Content + for _, fragment := range []string{(*guidance)[0], "For `acquired` occurrences", "`from` to JSON null", "item \"Torch\"", "source unit 2"} { + if !strings.Contains(correction, fragment) { + t.Fatalf("missing holder guidance %q", fragment) + } + } + for _, forbidden := range []string{occurrence.ItemID, "item_id", "sha256", shape.Key, shape.ReasonCode} { + if strings.Contains(correction, forbidden) { + t.Fatalf("operator identifier became guidance: %q", forbidden) + } + } + for i, call := range client.requests { + id := PromptID + if i > 0 { + id = CorrectionPromptID + } + if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "item-occurrence-override" || call.Prompt.SessionID != request.SessionID { + t.Fatal("native profile/session/override lost") + } + if string(renderedInput(t, call, ItemRegistryReferenceSlot)) != `{"items":[{"name":"Torch"}]}` { + t.Fatal("item registry was not fixed and names-only") + } + } + content, err := occurrencecodec.New().Encode(result.Value) + if err != nil { + t.Fatal(err) + } + var wire struct { + Occurrences []map[string]json.RawMessage `json:"occurrences"` + } + if err := json.Unmarshal(content, &wire); err != nil { + t.Fatal(err) + } + if _, present := wire.Occurrences[0]["from"]; present { + t.Fatal("private null holder became a durable field") + } +} + +func TestItemOccurrenceTerminalModesDoNotSynthesizeHolders(t *testing.T) { + failure := errors.New("provider unavailable") + for _, test := range []struct { + name string + mode contracts.ExecutionMode + response string + failures []error + assessments, runs int + wantError bool + }{ + {"strict holder rejection", contracts.ModeStrict, rejectedHolder, nil, 2, 2, true}, + {"permissive holder rejection", contracts.ModePermissive, rejectedHolder, nil, 2, 2, false}, + {"strict unknown item", contracts.ModeStrict, unknownItem, nil, 0, 2, true}, + {"permissive unknown item", contracts.ModePermissive, unknownItem, nil, 0, 2, false}, + {"initial provider failure", contracts.ModePermissive, "", []error{failure}, 0, 1, false}, + {"failed correction", contracts.ModePermissive, rejectedHolder, []error{nil, failure}, 1, 2, false}, + {"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false}, + } { + t.Run(test.name, func(t *testing.T) { + request := extractionRequest() + request.References = itemRegistryReferences(t) + request.Operation.Controls.Mode = test.mode + request.Operation.Controls.MaxAdditionalTurns = 1 + calls, _ := bindAssessment(t, &request) + runs := 0 + request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { runs++ } + client := &fakeItemOccurrencesLLMClient{content: []byte(test.response), errors: test.failures} + result, err := newExtractor(t, client, request.References).Extract(context.Background(), request) + if (err != nil) != test.wantError || len(*calls) != test.assessments*4 || runs != test.runs { + t.Fatalf("terminal = %#v %v, validators=%#v runs=%d", result, err, *calls, runs) + } + if test.wantError { + if result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 { + t.Fatal("strict failure returned output") + } + return + } + if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 { + t.Fatalf("fallback invented an item or holder: %#v", result) + } + if _, err := occurrencecodec.New().Encode(result.Value); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestItemOccurrenceCancellationCannotPublishFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request := extractionRequest() + request.References = itemRegistryReferences(t) + request.Operation.Controls.Mode = contracts.ModePermissive + client := &fakeItemOccurrencesLLMClient{content: []byte(correctedHolder), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }} + result, err := newExtractor(t, client, request.References).Extract(ctx, request) + if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 { + t.Fatalf("canceled delivery = %#v %v", result, err) + } +} + +func TestItemOccurrenceKindsHoldersQuantityAndDocumentOrderSurvive(t *testing.T) { + quantity := 3 + client := &fakeItemOccurrencesLLMClient{response: extractionResponse{Occurrences: []itemOccurrenceResponse{ + {Name: "Torch", Kind: "transferred", From: "Aria", To: "Borin", SourceRefs: responseRefs(5, 5)}, + {Name: "Torch", Kind: "consumed", From: "Aria", SourceRefs: responseRefs(4, 4)}, + {Name: "Torch", Kind: "lost", From: "party", SourceRefs: responseRefs(3, 3)}, + {Name: "Torch", Kind: "acquired", To: "party", Quantity: &quantity, SourceRefs: responseRefs(2, 2)}, + {Name: "Torch", Kind: "discovered", SourceRefs: responseRefs(1, 1)}, + }}} + request := extractionRequest() + request.References = itemRegistryReferences(t) + bindAssessment(t, &request) + result, err := newExtractor(t, client).Extract(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || len(result.Value.Occurrences) != 5 { + t.Fatalf("kinds = %#v %v", result, err) + } + kinds := []dnd.ItemOccurrenceKind{dnd.ItemOccurrenceKindDiscovered, dnd.ItemOccurrenceKindAcquired, dnd.ItemOccurrenceKindLost, dnd.ItemOccurrenceKindConsumed, dnd.ItemOccurrenceKindTransferred} + for i, occurrence := range result.Value.Occurrences { + if occurrence.Kind != kinds[i] || occurrence.SourceRefs[0].StartUnitID != i+1 { + t.Fatalf("ordered occurrence = %#v", occurrence) + } + } + quantity = 9 + if *result.Value.Occurrences[1].Quantity != 3 || result.Value.Occurrences[4].From != "Aria" || result.Value.Occurrences[4].To != "Borin" { + t.Fatal("holder or quantity interpretation changed") + } + if _, err := occurrencecodec.New().Encode(result.Value); err != nil { + t.Fatal(err) + } +} diff --git a/internal/modules/dnd/extract/itemoccurrences/prompt_assets.go b/internal/modules/dnd/extract/itemoccurrences/prompt_assets.go deleted file mode 100644 index 5babe99b..00000000 --- a/internal/modules/dnd/extract/itemoccurrences/prompt_assets.go +++ /dev/null @@ -1,75 +0,0 @@ -package itemoccurrences - -import ( - "fmt" - "io/fs" - "sync" - - rootassets "gitea.maximumdirect.net/eric/notarius/assets" - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" -) - -const promptAssetRoot = "assets/prompts" - -var promptAssetManifest = shared.PromptAssetManifest{ - ModuleDir: "dnd.item_occurrences", - ModuleFiles: []promptfs.ModulePromptFile{ - {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, - {Name: "instructions.md", Path: "prompts/instructions.md"}, - {Name: "item-registry.md", Path: "prompts/item-registry.md"}, - }, - SharedFiles: []string{ - "common-dnd-system.md", - "common-dnd-extraction-evidence.md", - "common-dnd-identity.md", - "common-dnd-references.md", - "common-dnd-transcript-chunk.md", - }, -} - -func moduleAssetFS() (fs.FS, error) { - assets, err := fs.Sub(rootassets.FS(), "dnd/item-occurrences") - if err != nil { - return nil, fmt.Errorf("scope item-occurrence assets: %w", err) - } - return assets, nil -} - -func RegisterPromptAssets(registry *llm.AssetRegistry) error { - assets, err := moduleAssetFS() - if err != nil { - return err - } - promptFS, err := promptAssetManifest.PromptFS(assets) - if err != nil { - return fmt.Errorf("prepare item occurrence prompt assets: %w", err) - } - if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { - return err - } - schemas, err := fs.Sub(assets, "schemas") - if err != nil { - return fmt.Errorf("scope item-occurrence schemas: %w", err) - } - return registry.RegisterSchemaFS(schemas, ".") -} - -func promptAssetMetadata() (string, error) { - promptAssetHashOnce.Do(func() { - assets, err := moduleAssetFS() - if err != nil { - promptAssetHashErr = err - return - } - promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) - }) - return promptAssetHash, promptAssetHashErr -} - -var ( - promptAssetHashOnce sync.Once - promptAssetHash string - promptAssetHashErr error -) diff --git a/internal/modules/dnd/extract/itemoccurrences/prompt_assets_test.go b/internal/modules/dnd/extract/itemoccurrences/prompt_assets_test.go deleted file mode 100644 index 58ec0aeb..00000000 --- a/internal/modules/dnd/extract/itemoccurrences/prompt_assets_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package itemoccurrences - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture" - "gitea.maximumdirect.net/eric/promptkit" -) - -func TestPromptAssetsPrepareItemOccurrencePrompt(t *testing.T) { - registry := llm.NewAssetRegistry() - if err := RegisterPromptAssets(registry); err != nil { - t.Fatal(err) - } - engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{ - "item-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "item-occurrences-test-model"), - }) - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "item-occurrences-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI("file:///session.json", `{"units":[{"sentinel":"item-occurrence-transcript"}]}`), - "players": promptkit.Inline("item-occurrence-player"), - "party": promptkit.Inline(" "), - "glossary": promptkit.Inline(" "), - "item_registry": promptkit.Inline(`{"items":[{"name":"Torch"}]}`), - }, - }) - if err != nil { - t.Fatal(err) - } - if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_item_occurrences_llm.v1.json" { - t.Fatalf("prepared prompt = %#v", prepared) - } - rendered := make([]string, len(prepared.Messages)) - for index, message := range prepared.Messages { - rendered[index] = message.Content - } - content := strings.Join(rendered, "\n") - for _, field := range []string{"start_unit_id", "end_unit_id"} { - if !strings.Contains(content, field) { - t.Fatalf("prepared prompt does not include shared evidence field %q", field) - } - } - for _, obsolete := range []string{"source_id", "start_segment", "end_segment"} { - if strings.Contains(content, obsolete) { - t.Fatalf("prepared prompt contains obsolete evidence field %q: %s", obsolete, content) - } - } - for _, holderRule := range []string{ - "| `discovered` | `null` | `null` |", - "| `acquired` | `null` | `party` or the named party member gaining possession |", - "| `lost` | `party` or the named party member losing possession | `null` |", - "| `consumed` | `party` or the named party member consuming the item | `null` |", - "| `transferred` | one named party member | a different named party member |", - } { - if !strings.Contains(content, holderRule) { - t.Fatalf("prepared prompt does not include holder rule %q", holderRule) - } - } -} - -func TestPromptAssetsDoNotLeakIntoMetadata(t *testing.T) { - hash, err := promptAssetMetadata() - if err != nil || !strings.HasPrefix(hash, "sha256:") { - t.Fatalf("promptAssetMetadata() = %q, %v", hash, err) - } - metadata := newExtractor(t, &fakeItemOccurrencesLLMClient{}).ManifestMetadata() - payload, err := json.Marshal(metadata) - if err != nil { - t.Fatal(err) - } - for _, forbidden := range []string{"common-dnd-system", "dnd_item_occurrences_llm.v1.json"} { - if strings.Contains(string(payload), forbidden) { - t.Fatalf("metadata leaked raw asset content %q: %s", forbidden, payload) - } - } -} diff --git a/internal/modules/dnd/extract/itemoccurrences/registry_test.go b/internal/modules/dnd/extract/itemoccurrences/registry_test.go index 74242069..89d99430 100644 --- a/internal/modules/dnd/extract/itemoccurrences/registry_test.go +++ b/internal/modules/dnd/extract/itemoccurrences/registry_test.go @@ -36,8 +36,8 @@ func testSourceRefs() []source.SourceRef { func TestModuleSpecDeclaresRequiredRegistry(t *testing.T) { spec := ModuleSpec() - if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 { - t.Fatalf("correction protocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1) + if !reflect.DeepEqual(spec.Prompts, []string{PromptID, CorrectionPromptID}) { + t.Fatalf("prompt dependencies = %#v", spec.Prompts) } var slot contracts.ReferenceSlot for _, candidate := range spec.ReferenceSlots { diff --git a/internal/modules/dnd/extract/itemoccurrences/schema.go b/internal/modules/dnd/extract/itemoccurrences/schema.go deleted file mode 100644 index 506c30eb..00000000 --- a/internal/modules/dnd/extract/itemoccurrences/schema.go +++ /dev/null @@ -1,25 +0,0 @@ -package itemoccurrences - -import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - -const ( - PromptID = "dnd.item_occurrences" - ResponseSchemaKey = llm.ResponseSchemaKey("dnd_item_occurrences_llm") - ResponseSchemaID = "notarius.dnd.item_occurrences.llm" - ResponseSchemaName = "notarius_dnd_item_occurrences_llm_v1" - SchemaVersion = "v1" -) - -func loadResponseSchema() (llm.ResponseSchema, error) { - assets, err := moduleAssetFS() - if err != nil { - return llm.ResponseSchema{}, err - } - return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ - Key: ResponseSchemaKey, - ID: ResponseSchemaID, - Version: SchemaVersion, - Name: ResponseSchemaName, - AssetPath: "schemas/dnd_item_occurrences_llm.v1.json", - }) -} diff --git a/internal/modules/dnd/extract/itemoccurrences/schema_test.go b/internal/modules/dnd/extract/itemoccurrences/schema_test.go deleted file mode 100644 index bff87624..00000000 --- a/internal/modules/dnd/extract/itemoccurrences/schema_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package itemoccurrences - -import ( - "bytes" - "encoding/json" - "strings" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" -) - -func TestResponseSchemaIsStrictlyStructuralAndPrivate(t *testing.T) { - schema, err := loadResponseSchema() - if err != nil { - t.Fatal(err) - } - if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { - t.Fatalf("schema = %#v", schema) - } - valid := map[string]any{"occurrences": []any{ - map[string]any{ - "name": "", "kind": "transferred", "quantity": 0, "from": "party", "to": "Party", - "source_refs": []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}}, - }, - map[string]any{ - "name": "Hidden Cache", "kind": "discovered", "quantity": nil, "from": nil, "to": nil, - "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1}}, - }, - }} - content, err := json.Marshal(valid) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err != nil { - t.Fatalf("private schema rejected validator-owned values: %v", err) - } - for _, test := range []struct { - name string - value map[string]any - }{ - {"missing occurrences", map[string]any{}}, - {"missing occurrence name", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "name")}}}, - {"missing nullable field", map[string]any{"occurrences": []any{withoutField(responseOccurrence(), "quantity")}}}, - {"unsupported kind", map[string]any{"occurrences": []any{withField(responseOccurrence(), "kind", "unsupported")}}}, - {"opaque item identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "item_id", "item:sha256:opaque")}}}, - {"unknown occurrence field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "extra", true)}}}, - {"segment-named range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_segment": 1, "end_segment": 1}})}}}, - {"opaque source identifier", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "source_id": "session-alpha"}})}}}, - {"unknown reference field", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1, "end_unit_id": 1, "extra": true}})}}}, - {"noninteger range", map[string]any{"occurrences": []any{withField(responseOccurrence(), "source_refs", []any{map[string]any{"start_unit_id": 1.5, "end_unit_id": 1}})}}}, - } { - t.Run(test.name, func(t *testing.T) { - content, err := json.Marshal(test.value) - if err != nil { - t.Fatal(err) - } - if err := validateJSONSchema(content, schema.JSONSchema); err == nil { - t.Fatal("private schema accepted invalid structure") - } - }) - } - first := schema.JSONSchema - first[0] = '[' - second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first, second.JSONSchema) { - t.Fatalf("defensive schema copy = %s, %v", second.JSONSchema, err) - } - if _, ok := second.DiagnosticsMap()["json_schema"]; ok { - t.Fatalf("schema diagnostics leaked content: %#v", second.DiagnosticsMap()) - } -} - -func responseOccurrence() map[string]any { - return map[string]any{ - "name": "Ring", "kind": "acquired", "quantity": nil, "from": nil, "to": "party", "source_refs": []any{}, - } -} - -func withoutField(value map[string]any, name string) map[string]any { - delete(value, name) - return value -} - -func withField(value map[string]any, name string, fieldValue any) map[string]any { - value[name] = fieldValue - return value -} - -func validateJSONSchema(instanceContent, schemaContent []byte) error { - instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) - if err != nil { - return err - } - document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - if err := compiler.AddResource("schema.json", document); err != nil { - return err - } - compiled, err := compiler.Compile("schema.json") - if err != nil { - return err - } - return compiled.Validate(instance) -} diff --git a/internal/modules/dnd/extract/itemoccurrences/test_helpers_test.go b/internal/modules/dnd/extract/itemoccurrences/test_helpers_test.go index 41d83332..ee85875f 100644 --- a/internal/modules/dnd/extract/itemoccurrences/test_helpers_test.go +++ b/internal/modules/dnd/extract/itemoccurrences/test_helpers_test.go @@ -3,14 +3,19 @@ package itemoccurrences import ( "context" "encoding/json" - "errors" + "strings" "testing" + "gitea.maximumdirect.net/eric/notarius/assets" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" ) -func extractionRequest() contracts.TypedExtractionRequest { +func extractionRequest() contracts.TypedExtractionRequest[dnd.ItemOccurrenceList] { doc := sourceDocument() chunk := &source.Chunk{ ID: "session-alpha:chunk:0", @@ -21,12 +26,15 @@ func extractionRequest() contracts.TypedExtractionRequest { MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...), } - return contracts.TypedExtractionRequest{ + return contracts.TypedExtractionRequest[dnd.ItemOccurrenceList]{ Source: doc, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"), SessionID: "session-123", LLMProfile: "profile-item-occurrences", + Operation: contracts.ProducerOperation[dnd.ItemOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.ItemOccurrenceList) (contracts.CandidateAssessment, error) { + return contracts.CandidateAssessment{}, nil + }}, } } @@ -47,51 +55,111 @@ func responseRefs(start, end int) []itemOccurrenceSourceRefResponse { return []itemOccurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}} } -func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor { +func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor { t.Helper() - extractor, err := New(client, Options{}, references...) + type settings struct { + PromptKit promptkit.Settings `yaml:"promptkit"` + } + definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit }) + filesystem := configkit.NewFilesystem("Notarius assets", assets.FS()) + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit: + profiles: + - id: profile-item-occurrences + endpoint: https://example.invalid/v1 + model: occurrence-test-model + api_key_required: false +`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{ + promptkit.BundledOpenRouterUnderlay(), + {Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}}, + }}) if err != nil { - t.Fatalf("New() error = %v", err) + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + extractor, err := New(engine, Options{}, references...) + if err != nil { + t.Fatal(err) } return extractor } -func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest { - req.Inputs = req.Inputs.Clone() - correction, err := contracts.CloneSemanticCorrection(req.Correction) - if err != nil { - panic(err) +func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte { + t.Helper() + key := map[string]string{"transcript": "segments", "item_registry": "items"}[name] + for _, message := range request.Prompt.Messages { + start := strings.Index(message.Content, "{") + if start < 0 { + continue + } + content := []byte(message.Content[start:]) + var fields map[string]json.RawMessage + if json.Unmarshal(content, &fields) == nil && fields[key] != nil { + return content + } } - req.Correction = correction - return req + t.Fatalf("missing rendered %s input", name) + return nil } type fakeItemOccurrencesLLMClient struct { - response extractionResponse - content []byte - err error - requests []contracts.StructuredCompletionRequest + response extractionResponse + responses []string + content []byte + err error + errors []error + requests []promptkit.GenerateRequest + onGenerate func(context.Context, promptkit.GenerateRequest) } -func (client *fakeItemOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.requests = append(client.requests, cloneStructuredCompletionRequest(req)) +func (client *fakeItemOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + index := len(client.requests) + client.requests = append(client.requests, request) + if client.onGenerate != nil { + client.onGenerate(ctx, request) + } + if err := ctx.Err(); err != nil { + return nil, err + } if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err + return nil, client.err } - target, ok := out.(*extractionResponse) - if !ok { - return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target") + if index < len(client.errors) && client.errors[index] != nil { + return nil, client.errors[index] } - content := append([]byte(nil), client.content...) - if len(content) == 0 { - var err error - content, err = json.Marshal(client.response) + content := string(client.content) + if index < len(client.responses) { + content = client.responses[index] + } + if content == "" { + encoded, err := completeResponse(client.response) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err + } + content = string(encoded) + } + return &promptkit.GenerateResponse{Content: content}, nil +} + +func completeResponse(response extractionResponse) ([]byte, error) { + content, err := json.Marshal(response) + if err != nil { + return nil, err + } + var wire struct { + Occurrences []map[string]any `json:"occurrences"` + } + if err := json.Unmarshal(content, &wire); err != nil { + return nil, err + } + for _, occurrence := range wire.Occurrences { + for _, name := range []string{"quantity", "from", "to"} { + if _, present := occurrence[name]; !present { + occurrence[name] = nil + } } } - if err := json.Unmarshal(content, target); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: content}, nil + return json.Marshal(wire) }