Migrate NPC and location occurrences to native correction and explicit empty fallbacks
This commit is contained in:
7
assets/dnd/location-occurrences/prompts/correction.md
Normal file
7
assets/dnd/location-occurrences/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous location occurrence list using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return a complete replacement occurrence list using the original transcript and
|
||||
registry context. Keep identity-grounding context separate from current-transcript
|
||||
evidence, and return only JSON matching the original response contract.
|
||||
@@ -20,7 +20,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -30,11 +30,11 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ./location-registry.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -42,3 +42,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-occurrences/schemas/dnd_location_occurrences_llm.v1.json
|
||||
|
||||
- id: dnd.location_occurrences.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/location-occurrences/schemas/dnd_location_occurrences_llm.v1.json
|
||||
|
||||
7
assets/dnd/npc-occurrences/prompts/correction.md
Normal file
7
assets/dnd/npc-occurrences/prompts/correction.md
Normal file
@@ -0,0 +1,7 @@
|
||||
Revise the previous NPC occurrence list using these findings:
|
||||
|
||||
{{ input "guidance" }}
|
||||
|
||||
Return a complete replacement occurrence list using the original transcript and
|
||||
registry context. Keep identity-grounding context separate from current-transcript
|
||||
evidence, and return only JSON matching the original response contract.
|
||||
@@ -20,7 +20,7 @@ prompts:
|
||||
messages:
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-system.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-identity.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-references.md
|
||||
@@ -30,11 +30,11 @@ prompts:
|
||||
content_file: ../../shared/prompts/common-dnd-transcript-chunk.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ../../shared/prompts/common-dnd-extraction-evidence.md
|
||||
- role: user
|
||||
content_file: ../../shared/prompts/common-dnd-npc-registry.md
|
||||
- role: user
|
||||
- role: system
|
||||
content_file: ./instructions.md
|
||||
cache_control:
|
||||
type: ephemeral
|
||||
@@ -42,3 +42,17 @@ prompts:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-occurrences/schemas/dnd_npc_occurrences_llm.v1.json
|
||||
|
||||
- id: dnd.npc_occurrences.repair
|
||||
default_profile: notarius/dnd-extraction
|
||||
inputs:
|
||||
- name: guidance
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./correction.md
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: dnd/npc-occurrences/schemas/dnd_npc_occurrences_llm.v1.json
|
||||
|
||||
140
assets/occurrence_extraction_test.go
Normal file
140
assets/occurrence_extraction_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestOccurrencePromptPairsPreserveRegistryAndTranscriptBoundaries(t *testing.T) {
|
||||
_, engine := loadNativeAssets(t)
|
||||
for _, test := range []struct {
|
||||
family, registry string
|
||||
rules []string
|
||||
}{
|
||||
{"npc", `{"npcs":[{"name":"occurrence-registry-name"}]}`, []string{"Use the supplied canonical NPC", "noncombat_presence", "combat_opponent"}},
|
||||
{"location", `{"locations":[{"name":"occurrence-registry-name","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry-context"}]}]}`,
|
||||
[]string{"context supports that coreference", "must not create a registry location", "provenance must never replace current-chunk evidence", "complete ordered"}},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
id := "notarius/dnd." + test.family + "_occurrences"
|
||||
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id, APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"occurrence-transcript"}]}`),
|
||||
test.family + "_registry": promptkit.Inline(test.registry),
|
||||
"players": promptkit.Inline("occurrence-players"), "party": promptkit.Inline("occurrence-party"), "glossary": promptkit.Inline("occurrence-glossary"),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer initial.Discard()
|
||||
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||
PromptID: id + ".repair", APIKey: "test-only",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("Correct the unsupported occurrence.")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer correction.Discard()
|
||||
first, next := initial.Details(), correction.Details()
|
||||
if first.SelectedProfileID != "notarius/dnd-extraction" || first.SelectedProfileID != next.SelectedProfileID ||
|
||||
first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
|
||||
t.Fatal("occurrence native pair lost schema/defaults")
|
||||
}
|
||||
var system, all string
|
||||
for _, message := range first.Messages {
|
||||
all += message.Content
|
||||
if message.Role == "system" {
|
||||
system += message.Content
|
||||
}
|
||||
}
|
||||
for _, rule := range append(test.rules, "Transcript units are the only evidence") {
|
||||
if !strings.Contains(strings.Join(strings.Fields(system), " "), rule) {
|
||||
t.Fatalf("missing enduring rule %q", rule)
|
||||
}
|
||||
}
|
||||
for _, content := range []string{"occurrence-registry-name", "occurrence-transcript", "occurrence-players", "occurrence-party", "occurrence-glossary"} {
|
||||
if strings.Count(all, content) != 1 {
|
||||
t.Fatalf("input not rendered once: %q", content)
|
||||
}
|
||||
}
|
||||
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, "Correct the unsupported occurrence.") ||
|
||||
strings.Contains(next.Messages[0].Content, "occurrence-registry-name") {
|
||||
t.Fatal("correction repeated fixed inputs")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOccurrenceSchemasRetainSupportedKindsAndPrivateSelectors(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
family string
|
||||
kinds []string
|
||||
}{
|
||||
{"npc", []string{"mentioned", "noncombat_presence", "dialogue", "combat_ally", "combat_opponent", "other"}},
|
||||
{"location", []string{"visited", "planned", "recalled", "mentioned"}},
|
||||
} {
|
||||
t.Run(test.family, func(t *testing.T) {
|
||||
schema, err := fs.ReadFile(assets.FS(), "dnd/"+test.family+"-occurrences/schemas/dnd_"+test.family+"_occurrences_llm.v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record := func() map[string]any {
|
||||
value := map[string]any{"name": "Canonical Name", "kind": test.kinds[0], "source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}}
|
||||
if test.family == "location" {
|
||||
value["registry_refs"] = []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}
|
||||
}
|
||||
return value
|
||||
}
|
||||
check := func(value map[string]any, valid bool) {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(map[string]any{"occurrences": []any{value}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema); (err == nil) != valid {
|
||||
t.Fatalf("valid=%t response=%s error=%v", valid, content, err)
|
||||
}
|
||||
}
|
||||
for _, kind := range test.kinds {
|
||||
value := record()
|
||||
value["kind"] = kind
|
||||
check(value, true)
|
||||
}
|
||||
semantic := record()
|
||||
semantic["name"] = ""
|
||||
semantic["source_refs"] = []any{map[string]any{"start_unit_id": 0, "end_unit_id": -1}}
|
||||
check(semantic, true)
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(value map[string]any) { delete(value, "name") },
|
||||
func(value map[string]any) { value["kind"] = 1 },
|
||||
func(value map[string]any) { value["kind"] = "unsupported" },
|
||||
func(value map[string]any) { value[test.family+"_id"] = "assigned-later" },
|
||||
func(value map[string]any) { value["unexpected"] = true },
|
||||
func(value map[string]any) {
|
||||
value["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned-later"
|
||||
},
|
||||
} {
|
||||
value := record()
|
||||
mutate(value)
|
||||
check(value, false)
|
||||
}
|
||||
if test.family == "location" {
|
||||
value := record()
|
||||
delete(value, "registry_refs")
|
||||
check(value, false)
|
||||
value = record()
|
||||
value["registry_refs"].([]any)[0].(map[string]any)["start_unit_id"] = 0
|
||||
check(value, false)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -858,6 +858,28 @@ from Stage 12. Retain meaningful per-family grounding and identity regressions.
|
||||
|
||||
## Stage 19: Migrate NPC And Location Occurrence Extraction
|
||||
|
||||
**Status:** Complete. Both occurrence producers now use fixed transcript/registry
|
||||
materials, native initial/correction requests, and the shared typed producer
|
||||
helper. Added same-schema/default-profile correction pairs and static prompt
|
||||
dependencies; removed private completion, schema/asset registration, and
|
||||
manifest/fingerprint metadata. Names-only NPC resolution and complete ordered
|
||||
contextual location selectors retain deterministic durable identity, supported
|
||||
kinds, source-order canonicalization, and current-transcript evidence. Unknown
|
||||
or ambiguous selections supply actionable interpretation guidance before typed
|
||||
assessment rather than guessing an identity or returning partial output.
|
||||
Permissive fallbacks are codec-valid nonnil empty collections, explicitly
|
||||
qualified and reuse-ineligible; accepted work inherits incoming ineligibility.
|
||||
Retained domain regressions with native provider fakes and added fixed resolution/
|
||||
history, source correction, run override, strict/permissive, failed continuation,
|
||||
structural-failure, and cancellation cases. Prompt-pair/schema tests run in the
|
||||
content package. Asset and shape tests/vet/race checks, independent NPC response
|
||||
model tests/race checks, focused pipeline tests, and `git diff --check` pass.
|
||||
Updated the location-occurrence shape spec comparison for slice-bearing native
|
||||
dependencies. Both full extractor packages remain blocked at setup by shared
|
||||
legacy helpers importing old ConfigKit's removed YAML `plugin/limit`; their
|
||||
native runtime tests are not claimed passing. Scheduled dependency cleanup and
|
||||
final whole-tree verification must run those tests.
|
||||
|
||||
**Goal:** registry-consuming occurrence lanes retain contextual resolution.
|
||||
|
||||
**Inspect/change:** `internal/modules/dnd/extract/{npcoccurrences,locationoccurrences}`,
|
||||
|
||||
@@ -3,8 +3,12 @@ package locationoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
@@ -13,10 +17,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/location-occurrences"
|
||||
mappingPolicy = "dnd.location_occurrences.extract_mapping.v3"
|
||||
)
|
||||
const Key = "dnd/location-occurrences"
|
||||
const PromptID = "notarius/dnd.location_occurrences"
|
||||
const CorrectionPromptID = "notarius/dnd.location_occurrences.repair"
|
||||
|
||||
const (
|
||||
LocationRegistryReferenceSlot = locationregistry.ReferenceSlot
|
||||
@@ -48,21 +51,17 @@ func referenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.LocationOccurrenceList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
locationResolver *locationregistry.Resolver
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
engine *promptkit.Engine
|
||||
locationResolver *locationregistry.Resolver
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
@@ -75,62 +74,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("prepare location registry prompt input: %w", err)
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{llm: llmClient, locationResolver: locationResolver, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||
return &Extractor{engine: engine, locationResolver: locationResolver}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"mapping_policy": mappingPolicy,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
}
|
||||
seeded := e.locationResolver.Seeded()
|
||||
if seeded.Bound() {
|
||||
metadata["location_registry_digest"] = seeded.Digest()
|
||||
metadata["location_count"] = seeded.Count()
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
{Name: "location_registry", Value: e.locationResolver.Seeded().IdentityDigest()},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.LocationOccurrenceList], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]) (contracts.TypedExtractionResult[dnd.LocationOccurrenceList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||
if err != nil {
|
||||
@@ -148,33 +104,46 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("prepare location grounding: %w", err)
|
||||
}
|
||||
|
||||
var response extractionResponse
|
||||
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs[LocationRegistryReferenceSlot] = grounding.PromptInput()
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction, Inputs: inputs,
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
order := shared.NewSourceRefOrder(req.Source)
|
||||
|
||||
sourceID := req.Source.ID
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
materials[LocationRegistryReferenceSlot] = grounding.PromptInput()
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationOccurrenceList, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
value, err := canonicalOccurrenceList(response, order, sourceID, grounding)
|
||||
if err != nil {
|
||||
return nil, []string{"Resolve every occurrence to exactly one supplied location selector. Copy its canonical name and complete ordered registry_refs array; use an empty registry_refs array for a unique name. Omit unsupported or ambiguous selections. Keep occurrence evidence in source_refs separate from registry context, and return the complete replacement occurrence list."}, nil
|
||||
}
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *dnd.LocationOccurrenceList) (dnd.LocationOccurrenceList, error) {
|
||||
return dnd.LocationOccurrenceList{Occurrences: []dnd.LocationOccurrence{}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err)
|
||||
}
|
||||
occurrences, err := canonicalOccurrenceList(response, shared.NewSourceRefOrder(req.Source), req.Source.ID, grounding)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("resolve location grounding: %w", err)
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{Value: occurrences, ModelCandidate: candidate}, nil
|
||||
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.LocationOccurrenceListKind, ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
@@ -185,7 +154,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package locationoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -70,22 +69,19 @@ func TestExtractResolvesContextualSelectorsAndUsesCurrentTranscriptEvidenceOnly(
|
||||
if occurrence := result.Value.Occurrences[0]; occurrence.LocationID != second.ID || occurrence.Name != second.Name || occurrence.SourceRefs[0].SourceID != req.Source.ID {
|
||||
t.Fatalf("occurrence = %#v", occurrence)
|
||||
}
|
||||
input := client.requests[0].Inputs[LocationRegistryReferenceSlot]
|
||||
if input.Name != LocationRegistryReferenceSlot || !strings.Contains(string(input.Content), `"registry_refs":[{"start_unit_id":20,"end_unit_id":20}]`) {
|
||||
t.Fatalf("location prompt input = %#v", input)
|
||||
input := renderedInput(t, client.requests[0], LocationRegistryReferenceSlot)
|
||||
if !strings.Contains(string(input), `"registry_refs":[{"start_unit_id":20,"end_unit_id":20}]`) {
|
||||
t.Fatalf("contextual registry=%s", input)
|
||||
}
|
||||
for _, forbidden := range []string{"source_refs", "source_id", first.ID, second.ID} {
|
||||
if strings.Contains(string(input.Content), forbidden) {
|
||||
t.Fatalf("location prompt leaked %q: %s", forbidden, input.Content)
|
||||
if strings.Contains(string(input), forbidden) {
|
||||
t.Fatalf("location projection exposed %q", forbidden)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(client.requests[0].Inputs["transcript"].Content), "other-session") {
|
||||
t.Fatal("transcript input contains registry evidence")
|
||||
}
|
||||
metadata, err := json.Marshal(newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata())
|
||||
if err != nil || strings.Contains(string(metadata), "other-session") || strings.Contains(string(metadata), first.ID) {
|
||||
t.Fatalf("manifest metadata = %s, %v", metadata, err)
|
||||
if strings.Contains(string(renderedInput(t, client.requests[0], "transcript")), "other-session") {
|
||||
t.Fatal("registry provenance became occurrence evidence")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testing.T) {
|
||||
@@ -101,7 +97,7 @@ func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testin
|
||||
{{Name: known.Name, RegistryRefs: []locationregistry.RegistryRef{}, Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)}, {Name: "Unknown", RegistryRefs: []locationregistry.RegistryRef{}, Kind: "mentioned", SourceRefs: occurrenceRefs(20, 20)}},
|
||||
} {
|
||||
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: occurrences}}
|
||||
if result, err := newExtractor(t, client, references).Extract(context.Background(), req); err == nil || result.Value.Occurrences != nil || !strings.Contains(err.Error(), "location selector") {
|
||||
if result, err := newExtractor(t, client, references).Extract(context.Background(), req); err == nil || result.Value.Occurrences != nil {
|
||||
t.Fatalf("Extract() = %#v, %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -125,7 +121,7 @@ func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testin
|
||||
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{Name: selector.name, RegistryRefs: selector.refs, Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)}}}}
|
||||
duplicateRequest := extractionRequest()
|
||||
duplicateRequest.References = duplicateReferences
|
||||
if _, err := newExtractor(t, client, duplicateReferences).Extract(context.Background(), duplicateRequest); err == nil || !strings.Contains(err.Error(), "location selector") {
|
||||
if result, err := newExtractor(t, client, duplicateReferences).Extract(context.Background(), duplicateRequest); err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
|
||||
t.Fatalf("Extract(%#v) error = %v", selector, err)
|
||||
}
|
||||
}
|
||||
@@ -163,28 +159,25 @@ func TestExtractResolvesGeneratedRegistryAtOperationTimeAndDoesNotMutateResponse
|
||||
if err != nil || result.Value.Occurrences[0].Name != "The Mill" {
|
||||
t.Fatalf("Extract() = %#v, %v", result, err)
|
||||
}
|
||||
if input := client.requests[0].Inputs[LocationRegistryReferenceSlot]; strings.Contains(string(input.Content), location.ID) || input.OriginURI != "" {
|
||||
t.Fatalf("generated registry prompt input = %#v", input)
|
||||
}
|
||||
if _, ok := extractor.ManifestMetadata()["location_registry_digest"]; ok {
|
||||
t.Fatalf("operation registry leaked into static metadata: %#v", extractor.ManifestMetadata())
|
||||
if strings.Contains(string(renderedInput(t, client.requests[0], LocationRegistryReferenceSlot)), location.ID) {
|
||||
t.Fatal("generated registry exposed a durable ID")
|
||||
}
|
||||
if client.response.Occurrences[0].SourceRefs[0].StartUnitID != 30 {
|
||||
t.Fatalf("model response mutated: %#v", client.response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorContractsMetadataAndFailures(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
func TestExtractorContractsDependenciesAndFailures(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v", err)
|
||||
}
|
||||
if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{LocationRegistryReferenceSlot: {
|
||||
Items: []contracts.ReferenceItem{{SlotName: LocationRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"registry evidence"}`)}},
|
||||
}}}
|
||||
if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare location registry") || strings.Contains(err.Error(), "registry evidence") {
|
||||
if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare location registry") || strings.Contains(err.Error(), "registry evidence") {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
|
||||
@@ -197,11 +190,11 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
extractor *Extractor
|
||||
req contracts.TypedExtractionRequest
|
||||
req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]
|
||||
want string
|
||||
}{
|
||||
{"nil extractor", nilExtractor, req, "extractor"},
|
||||
{"nil client", &Extractor{}, req, "LLM client"},
|
||||
{"nil client", &Extractor{}, req, "native engine"},
|
||||
{"invalid request", extractor, mismatchedSourceInputRequest(req), "must match chunk"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -215,7 +208,7 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) {
|
||||
}
|
||||
|
||||
spec := ModuleSpec()
|
||||
if spec.Key != Key || spec.Stage != pipeline.StageExtract || spec.ExecutionClass != contracts.ExecutionClassLLMBacked || spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || spec.ArtifactKind != dnd.LocationOccurrenceListKind {
|
||||
if spec.Key != Key || spec.Stage != pipeline.StageExtract || spec.ExecutionClass != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(spec.Prompts, []string{PromptID, CorrectionPromptID}) || spec.ArtifactKind != dnd.LocationOccurrenceListKind {
|
||||
t.Fatalf("ModuleSpec() = %#v", spec)
|
||||
}
|
||||
var slot contracts.ReferenceSlot
|
||||
@@ -237,24 +230,6 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) {
|
||||
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
|
||||
t.Fatal("DecodeOptions() accepted unknown options")
|
||||
}
|
||||
metadata := newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata()
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256", "location_registry_digest"} {
|
||||
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v", key, metadata[key])
|
||||
}
|
||||
}
|
||||
if got := newExtractor(t, &fakeOccurrencesLLMClient{}, references).CheckpointFingerprints(); len(got) != 4 || got[3].Name != "location_registry" || got[3].Value != locationRegistryIdentityDigest(t, references) {
|
||||
t.Fatalf("fingerprints = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func locationRegistryIdentityDigest(t *testing.T, references contracts.ReferenceSet) string {
|
||||
t.Helper()
|
||||
resolver, err := locationregistry.NewResolver(references)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resolver.Seeded().IdentityDigest()
|
||||
}
|
||||
|
||||
func locationRegistry(t *testing.T, names ...string) dnd.LocationRegistry {
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package locationoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences"
|
||||
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationoccurrences/source_refs"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const unresolvedOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
|
||||
const knownOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
|
||||
const invalidCitation = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||||
|
||||
func operationReferences(t *testing.T) (contracts.ReferenceSet, string) {
|
||||
locations := locationRegistry(t, "The Tavern", "The Tavern")
|
||||
return registryReferences(t, locations), locations.Locations[1].ID
|
||||
}
|
||||
|
||||
func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationOccurrenceList], value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
validator := refs.New(refs.Options{})
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil
|
||||
}
|
||||
|
||||
func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) {
|
||||
for _, first := range []string{unresolvedOccurrence, invalidCitation} {
|
||||
request := extractionRequest()
|
||||
references, expectedID := operationReferences(t)
|
||||
request.References = references
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
// Accepted work consuming an ineligible generated reference stays ineligible.
|
||||
request.Operation.ReuseEligible = false
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return sourceAssessment(ctx, request, value)
|
||||
}
|
||||
client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}}
|
||||
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
|
||||
if len(client.requests) == 1 {
|
||||
request.References.Slots[LocationRegistryReferenceSlot].Items[0].Content[0] = '!'
|
||||
}
|
||||
}
|
||||
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
expectedAssessments := 1
|
||||
if first == invalidCitation {
|
||||
expectedAssessments = 2
|
||||
}
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||||
len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 {
|
||||
t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||||
}
|
||||
occurrence := result.Value.Occurrences[0]
|
||||
if occurrence.LocationID != expectedID || occurrence.Name != "The Tavern" || len(occurrence.SourceRefs) != 1 ||
|
||||
occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 {
|
||||
t.Fatalf("identity/evidence=%#v", occurrence)
|
||||
}
|
||||
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("durable occurrence=%v", err)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native selection lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
|
||||
t.Fatal("fixed native context/history changed")
|
||||
}
|
||||
input := string(renderedInput(t, client.requests[1], LocationRegistryReferenceSlot))
|
||||
if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) {
|
||||
t.Fatal("registry projection exposed an opaque identity")
|
||||
}
|
||||
if !reflect.DeepEqual(renderedInput(t, client.requests[0], LocationRegistryReferenceSlot), renderedInput(t, client.requests[1], LocationRegistryReferenceSlot)) {
|
||||
t.Fatal("registry projection was regenerated")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
response string
|
||||
errors []error
|
||||
wantAssessments, wantRuns int
|
||||
wantError bool
|
||||
}{
|
||||
{"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true},
|
||||
{"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false},
|
||||
{"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false},
|
||||
{"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
|
||||
{"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false},
|
||||
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.References, _ = operationReferences(t)
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments, prepared := 0, 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return sourceAssessment(ctx, request, value)
|
||||
}
|
||||
client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors}
|
||||
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
|
||||
t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" {
|
||||
t.Fatal("strict failure published an outcome")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
|
||||
t.Fatalf("unqualified fallback=%#v", result)
|
||||
}
|
||||
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("empty occurrence shape=%v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
request := extractionRequest()
|
||||
request.References, _ = operationReferences(t)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package locationoccurrences
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: PromptID,
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "location-registry.md", Path: "prompts/location-registry.md"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-references.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/location-occurrences")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope location-occurrence assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare location-occurrence prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope location-occurrence schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,53 +0,0 @@
|
||||
package locationoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesLocationOccurrencePrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"location-occurrences-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-occurrences-test",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-occurrence-transcript"}]}`), "players": promptkit.Inline("location-occurrence-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
|
||||
"location_registry": promptkit.Inline(`{"locations":[{"name":"location-occurrence-registry","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry context"}]}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_location_occurrences_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
var registryMessage string
|
||||
for _, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, "contextual location registry") {
|
||||
registryMessage = message.Content
|
||||
}
|
||||
}
|
||||
if !strings.Contains(registryMessage, "location-occurrence-registry") || !strings.Contains(registryMessage, `"registry_refs":[{"start_unit_id":7,"end_unit_id":7}]`) || strings.Contains(registryMessage, "source_id") || strings.Contains(registryMessage, "location:sha256") {
|
||||
t.Fatalf("rendered prompt did not preserve source-free registry grounding: %s", registryMessage)
|
||||
}
|
||||
content := make([]string, len(prepared.Messages))
|
||||
for index, message := range prepared.Messages {
|
||||
content[index] = message.Content
|
||||
}
|
||||
rendered := strings.Join(content, "\n")
|
||||
policy := strings.ReplaceAll(rendered, "\n", " ")
|
||||
if !strings.Contains(policy, "context supports that coreference") || !strings.Contains(policy, "must not create a registry location") || !strings.Contains(policy, "provenance must never replace current-chunk evidence") || !strings.Contains(policy, "complete, ordered") {
|
||||
t.Fatalf("rendered prompt = %q, want contextual coreference without registry-derived evidence", rendered)
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package locationoccurrences
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.location_occurrences"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_location_occurrences_llm")
|
||||
ResponseSchemaID = "notarius.dnd.location_occurrences.llm"
|
||||
ResponseSchemaName = "notarius_dnd_location_occurrences_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_location_occurrences_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package locationoccurrences
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestResponseSchemaRestrictsPrivateOccurrenceStructureAndKinds(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v", schema)
|
||||
}
|
||||
valid := map[string]any{"occurrences": []any{map[string]any{
|
||||
"name": "The Mill", "registry_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, "kind": "visited",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
|
||||
}}}
|
||||
if err := validateSchema(t, valid, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid response rejected: %v", err)
|
||||
}
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(value map[string]any) { delete(value, "registry_refs") },
|
||||
func(value map[string]any) { value["kind"] = "other" },
|
||||
func(value map[string]any) { value["unexpected"] = true },
|
||||
func(value map[string]any) { value["registry_refs"].([]any)[0].(map[string]any)["start_unit_id"] = 0 },
|
||||
func(value map[string]any) {
|
||||
value["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later"
|
||||
},
|
||||
} {
|
||||
candidate := cloneCandidate(t, valid)
|
||||
mutate(candidate["occurrences"].([]any)[0].(map[string]any))
|
||||
if err := validateSchema(t, candidate, schema.JSONSchema); err == nil {
|
||||
t.Fatal("schema accepted structurally invalid response")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("schema = %s, %v", second.JSONSchema, err)
|
||||
}
|
||||
if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID {
|
||||
t.Fatalf("diagnostics = %#v", diagnostics)
|
||||
} else if _, ok := diagnostics["json_schema"]; ok {
|
||||
t.Fatalf("diagnostics leaked schema content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneCandidate(t *testing.T, value map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var clone map[string]any
|
||||
if err := json.Unmarshal(content, &clone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func validateSchema(t *testing.T, value map[string]any, schemaContent []byte) error {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(content))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", document); err != nil {
|
||||
return err
|
||||
}
|
||||
schema, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schema.Validate(instance)
|
||||
}
|
||||
@@ -3,14 +3,19 @@ package locationoccurrences
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
func extractionRequest() contracts.TypedExtractionRequest[dnd.LocationOccurrenceList] {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-occurrences:chunk:0", SourceID: doc.ID, Index: 0,
|
||||
@@ -18,10 +23,13 @@ func extractionRequest() contracts.TypedExtractionRequest {
|
||||
Content: []byte(`{"units":[10,20,30]}`), MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
return contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]{
|
||||
Source: doc, Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-occurrences.json"),
|
||||
SessionID: "occurrence-session", LLMProfile: "occurrence-profile",
|
||||
Operation: contracts.ProducerOperation[dnd.LocationOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,51 +45,95 @@ func occurrenceRefs(start, end int) []occurrenceSourceRefResponse {
|
||||
return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: occurrence-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: occurrence-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name]
|
||||
for _, message := range request.Prompt.Messages {
|
||||
start := strings.Index(message.Content, "{")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
content := []byte(message.Content[start:])
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]) contracts.TypedExtractionRequest[dnd.LocationOccurrenceList] {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "")
|
||||
return req
|
||||
}
|
||||
|
||||
type fakeOccurrencesLLMClient struct {
|
||||
response extractionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response extractionResponse
|
||||
responses []string
|
||||
raw string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
func (client *fakeOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
return nil, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
content := client.raw
|
||||
if index < len(client.responses) {
|
||||
content = client.responses[index]
|
||||
}
|
||||
if err := json.Unmarshal(content, target); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
if content == "" {
|
||||
encoded, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = string(encoded)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
return req
|
||||
return &promptkit.GenerateResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
@@ -2,8 +2,12 @@ package npcoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
@@ -12,10 +16,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "dnd/npc-occurrences"
|
||||
mappingPolicy = "dnd.npc_occurrences.extract_mapping.v4"
|
||||
)
|
||||
const Key = "dnd/npc-occurrences"
|
||||
const PromptID = "notarius/dnd.npc_occurrences"
|
||||
const CorrectionPromptID = "notarius/dnd.npc_occurrences.repair"
|
||||
|
||||
const (
|
||||
NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
|
||||
@@ -53,21 +56,17 @@ func referenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.NPCOccurrenceList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
npcResolver *npcregistry.Resolver
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
engine *promptkit.Engine
|
||||
npcResolver *npcregistry.Resolver
|
||||
}
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
@@ -80,68 +79,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("prepare NPC registry prompt input: %w", err)
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{
|
||||
llm: llmClient,
|
||||
npcResolver: npcResolver,
|
||||
promptSHA: promptSHA,
|
||||
responseSchemaSHA: responseSchema.SHA256,
|
||||
}, nil
|
||||
return &Extractor{engine: engine, npcResolver: npcResolver}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
metadata := map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"mapping_policy": mappingPolicy,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
}
|
||||
seeded := e.npcResolver.Seeded()
|
||||
if seeded.Bound() {
|
||||
metadata["npc_registry_digest"] = seeded.Digest()
|
||||
metadata["npc_count"] = seeded.Count()
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
seeded := e.npcResolver.Seeded()
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
{Name: "npc_registry", Value: seeded.IdentityDigest()},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCOccurrenceList], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]) (contracts.TypedExtractionResult[dnd.NPCOccurrenceList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||
if err != nil {
|
||||
@@ -156,44 +106,50 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("NPC registry reference is required")
|
||||
}
|
||||
|
||||
var response extractionResponse
|
||||
inputs := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction,
|
||||
Inputs: inputs,
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
sourceID := req.Source.ID
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
materials[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.NPCOccurrenceList, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, sourceID)
|
||||
value, err := canonicalOccurrenceList(response, sourceID, npcRegistry)
|
||||
if err != nil {
|
||||
return nil, []string{"Resolve every occurrence to an exact canonical name from the supplied NPC registry. Omit occurrences for identities absent from that registry; do not invent an identity. Return the complete replacement occurrence list."}, nil
|
||||
}
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(context.Context, *dnd.NPCOccurrenceList) (dnd.NPCOccurrenceList, error) {
|
||||
return dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err)
|
||||
}
|
||||
canonicalizeResponse(&response, order, req.Source.ID)
|
||||
value, err := canonicalOccurrenceList(response, req.Source.ID, npcRegistry)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("resolve NPC names against registry: %w", err)
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{Value: value, ModelCandidate: candidate}, nil
|
||||
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.NPCOccurrenceListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.NPCOccurrenceListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,7 +159,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package npcoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -105,10 +104,10 @@ func TestExtractUsesDocumentOrderForReferencesAndOccurrences(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNewRequiresLLMAndRejectsAmbiguousReferenceSets(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v", err)
|
||||
}
|
||||
if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -124,25 +123,19 @@ func TestExtractUsesNamesOnlyRegistryAndCurrentTranscriptEvidence(t *testing.T)
|
||||
t.Fatalf("Extract() error = %v", err)
|
||||
}
|
||||
request := client.requests[0]
|
||||
registry := request.Inputs[NPCRegistryReferenceSlot]
|
||||
if registry.Name != NPCRegistryReferenceSlot || registry.MediaType != "application/json" || string(registry.Content) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` {
|
||||
t.Fatalf("registry prompt input = %#v, want names-only projection", registry)
|
||||
registry := renderedInput(t, request, NPCRegistryReferenceSlot)
|
||||
if string(registry) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` {
|
||||
t.Fatalf("names-only registry=%s", registry)
|
||||
}
|
||||
for _, forbidden := range []string{"npc:sha256:", "other-session", "start_unit_id"} {
|
||||
if strings.Contains(string(registry.Content), forbidden) {
|
||||
t.Fatalf("registry prompt input leaked %q: %s", forbidden, registry.Content)
|
||||
if strings.Contains(string(registry), forbidden) {
|
||||
t.Fatalf("registry projection exposed %q", forbidden)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(request.Inputs["transcript"].Content), "other-session") {
|
||||
t.Fatal("transcript input contains registry evidence")
|
||||
}
|
||||
metadata, err := json.Marshal(newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(metadata), "Mira Thorn") || strings.Contains(string(metadata), "other-session") {
|
||||
t.Fatalf("manifest metadata leaked registry content: %s", metadata)
|
||||
if strings.Contains(string(renderedInput(t, request, "transcript")), "other-session") {
|
||||
t.Fatal("registry provenance became occurrence evidence")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) {
|
||||
@@ -160,7 +153,7 @@ func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) {
|
||||
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private source"}`)}},
|
||||
},
|
||||
}}
|
||||
if _, err := New(client, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") {
|
||||
if _, err := New(newExtractor(t, client).engine, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") {
|
||||
t.Fatalf("New() error = %v, want content-safe malformed registry error", err)
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
@@ -177,7 +170,7 @@ func TestExtractRejectsUnknownNamesWithoutPartialResult(t *testing.T) {
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") {
|
||||
if err == nil || result.Outcome.Disposition != "" {
|
||||
t.Fatalf("Extract() error = %v, want unknown name failure", err)
|
||||
}
|
||||
if len(result.Value.Occurrences) != 0 {
|
||||
@@ -211,13 +204,10 @@ func TestExtractResolvesGeneratedRegistryAtOperationTime(t *testing.T) {
|
||||
if _, err := extractor.Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v", err)
|
||||
}
|
||||
if input := client.requests[0].Inputs[NPCRegistryReferenceSlot]; string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" {
|
||||
t.Fatalf("generated registry prompt input = %#v", input)
|
||||
}
|
||||
metadata := extractor.ManifestMetadata()
|
||||
if _, ok := metadata["npc_registry_digest"]; ok {
|
||||
t.Fatalf("operation-time registry leaked into static metadata: %#v", metadata)
|
||||
if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` {
|
||||
t.Fatalf("generated registry=%s", input)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestExtractAcceptsEmptyBoundRegistryAndEmptyResponse(t *testing.T) {
|
||||
@@ -253,7 +243,7 @@ func TestExtractRejectsNonemptyResponseForEmptyBoundRegistry(t *testing.T) {
|
||||
req := extractionRequest()
|
||||
req.References = references
|
||||
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") || len(result.Value.Occurrences) != 0 {
|
||||
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
|
||||
t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err)
|
||||
}
|
||||
}
|
||||
@@ -268,11 +258,11 @@ func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) {
|
||||
name string
|
||||
extractor *Extractor
|
||||
ctx context.Context
|
||||
req contracts.TypedExtractionRequest
|
||||
req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]
|
||||
want string
|
||||
}{
|
||||
{"nil extractor", nilExtractor, context.Background(), valid, "extractor"},
|
||||
{"nil LLM client", &Extractor{}, context.Background(), valid, "LLM client"},
|
||||
{"nil native engine", &Extractor{}, context.Background(), valid, "native engine"},
|
||||
{"wrapped preflight failure", extractor, context.Background(), mismatchedSourceInputRequest(valid), "must match chunk"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -286,9 +276,9 @@ func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) {
|
||||
func TestModuleSpecRegistrationAndNativeDependencies(t *testing.T) {
|
||||
got := ModuleSpec()
|
||||
if got.Key != Key || got.Stage != pipeline.StageExtract || got.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) {
|
||||
if got.Key != Key || got.Stage != pipeline.StageExtract || !reflect.DeepEqual(got.Prompts, []string{PromptID, CorrectionPromptID}) || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) {
|
||||
t.Fatalf("ModuleSpec() = %#v", got)
|
||||
}
|
||||
var registrySlot contracts.ReferenceSlot
|
||||
@@ -300,6 +290,10 @@ func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) {
|
||||
if !registrySlot.Required || !reflect.DeepEqual(registrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCRegistryKind}) || registrySlot.MaxBytes != NPCRegistryMaxBytes {
|
||||
t.Fatalf("NPC registry slot = %#v", registrySlot)
|
||||
}
|
||||
got.Prompts[0] = "changed"
|
||||
if ModuleSpec().Prompts[0] == "changed" {
|
||||
t.Fatal("prompt dependencies alias caller memory")
|
||||
}
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
if ModuleSpec().ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
|
||||
t.Fatal("ModuleSpec() returned mutable reference slots")
|
||||
@@ -320,145 +314,4 @@ func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) {
|
||||
t.Fatal("DecodeOptions() accepted unknown option")
|
||||
}
|
||||
|
||||
references := requiredRegistryReferences(t, "Mira Thorn")
|
||||
extractor := newExtractor(t, &fakeOccurrencesLLMClient{}, references)
|
||||
metadata := extractor.ManifestMetadata()
|
||||
for key, want := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
} {
|
||||
if metadata[key] != want {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256", "npc_registry_digest"} {
|
||||
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key])
|
||||
}
|
||||
}
|
||||
wantFingerprints := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "npc_registry": {}}
|
||||
for _, fingerprint := range extractor.CheckpointFingerprints() {
|
||||
delete(wantFingerprints, fingerprint.Name)
|
||||
}
|
||||
if len(wantFingerprints) != 0 {
|
||||
t.Fatalf("missing fingerprints = %#v", wantFingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0,
|
||||
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30},
|
||||
Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
Source: doc, Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123", LLMProfile: "profile-npc-occurrences",
|
||||
}
|
||||
}
|
||||
|
||||
func sourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."},
|
||||
{ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."},
|
||||
{ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."},
|
||||
{ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."},
|
||||
{ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."},
|
||||
{ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet {
|
||||
t.Helper()
|
||||
npcs := make([]dnd.NPC, len(names))
|
||||
for index, name := range names {
|
||||
npcs[index] = dnd.NPC{
|
||||
ID: identity.DeriveID(name), Name: name,
|
||||
SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}},
|
||||
}
|
||||
}
|
||||
content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: npcs})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
NPCRegistryReferenceSlot: {
|
||||
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
||||
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func occurrenceRefs(start, end int) []occurrenceSourceRefResponse {
|
||||
return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
|
||||
}
|
||||
|
||||
func occurrenceNames(value dnd.NPCOccurrenceList) []string {
|
||||
names := make([]string, len(value.Occurrences))
|
||||
for index, occurrence := range value.Occurrences {
|
||||
names[index] = occurrence.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func occurrenceKinds(value dnd.NPCOccurrenceList) []dnd.NPCOccurrenceKind {
|
||||
kinds := make([]dnd.NPCOccurrenceKind, len(value.Occurrences))
|
||||
for index, occurrence := range value.Occurrences {
|
||||
kinds[index] = occurrence.Kind
|
||||
}
|
||||
return kinds
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "")
|
||||
return req
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v", err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
type fakeOccurrencesLLMClient struct {
|
||||
response extractionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (client *fakeOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
}
|
||||
*target = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
return req
|
||||
}
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package npcoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcoccurrences/source_refs"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const unresolvedOccurrence = `{"occurrences":[{"name":"Unknown NPC","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
|
||||
const knownOccurrence = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
|
||||
const invalidCitation = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
|
||||
|
||||
func operationReferences(t *testing.T) (contracts.ReferenceSet, string) {
|
||||
return requiredRegistryReferences(t, "Mira Thorn"), identity.DeriveID("Mira Thorn")
|
||||
}
|
||||
|
||||
func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.NPCOccurrenceList], value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
validator := refs.New(refs.Options{})
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.NPCOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil
|
||||
}
|
||||
|
||||
func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) {
|
||||
for _, first := range []string{unresolvedOccurrence, invalidCitation} {
|
||||
request := extractionRequest()
|
||||
references, expectedID := operationReferences(t)
|
||||
request.References = references
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
// Accepted work consuming an ineligible generated reference stays ineligible.
|
||||
request.Operation.ReuseEligible = false
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return sourceAssessment(ctx, request, value)
|
||||
}
|
||||
client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}}
|
||||
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
|
||||
if len(client.requests) == 1 {
|
||||
request.References.Slots[NPCRegistryReferenceSlot].Items[0].Content[0] = '!'
|
||||
}
|
||||
}
|
||||
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
expectedAssessments := 1
|
||||
if first == invalidCitation {
|
||||
expectedAssessments = 2
|
||||
}
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||||
len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 {
|
||||
t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared))
|
||||
}
|
||||
occurrence := result.Value.Occurrences[0]
|
||||
if occurrence.NPCID != expectedID || occurrence.Name != "Mira Thorn" || len(occurrence.SourceRefs) != 1 ||
|
||||
occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 {
|
||||
t.Fatalf("identity/evidence=%#v", occurrence)
|
||||
}
|
||||
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("durable occurrence=%v", err)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id := PromptID
|
||||
if i > 0 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("native selection lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
|
||||
t.Fatal("fixed native context/history changed")
|
||||
}
|
||||
input := string(renderedInput(t, client.requests[1], NPCRegistryReferenceSlot))
|
||||
if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) {
|
||||
t.Fatal("registry projection exposed an opaque identity")
|
||||
}
|
||||
if !reflect.DeepEqual(renderedInput(t, client.requests[0], NPCRegistryReferenceSlot), renderedInput(t, client.requests[1], NPCRegistryReferenceSlot)) {
|
||||
t.Fatal("registry projection was regenerated")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
response string
|
||||
errors []error
|
||||
wantAssessments, wantRuns int
|
||||
wantError bool
|
||||
}{
|
||||
{"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true},
|
||||
{"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false},
|
||||
{"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false},
|
||||
{"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
|
||||
{"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false},
|
||||
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.References, _ = operationReferences(t)
|
||||
request.Operation.Controls.Mode = test.mode
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments, prepared := 0, 0
|
||||
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return sourceAssessment(ctx, request, value)
|
||||
}
|
||||
client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors}
|
||||
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
|
||||
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
|
||||
t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared)
|
||||
}
|
||||
if test.wantError {
|
||||
if result.Outcome.Disposition != "" {
|
||||
t.Fatal("strict failure published an outcome")
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
|
||||
t.Fatalf("unqualified fallback=%#v", result)
|
||||
}
|
||||
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatalf("empty occurrence shape=%v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
request := extractionRequest()
|
||||
request.References, _ = operationReferences(t)
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package npcoccurrences
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.npc_occurrences",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-extraction-evidence.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-npc-registry.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/npc-occurrences")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope NPC-occurrence assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare NPC-occurrence prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope NPC-occurrence schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,70 +0,0 @@
|
||||
package npcoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsAndPrepareOccurrencePrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schemaFS, err := registry.SchemaFS()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := fs.ReadFile(schemaFS, "dnd_npc_occurrences_llm.v1.json"); err != nil {
|
||||
t.Fatalf("response schema asset: %v", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"npc-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-occurrences-test-model"),
|
||||
})
|
||||
transcript := `{"units":[{"sentinel":"occurrence-transcript"}]}`
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-occurrences-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
|
||||
"players": promptkit.Inline("occurrence-player"),
|
||||
"party": promptkit.Inline("Mira: ranger"),
|
||||
"glossary": promptkit.Inline("Greencloak: title"),
|
||||
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"occurrence-npc"}]}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_occurrences_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
|
||||
hash, err := promptAssetMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("promptAssetMetadata() = %q, %v", hash, err)
|
||||
}
|
||||
metadata := newExtractor(t, &fakeOccurrencesLLMClient{}).ManifestMetadata()
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_npc_occurrences_llm.v1.json"} {
|
||||
if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) {
|
||||
t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mapValues(values map[string]any) []string {
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
if text, ok := value.(string); ok {
|
||||
out = append(out, text)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package npcoccurrences
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.npc_occurrences"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npc_occurrences_llm")
|
||||
ResponseSchemaID = "notarius.dnd.npc_occurrences.llm"
|
||||
ResponseSchemaName = "notarius_dnd_npc_occurrences_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_npc_occurrences_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package npcoccurrences
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v", schema)
|
||||
}
|
||||
valid := validOccurrenceResponse()
|
||||
content, err := json.Marshal(valid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid response rejected: %v", err)
|
||||
}
|
||||
|
||||
semanticCandidate := validOccurrenceResponse()
|
||||
occurrence := semanticCandidate["occurrences"].([]any)[0].(map[string]any)
|
||||
occurrence["name"] = ""
|
||||
ref := occurrence["source_refs"].([]any)[0].(map[string]any)
|
||||
ref["start_unit_id"] = 0
|
||||
ref["end_unit_id"] = -1
|
||||
content, err = json.Marshal(semanticCandidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("schema rejected non-categorical validator-owned semantics: %v", err)
|
||||
}
|
||||
|
||||
for _, mutate := range []func(map[string]any){
|
||||
func(record map[string]any) { delete(record, "name") },
|
||||
func(record map[string]any) { record["kind"] = 1 },
|
||||
func(record map[string]any) { record["kind"] = "unsupported" },
|
||||
func(record map[string]any) { record["npc_id"] = "npc:sha256:opaque" },
|
||||
func(record map[string]any) { record["unexpected"] = true },
|
||||
func(record map[string]any) {
|
||||
record["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later"
|
||||
},
|
||||
} {
|
||||
candidate := validOccurrenceResponse()
|
||||
mutate(candidate["occurrences"].([]any)[0].(map[string]any))
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
|
||||
t.Fatal("schema accepted structurally invalid response")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v", second.JSONSchema, err)
|
||||
}
|
||||
if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID {
|
||||
t.Fatalf("diagnostics = %#v", diagnostics)
|
||||
} else if _, ok := diagnostics["json_schema"]; ok {
|
||||
t.Fatalf("diagnostics leak schema content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{string(dnd.NPCOccurrenceKindMentioned), string(dnd.NPCOccurrenceKindNoncombatPresence), string(dnd.NPCOccurrenceKindDialogue), string(dnd.NPCOccurrenceKindCombatAlly), string(dnd.NPCOccurrenceKindCombatOpponent), string(dnd.NPCOccurrenceKindOther)} {
|
||||
candidate := validOccurrenceResponse()
|
||||
candidate["occurrences"].([]any)[0].(map[string]any)["kind"] = kind
|
||||
content, err := json.Marshal(candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("supported NPC-occurrence kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validOccurrenceResponse() map[string]any {
|
||||
return map[string]any{"occurrences": []any{map[string]any{
|
||||
"name": "Mira Thorn", "kind": "dialogue",
|
||||
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
|
||||
}}}
|
||||
}
|
||||
|
||||
func validateJSONSchema(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||
return err
|
||||
}
|
||||
schema, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return schema.Validate(instance)
|
||||
}
|
||||
184
internal/modules/dnd/extract/npcoccurrences/test_helpers_test.go
Normal file
184
internal/modules/dnd/extract/npcoccurrences/test_helpers_test.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package npcoccurrences
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest[dnd.NPCOccurrenceList] {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0,
|
||||
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30},
|
||||
Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
}
|
||||
return contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]{
|
||||
Source: doc, Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123", LLMProfile: "profile-npc-occurrences",
|
||||
Operation: contracts.ProducerOperation[dnd.NPCOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func sourceDocument() *source.SourceDocument {
|
||||
return &source.SourceDocument{
|
||||
ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."},
|
||||
{ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."},
|
||||
{ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."},
|
||||
{ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."},
|
||||
{ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."},
|
||||
{ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet {
|
||||
t.Helper()
|
||||
npcs := make([]dnd.NPC, len(names))
|
||||
for index, name := range names {
|
||||
npcs[index] = dnd.NPC{
|
||||
ID: identity.DeriveID(name), Name: name,
|
||||
SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}},
|
||||
}
|
||||
}
|
||||
content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: npcs})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
NPCRegistryReferenceSlot: {
|
||||
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
|
||||
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func occurrenceRefs(start, end int) []occurrenceSourceRefResponse {
|
||||
return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
|
||||
}
|
||||
|
||||
func occurrenceNames(value dnd.NPCOccurrenceList) []string {
|
||||
names := make([]string, len(value.Occurrences))
|
||||
for index, occurrence := range value.Occurrences {
|
||||
names[index] = occurrence.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func occurrenceKinds(value dnd.NPCOccurrenceList) []dnd.NPCOccurrenceKind {
|
||||
kinds := make([]dnd.NPCOccurrenceKind, len(value.Occurrences))
|
||||
for index, occurrence := range value.Occurrences {
|
||||
kinds[index] = occurrence.Kind
|
||||
}
|
||||
return kinds
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]) contracts.TypedExtractionRequest[dnd.NPCOccurrenceList] {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "")
|
||||
return req
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: profile-npc-occurrences
|
||||
endpoint: https://example.invalid/v1
|
||||
model: occurrence-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name]
|
||||
for _, message := range request.Prompt.Messages {
|
||||
start := strings.Index(message.Content, "{")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
content := []byte(message.Content[start:])
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeOccurrencesLLMClient struct {
|
||||
response extractionResponse
|
||||
responses []string
|
||||
raw string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return nil, client.err
|
||||
}
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
content := client.raw
|
||||
if index < len(client.responses) {
|
||||
content = client.responses[index]
|
||||
}
|
||||
if content == "" {
|
||||
encoded, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = string(encoded)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: content}, nil
|
||||
}
|
||||
@@ -46,7 +46,7 @@ func TestValidatorRegisters(t *testing.T) {
|
||||
if err := Register(registry); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, ok := registry.Spec(Key); !ok || got != Spec() || got.ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
if got, ok := registry.Spec(Key); !ok || !reflect.DeepEqual(got, Spec()) || got.ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
t.Fatalf("registered spec = %#v, ok = %t", got, ok)
|
||||
}
|
||||
if got := New(Options{}).CheckpointFingerprints(); len(got) != 1 || got[0].Value != policy {
|
||||
|
||||
Reference in New Issue
Block a user