Migrate NPC and location occurrences to native correction and explicit empty fallbacks

This commit is contained in:
2026-11-22 01:03:06 +00:00
parent 303d6589c5
commit 2c1d727232
23 changed files with 948 additions and 979 deletions

View File

@@ -3,8 +3,12 @@ package locationoccurrences
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -13,10 +17,9 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const (
Key = "dnd/location-occurrences"
mappingPolicy = "dnd.location_occurrences.extract_mapping.v3"
)
const Key = "dnd/location-occurrences"
const PromptID = "notarius/dnd.location_occurrences"
const CorrectionPromptID = "notarius/dnd.location_occurrences.repair"
const (
LocationRegistryReferenceSlot = locationregistry.ReferenceSlot
@@ -48,21 +51,17 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.LocationOccurrenceList] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
locationResolver *locationregistry.Resolver
promptSHA string
responseSchemaSHA string
engine *promptkit.Engine
locationResolver *locationregistry.Resolver
}
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
@@ -75,62 +74,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare location registry prompt input: %w", err)
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{llm: llmClient, locationResolver: locationResolver, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
return &Extractor{engine: engine, locationResolver: locationResolver}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"mapping_policy": mappingPolicy,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
}
seeded := e.locationResolver.Seeded()
if seeded.Bound() {
metadata["location_registry_digest"] = seeded.Digest()
metadata["location_count"] = seeded.Count()
}
return metadata
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "location_registry", Value: e.locationResolver.Seeded().IdentityDigest()},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.LocationOccurrenceList], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]) (contracts.TypedExtractionResult[dnd.LocationOccurrenceList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("native engine must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
@@ -148,33 +104,46 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("prepare location grounding: %w", err)
}
var response extractionResponse
inputs := shared.PromptInputs(sourceInput, req.References)
inputs[LocationRegistryReferenceSlot] = grounding.PromptInput()
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction, Inputs: inputs,
}, &response)
if err != nil {
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
order := shared.NewSourceRefOrder(req.Source)
sourceID := req.Source.ID
materials := shared.PromptInputs(sourceInput, req.References)
materials[LocationRegistryReferenceSlot] = grounding.PromptInput()
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationOccurrenceList, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
value, err := canonicalOccurrenceList(response, order, sourceID, grounding)
if err != nil {
return nil, []string{"Resolve every occurrence to exactly one supplied location selector. Copy its canonical name and complete ordered registry_refs array; use an empty registry_refs array for a unique name. Omit unsupported or ambiguous selections. Keep occurrence evidence in source_refs separate from registry context, and return the complete replacement occurrence list."}, nil
}
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(context.Context, *dnd.LocationOccurrenceList) (dnd.LocationOccurrenceList, error) {
return dnd.LocationOccurrenceList{Occurrences: []dnd.LocationOccurrence{}}, nil
})
if err != nil {
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err)
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err)
}
occurrences, err := canonicalOccurrenceList(response, shared.NewSourceRefOrder(req.Source), req.Source.ID, grounding)
if err != nil {
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{}, extractorErrorf("resolve location grounding: %w", err)
}
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{Value: occurrences, ModelCandidate: candidate}, nil
return contracts.TypedExtractionResult[dnd.LocationOccurrenceList]{Value: value, Outcome: outcome}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.LocationOccurrenceListKind, ReferenceSlots: referenceSlots(),
}
}
@@ -185,7 +154,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -2,7 +2,6 @@ package locationoccurrences
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
@@ -70,22 +69,19 @@ func TestExtractResolvesContextualSelectorsAndUsesCurrentTranscriptEvidenceOnly(
if occurrence := result.Value.Occurrences[0]; occurrence.LocationID != second.ID || occurrence.Name != second.Name || occurrence.SourceRefs[0].SourceID != req.Source.ID {
t.Fatalf("occurrence = %#v", occurrence)
}
input := client.requests[0].Inputs[LocationRegistryReferenceSlot]
if input.Name != LocationRegistryReferenceSlot || !strings.Contains(string(input.Content), `"registry_refs":[{"start_unit_id":20,"end_unit_id":20}]`) {
t.Fatalf("location prompt input = %#v", input)
input := renderedInput(t, client.requests[0], LocationRegistryReferenceSlot)
if !strings.Contains(string(input), `"registry_refs":[{"start_unit_id":20,"end_unit_id":20}]`) {
t.Fatalf("contextual registry=%s", input)
}
for _, forbidden := range []string{"source_refs", "source_id", first.ID, second.ID} {
if strings.Contains(string(input.Content), forbidden) {
t.Fatalf("location prompt leaked %q: %s", forbidden, input.Content)
if strings.Contains(string(input), forbidden) {
t.Fatalf("location projection exposed %q", forbidden)
}
}
if strings.Contains(string(client.requests[0].Inputs["transcript"].Content), "other-session") {
t.Fatal("transcript input contains registry evidence")
}
metadata, err := json.Marshal(newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata())
if err != nil || strings.Contains(string(metadata), "other-session") || strings.Contains(string(metadata), first.ID) {
t.Fatalf("manifest metadata = %s, %v", metadata, err)
if strings.Contains(string(renderedInput(t, client.requests[0], "transcript")), "other-session") {
t.Fatal("registry provenance became occurrence evidence")
}
}
func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testing.T) {
@@ -101,7 +97,7 @@ func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testin
{{Name: known.Name, RegistryRefs: []locationregistry.RegistryRef{}, Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)}, {Name: "Unknown", RegistryRefs: []locationregistry.RegistryRef{}, Kind: "mentioned", SourceRefs: occurrenceRefs(20, 20)}},
} {
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: occurrences}}
if result, err := newExtractor(t, client, references).Extract(context.Background(), req); err == nil || result.Value.Occurrences != nil || !strings.Contains(err.Error(), "location selector") {
if result, err := newExtractor(t, client, references).Extract(context.Background(), req); err == nil || result.Value.Occurrences != nil {
t.Fatalf("Extract() = %#v, %v", result, err)
}
}
@@ -125,7 +121,7 @@ func TestExtractRejectsUnknownMalformedOrMismatchedSelectorsAtomically(t *testin
client := &fakeOccurrencesLLMClient{response: extractionResponse{Occurrences: []occurrenceResponse{{Name: selector.name, RegistryRefs: selector.refs, Kind: "mentioned", SourceRefs: occurrenceRefs(10, 10)}}}}
duplicateRequest := extractionRequest()
duplicateRequest.References = duplicateReferences
if _, err := newExtractor(t, client, duplicateReferences).Extract(context.Background(), duplicateRequest); err == nil || !strings.Contains(err.Error(), "location selector") {
if result, err := newExtractor(t, client, duplicateReferences).Extract(context.Background(), duplicateRequest); err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract(%#v) error = %v", selector, err)
}
}
@@ -163,28 +159,25 @@ func TestExtractResolvesGeneratedRegistryAtOperationTimeAndDoesNotMutateResponse
if err != nil || result.Value.Occurrences[0].Name != "The Mill" {
t.Fatalf("Extract() = %#v, %v", result, err)
}
if input := client.requests[0].Inputs[LocationRegistryReferenceSlot]; strings.Contains(string(input.Content), location.ID) || input.OriginURI != "" {
t.Fatalf("generated registry prompt input = %#v", input)
}
if _, ok := extractor.ManifestMetadata()["location_registry_digest"]; ok {
t.Fatalf("operation registry leaked into static metadata: %#v", extractor.ManifestMetadata())
if strings.Contains(string(renderedInput(t, client.requests[0], LocationRegistryReferenceSlot)), location.ID) {
t.Fatal("generated registry exposed a durable ID")
}
if client.response.Occurrences[0].SourceRefs[0].StartUnitID != 30 {
t.Fatalf("model response mutated: %#v", client.response)
}
}
func TestExtractorContractsMetadataAndFailures(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
func TestExtractorContractsDependenciesAndFailures(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v", err)
}
if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
t.Fatalf("New() error = %v", err)
}
malformed := contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{LocationRegistryReferenceSlot: {
Items: []contracts.ReferenceItem{{SlotName: LocationRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"registry evidence"}`)}},
}}}
if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare location registry") || strings.Contains(err.Error(), "registry evidence") {
if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare location registry") || strings.Contains(err.Error(), "registry evidence") {
t.Fatalf("New() error = %v", err)
}
@@ -197,11 +190,11 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) {
for _, test := range []struct {
name string
extractor *Extractor
req contracts.TypedExtractionRequest
req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]
want string
}{
{"nil extractor", nilExtractor, req, "extractor"},
{"nil client", &Extractor{}, req, "LLM client"},
{"nil client", &Extractor{}, req, "native engine"},
{"invalid request", extractor, mismatchedSourceInputRequest(req), "must match chunk"},
} {
t.Run(test.name, func(t *testing.T) {
@@ -215,7 +208,7 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) {
}
spec := ModuleSpec()
if spec.Key != Key || spec.Stage != pipeline.StageExtract || spec.ExecutionClass != contracts.ExecutionClassLLMBacked || spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || spec.ArtifactKind != dnd.LocationOccurrenceListKind {
if spec.Key != Key || spec.Stage != pipeline.StageExtract || spec.ExecutionClass != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(spec.Prompts, []string{PromptID, CorrectionPromptID}) || spec.ArtifactKind != dnd.LocationOccurrenceListKind {
t.Fatalf("ModuleSpec() = %#v", spec)
}
var slot contracts.ReferenceSlot
@@ -237,24 +230,6 @@ func TestExtractorContractsMetadataAndFailures(t *testing.T) {
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
t.Fatal("DecodeOptions() accepted unknown options")
}
metadata := newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata()
for _, key := range []string{"prompt_sha256", "response_schema_sha256", "location_registry_digest"} {
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v", key, metadata[key])
}
}
if got := newExtractor(t, &fakeOccurrencesLLMClient{}, references).CheckpointFingerprints(); len(got) != 4 || got[3].Name != "location_registry" || got[3].Value != locationRegistryIdentityDigest(t, references) {
t.Fatalf("fingerprints = %#v", got)
}
}
func locationRegistryIdentityDigest(t *testing.T, references contracts.ReferenceSet) string {
t.Helper()
resolver, err := locationregistry.NewResolver(references)
if err != nil {
t.Fatal(err)
}
return resolver.Seeded().IdentityDigest()
}
func locationRegistry(t *testing.T, names ...string) dnd.LocationRegistry {

View File

@@ -0,0 +1,162 @@
package locationoccurrences
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences"
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationoccurrences/source_refs"
"go.maximumdirect.net/promptkit"
)
const unresolvedOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
const knownOccurrence = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
const invalidCitation = `{"occurrences":[{"name":"The Tavern","registry_refs":[{"start_unit_id":20,"end_unit_id":20}],"kind":"visited","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
func operationReferences(t *testing.T) (contracts.ReferenceSet, string) {
locations := locationRegistry(t, "The Tavern", "The Tavern")
return registryReferences(t, locations), locations.Locations[1].ID
}
func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationOccurrenceList], value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
validator := refs.New(refs.Options{})
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil
}
func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) {
for _, first := range []string{unresolvedOccurrence, invalidCitation} {
request := extractionRequest()
references, expectedID := operationReferences(t)
request.References = references
request.Operation.Controls.MaxAdditionalTurns = 1
// Accepted work consuming an ineligible generated reference stays ineligible.
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
assessments := 0
request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
assessments++
return sourceAssessment(ctx, request, value)
}
client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}}
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
if len(client.requests) == 1 {
request.References.Slots[LocationRegistryReferenceSlot].Items[0].Content[0] = '!'
}
}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
expectedAssessments := 1
if first == invalidCitation {
expectedAssessments = 2
}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 {
t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared))
}
occurrence := result.Value.Occurrences[0]
if occurrence.LocationID != expectedID || occurrence.Name != "The Tavern" || len(occurrence.SourceRefs) != 1 ||
occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 {
t.Fatalf("identity/evidence=%#v", occurrence)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatalf("durable occurrence=%v", err)
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native selection lost")
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
t.Fatal("fixed native context/history changed")
}
input := string(renderedInput(t, client.requests[1], LocationRegistryReferenceSlot))
if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) {
t.Fatal("registry projection exposed an opaque identity")
}
if !reflect.DeepEqual(renderedInput(t, client.requests[0], LocationRegistryReferenceSlot), renderedInput(t, client.requests[1], LocationRegistryReferenceSlot)) {
t.Fatal("registry projection was regenerated")
}
}
}
func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
errors []error
wantAssessments, wantRuns int
wantError bool
}{
{"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true},
{"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false},
{"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false},
{"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.References, _ = operationReferences(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
assessments, prepared := 0, 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
request.Operation.Assess = func(ctx context.Context, value dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
assessments++
return sourceAssessment(ctx, request, value)
}
client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared)
}
if test.wantError {
if result.Outcome.Disposition != "" {
t.Fatal("strict failure published an outcome")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
t.Fatalf("unqualified fallback=%#v", result)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatalf("empty occurrence shape=%v", err)
}
})
}
}
func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request := extractionRequest()
request.References, _ = operationReferences(t)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
}

View File

@@ -1,75 +0,0 @@
package locationoccurrences
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: PromptID,
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "location-registry.md", Path: "prompts/location-registry.md"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-transcript-chunk.md",
"common-dnd-references.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/location-occurrences")
if err != nil {
return nil, fmt.Errorf("scope location-occurrence assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare location-occurrence prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope location-occurrence schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,53 +0,0 @@
package locationoccurrences
import (
"context"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesLocationOccurrencePrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"location-occurrences-test": promptkitfixture.Profile("http://127.0.0.1:1/v1", "test"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-occurrences-test",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-occurrence-transcript"}]}`), "players": promptkit.Inline("location-occurrence-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
"location_registry": promptkit.Inline(`{"locations":[{"name":"location-occurrence-registry","registry_refs":[{"start_unit_id":7,"end_unit_id":7}],"context":[{"unit_id":7,"text":"registry context"}]}]}`),
},
})
if err != nil {
t.Fatal(err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_location_occurrences_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
var registryMessage string
for _, message := range prepared.Messages {
if strings.Contains(message.Content, "contextual location registry") {
registryMessage = message.Content
}
}
if !strings.Contains(registryMessage, "location-occurrence-registry") || !strings.Contains(registryMessage, `"registry_refs":[{"start_unit_id":7,"end_unit_id":7}]`) || strings.Contains(registryMessage, "source_id") || strings.Contains(registryMessage, "location:sha256") {
t.Fatalf("rendered prompt did not preserve source-free registry grounding: %s", registryMessage)
}
content := make([]string, len(prepared.Messages))
for index, message := range prepared.Messages {
content[index] = message.Content
}
rendered := strings.Join(content, "\n")
policy := strings.ReplaceAll(rendered, "\n", " ")
if !strings.Contains(policy, "context supports that coreference") || !strings.Contains(policy, "must not create a registry location") || !strings.Contains(policy, "provenance must never replace current-chunk evidence") || !strings.Contains(policy, "complete, ordered") {
t.Fatalf("rendered prompt = %q, want contextual coreference without registry-derived evidence", rendered)
}
}

View File

@@ -1,25 +0,0 @@
package locationoccurrences
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.location_occurrences"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_location_occurrences_llm")
ResponseSchemaID = "notarius.dnd.location_occurrences.llm"
ResponseSchemaName = "notarius_dnd_location_occurrences_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_location_occurrences_llm.v1.json",
})
}

View File

@@ -1,97 +0,0 @@
package locationoccurrences
import (
"bytes"
"encoding/json"
"strings"
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
)
func TestResponseSchemaRestrictsPrivateOccurrenceStructureAndKinds(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v", schema)
}
valid := map[string]any{"occurrences": []any{map[string]any{
"name": "The Mill", "registry_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}}, "kind": "visited",
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
}}}
if err := validateSchema(t, valid, schema.JSONSchema); err != nil {
t.Fatalf("valid response rejected: %v", err)
}
for _, mutate := range []func(map[string]any){
func(value map[string]any) { delete(value, "registry_refs") },
func(value map[string]any) { value["kind"] = "other" },
func(value map[string]any) { value["unexpected"] = true },
func(value map[string]any) { value["registry_refs"].([]any)[0].(map[string]any)["start_unit_id"] = 0 },
func(value map[string]any) {
value["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later"
},
} {
candidate := cloneCandidate(t, valid)
mutate(candidate["occurrences"].([]any)[0].(map[string]any))
if err := validateSchema(t, candidate, schema.JSONSchema); err == nil {
t.Fatal("schema accepted structurally invalid response")
}
}
}
func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
t.Fatalf("schema = %s, %v", second.JSONSchema, err)
}
if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID {
t.Fatalf("diagnostics = %#v", diagnostics)
} else if _, ok := diagnostics["json_schema"]; ok {
t.Fatalf("diagnostics leaked schema content: %#v", diagnostics)
}
}
func cloneCandidate(t *testing.T, value map[string]any) map[string]any {
t.Helper()
content, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
var clone map[string]any
if err := json.Unmarshal(content, &clone); err != nil {
t.Fatal(err)
}
return clone
}
func validateSchema(t *testing.T, value map[string]any, schemaContent []byte) error {
t.Helper()
content, err := json.Marshal(value)
if err != nil {
return err
}
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(content))
if err != nil {
return err
}
document, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return err
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", document); err != nil {
return err
}
schema, err := compiler.Compile("schema.json")
if err != nil {
return err
}
return schema.Validate(instance)
}

View File

@@ -3,14 +3,19 @@ package locationoccurrences
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func extractionRequest() contracts.TypedExtractionRequest {
func extractionRequest() contracts.TypedExtractionRequest[dnd.LocationOccurrenceList] {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-occurrences:chunk:0", SourceID: doc.ID, Index: 0,
@@ -18,10 +23,13 @@ func extractionRequest() contracts.TypedExtractionRequest {
Content: []byte(`{"units":[10,20,30]}`), MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest{
return contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]{
Source: doc, Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-occurrences.json"),
SessionID: "occurrence-session", LLMProfile: "occurrence-profile",
Operation: contracts.ProducerOperation[dnd.LocationOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.LocationOccurrenceList) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}},
}
}
@@ -37,51 +45,95 @@ func occurrenceRefs(start, end int) []occurrenceSourceRefResponse {
return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
profiles:
- id: occurrence-profile
endpoint: https://example.invalid/v1
model: occurrence-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatalf("New() error = %v", err)
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
extractor, err := New(engine, Options{}, references...)
if err != nil {
t.Fatal(err)
}
return extractor
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
t.Helper()
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name]
for _, message := range request.Prompt.Messages {
start := strings.Index(message.Content, "{")
if start < 0 {
continue
}
content := []byte(message.Content[start:])
var fields map[string]json.RawMessage
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
return content
}
}
t.Fatalf("missing rendered %s input", name)
return nil
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.LocationOccurrenceList]) contracts.TypedExtractionRequest[dnd.LocationOccurrenceList] {
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "")
return req
}
type fakeOccurrencesLLMClient struct {
response extractionResponse
err error
requests []contracts.StructuredCompletionRequest
response extractionResponse
responses []string
raw string
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
func (client *fakeOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
return nil, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
content := client.raw
if index < len(client.responses) {
content = client.responses[index]
}
if err := json.Unmarshal(content, target); err != nil {
return contracts.StructuredCompletionResponse{}, err
if content == "" {
encoded, err := json.Marshal(client.response)
if err != nil {
return nil, err
}
content = string(encoded)
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
}
req.Correction = correction
return req
return &promptkit.GenerateResponse{Content: content}, nil
}

View File

@@ -2,8 +2,12 @@ package npcoccurrences
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -12,10 +16,9 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const (
Key = "dnd/npc-occurrences"
mappingPolicy = "dnd.npc_occurrences.extract_mapping.v4"
)
const Key = "dnd/npc-occurrences"
const PromptID = "notarius/dnd.npc_occurrences"
const CorrectionPromptID = "notarius/dnd.npc_occurrences.repair"
const (
NPCRegistryReferenceSlot = npcregistry.ReferenceSlot
@@ -53,21 +56,17 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.NPCOccurrenceList] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
npcResolver *npcregistry.Resolver
promptSHA string
responseSchemaSHA string
engine *promptkit.Engine
npcResolver *npcregistry.Resolver
}
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
@@ -80,68 +79,19 @@ func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contr
if err != nil {
return nil, extractorErrorf("prepare NPC registry prompt input: %w", err)
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{
llm: llmClient,
npcResolver: npcResolver,
promptSHA: promptSHA,
responseSchemaSHA: responseSchema.SHA256,
}, nil
return &Extractor{engine: engine, npcResolver: npcResolver}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
metadata := map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"mapping_policy": mappingPolicy,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
}
seeded := e.npcResolver.Seeded()
if seeded.Bound() {
metadata["npc_registry_digest"] = seeded.Digest()
metadata["npc_count"] = seeded.Count()
}
return metadata
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
seeded := e.npcResolver.Seeded()
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "mapping_policy", Value: mappingPolicy},
{Name: "npc_registry", Value: seeded.IdentityDigest()},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.NPCOccurrenceList], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]) (contracts.TypedExtractionResult[dnd.NPCOccurrenceList], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("native engine must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
@@ -156,44 +106,50 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("NPC registry reference is required")
}
var response extractionResponse
inputs := shared.PromptInputs(sourceInput, req.References)
inputs[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile,
SessionID: req.SessionID,
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction,
Inputs: inputs,
}, &response)
if err != nil {
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("complete structured output: %w", err)
sourceID := req.Source.ID
materials := shared.PromptInputs(sourceInput, req.References)
materials[NPCRegistryReferenceSlot] = npcRegistry.PromptInput()
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.NPCOccurrenceList, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
canonicalizeResponse(&response, order, sourceID)
value, err := canonicalOccurrenceList(response, sourceID, npcRegistry)
if err != nil {
return nil, []string{"Resolve every occurrence to an exact canonical name from the supplied NPC registry. Omit occurrences for identities absent from that registry; do not invent an identity. Return the complete replacement occurrence list."}, nil
}
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(context.Context, *dnd.NPCOccurrenceList) (dnd.NPCOccurrenceList, error) {
return dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{}}, nil
})
if err != nil {
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("capture model candidate: %w", err)
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("produce occurrences: %w", err)
}
canonicalizeResponse(&response, order, req.Source.ID)
value, err := canonicalOccurrenceList(response, req.Source.ID, npcRegistry)
if err != nil {
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{}, extractorErrorf("resolve NPC names against registry: %w", err)
}
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{Value: value, ModelCandidate: candidate}, nil
return contracts.TypedExtractionResult[dnd.NPCOccurrenceList]{Value: value, Outcome: outcome}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.NPCOccurrenceListKind,
ReferenceSlots: referenceSlots(),
Key: Key,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassLLMBacked,
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...),
Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.NPCOccurrenceListKind,
ReferenceSlots: referenceSlots(),
}
}
@@ -203,7 +159,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -2,7 +2,6 @@ package npcoccurrences
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
@@ -105,10 +104,10 @@ func TestExtractUsesDocumentOrderForReferencesAndOccurrences(t *testing.T) {
}
func TestNewRequiresLLMAndRejectsAmbiguousReferenceSets(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v", err)
}
if _, err := New(&fakeOccurrencesLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
if _, err := New(newExtractor(t, &fakeOccurrencesLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
t.Fatalf("New() error = %v", err)
}
}
@@ -124,25 +123,19 @@ func TestExtractUsesNamesOnlyRegistryAndCurrentTranscriptEvidence(t *testing.T)
t.Fatalf("Extract() error = %v", err)
}
request := client.requests[0]
registry := request.Inputs[NPCRegistryReferenceSlot]
if registry.Name != NPCRegistryReferenceSlot || registry.MediaType != "application/json" || string(registry.Content) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` {
t.Fatalf("registry prompt input = %#v, want names-only projection", registry)
registry := renderedInput(t, request, NPCRegistryReferenceSlot)
if string(registry) != `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` {
t.Fatalf("names-only registry=%s", registry)
}
for _, forbidden := range []string{"npc:sha256:", "other-session", "start_unit_id"} {
if strings.Contains(string(registry.Content), forbidden) {
t.Fatalf("registry prompt input leaked %q: %s", forbidden, registry.Content)
if strings.Contains(string(registry), forbidden) {
t.Fatalf("registry projection exposed %q", forbidden)
}
}
if strings.Contains(string(request.Inputs["transcript"].Content), "other-session") {
t.Fatal("transcript input contains registry evidence")
}
metadata, err := json.Marshal(newExtractor(t, &fakeOccurrencesLLMClient{}, references).ManifestMetadata())
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(metadata), "Mira Thorn") || strings.Contains(string(metadata), "other-session") {
t.Fatalf("manifest metadata leaked registry content: %s", metadata)
if strings.Contains(string(renderedInput(t, request, "transcript")), "other-session") {
t.Fatal("registry provenance became occurrence evidence")
}
}
func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) {
@@ -160,7 +153,7 @@ func TestExtractRequiresBoundRegistryBeforeLLMCall(t *testing.T) {
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: "application/json", Content: []byte(`{"secret":"private source"}`)}},
},
}}
if _, err := New(client, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") {
if _, err := New(newExtractor(t, client).engine, Options{}, malformed); err == nil || !strings.Contains(err.Error(), "prepare NPC registry") || strings.Contains(err.Error(), "private source") {
t.Fatalf("New() error = %v, want content-safe malformed registry error", err)
}
if len(client.requests) != 0 {
@@ -177,7 +170,7 @@ func TestExtractRejectsUnknownNamesWithoutPartialResult(t *testing.T) {
req := extractionRequest()
req.References = references
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") {
if err == nil || result.Outcome.Disposition != "" {
t.Fatalf("Extract() error = %v, want unknown name failure", err)
}
if len(result.Value.Occurrences) != 0 {
@@ -211,13 +204,10 @@ func TestExtractResolvesGeneratedRegistryAtOperationTime(t *testing.T) {
if _, err := extractor.Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v", err)
}
if input := client.requests[0].Inputs[NPCRegistryReferenceSlot]; string(input.Content) != `{"npcs":[{"name":"Mira Thorn"}]}` || input.OriginURI != "" {
t.Fatalf("generated registry prompt input = %#v", input)
}
metadata := extractor.ManifestMetadata()
if _, ok := metadata["npc_registry_digest"]; ok {
t.Fatalf("operation-time registry leaked into static metadata: %#v", metadata)
if input := renderedInput(t, client.requests[0], NPCRegistryReferenceSlot); string(input) != `{"npcs":[{"name":"Mira Thorn"}]}` {
t.Fatalf("generated registry=%s", input)
}
}
func TestExtractAcceptsEmptyBoundRegistryAndEmptyResponse(t *testing.T) {
@@ -253,7 +243,7 @@ func TestExtractRejectsNonemptyResponseForEmptyBoundRegistry(t *testing.T) {
req := extractionRequest()
req.References = references
result, err := newExtractor(t, client, references).Extract(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "name is not in the NPC registry") || len(result.Value.Occurrences) != 0 {
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Occurrences) != 0 {
t.Fatalf("Extract() = %#v, %v; want no accepted occurrences", result, err)
}
}
@@ -268,11 +258,11 @@ func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) {
name string
extractor *Extractor
ctx context.Context
req contracts.TypedExtractionRequest
req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]
want string
}{
{"nil extractor", nilExtractor, context.Background(), valid, "extractor"},
{"nil LLM client", &Extractor{}, context.Background(), valid, "LLM client"},
{"nil native engine", &Extractor{}, context.Background(), valid, "native engine"},
{"wrapped preflight failure", extractor, context.Background(), mismatchedSourceInputRequest(valid), "must match chunk"},
} {
t.Run(test.name, func(t *testing.T) {
@@ -286,9 +276,9 @@ func TestExtractRejectsInvalidRequestsAndProviderFailures(t *testing.T) {
}
}
func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) {
func TestModuleSpecRegistrationAndNativeDependencies(t *testing.T) {
got := ModuleSpec()
if got.Key != Key || got.Stage != pipeline.StageExtract || got.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) {
if got.Key != Key || got.Stage != pipeline.StageExtract || !reflect.DeepEqual(got.Prompts, []string{PromptID, CorrectionPromptID}) || got.ArtifactKind != dnd.NPCOccurrenceListKind || !reflect.DeepEqual(got.Requires, []string{"chunks", "source.transcript"}) || !reflect.DeepEqual(got.Provides, []string{"dnd.npc_occurrences"}) {
t.Fatalf("ModuleSpec() = %#v", got)
}
var registrySlot contracts.ReferenceSlot
@@ -300,6 +290,10 @@ func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) {
if !registrySlot.Required || !reflect.DeepEqual(registrySlot.AcceptedArtifactKinds, []contracts.ArtifactKind{dnd.NPCRegistryKind}) || registrySlot.MaxBytes != NPCRegistryMaxBytes {
t.Fatalf("NPC registry slot = %#v", registrySlot)
}
got.Prompts[0] = "changed"
if ModuleSpec().Prompts[0] == "changed" {
t.Fatal("prompt dependencies alias caller memory")
}
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
if ModuleSpec().ReferenceSlots[0].AcceptedMediaTypes[0] == "changed" {
t.Fatal("ModuleSpec() returned mutable reference slots")
@@ -320,145 +314,4 @@ func TestModuleSpecRegistrationMetadataAndFingerprints(t *testing.T) {
t.Fatal("DecodeOptions() accepted unknown option")
}
references := requiredRegistryReferences(t, "Mira Thorn")
extractor := newExtractor(t, &fakeOccurrencesLLMClient{}, references)
metadata := extractor.ManifestMetadata()
for key, want := range map[string]string{
"prompt_id": PromptID, "prompt_version": SchemaVersion, "mapping_policy": mappingPolicy,
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
} {
if metadata[key] != want {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
}
}
for _, key := range []string{"prompt_sha256", "response_schema_sha256", "npc_registry_digest"} {
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
t.Fatalf("metadata[%q] = %#v, want digest", key, metadata[key])
}
}
wantFingerprints := map[string]struct{}{"prompt": {}, "response_schema": {}, "mapping_policy": {}, "npc_registry": {}}
for _, fingerprint := range extractor.CheckpointFingerprints() {
delete(wantFingerprints, fingerprint.Name)
}
if len(wantFingerprints) != 0 {
t.Fatalf("missing fingerprints = %#v", wantFingerprints)
}
}
func extractionRequest() contracts.TypedExtractionRequest {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0,
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30},
Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest{
Source: doc, Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
SessionID: "session-123", LLMProfile: "profile-npc-occurrences",
}
}
func sourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."},
{ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."},
{ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."},
{ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."},
{ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."},
{ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."},
},
}
}
func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet {
t.Helper()
npcs := make([]dnd.NPC, len(names))
for index, name := range names {
npcs[index] = dnd.NPC{
ID: identity.DeriveID(name), Name: name,
SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}},
}
}
content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: npcs})
if err != nil {
t.Fatal(err)
}
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
NPCRegistryReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}},
},
}}
}
func occurrenceRefs(start, end int) []occurrenceSourceRefResponse {
return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
}
func occurrenceNames(value dnd.NPCOccurrenceList) []string {
names := make([]string, len(value.Occurrences))
for index, occurrence := range value.Occurrences {
names[index] = occurrence.Name
}
return names
}
func occurrenceKinds(value dnd.NPCOccurrenceList) []dnd.NPCOccurrenceKind {
kinds := make([]dnd.NPCOccurrenceKind, len(value.Occurrences))
for index, occurrence := range value.Occurrences {
kinds[index] = occurrence.Kind
}
return kinds
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "")
return req
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
if err != nil {
t.Fatalf("New() error = %v", err)
}
return extractor
}
type fakeOccurrencesLLMClient struct {
response extractionResponse
err error
requests []contracts.StructuredCompletionRequest
}
func (client *fakeOccurrencesLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
}
*target = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
}
req.Correction = correction
return req
}

View File

@@ -0,0 +1,161 @@
package npcoccurrences
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcoccurrences/source_refs"
"go.maximumdirect.net/promptkit"
)
const unresolvedOccurrence = `{"occurrences":[{"name":"Unknown NPC","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
const knownOccurrence = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":10,"end_unit_id":10}]}]}`
const invalidCitation = `{"occurrences":[{"name":"Mira Thorn","kind":"dialogue","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
func operationReferences(t *testing.T) (contracts.ReferenceSet, string) {
return requiredRegistryReferences(t, "Mira Thorn"), identity.DeriveID("Mira Thorn")
}
func sourceAssessment(ctx context.Context, request contracts.TypedExtractionRequest[dnd.NPCOccurrenceList], value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
validator := refs.New(refs.Options{})
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.NPCOccurrenceList]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: validator.Name(), Decision: decision, Result: result}}}, nil
}
func TestNativeCorrectionKeepsRegistryResolutionAndInputsFixed(t *testing.T) {
for _, first := range []string{unresolvedOccurrence, invalidCitation} {
request := extractionRequest()
references, expectedID := operationReferences(t)
request.References = references
request.Operation.Controls.MaxAdditionalTurns = 1
// Accepted work consuming an ineligible generated reference stays ineligible.
request.Operation.ReuseEligible = false
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "occurrence-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
assessments := 0
request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
assessments++
return sourceAssessment(ctx, request, value)
}
client := &fakeOccurrencesLLMClient{responses: []string{first, knownOccurrence}}
client.onGenerate = func(context.Context, promptkit.GenerateRequest) {
if len(client.requests) == 1 {
request.References.Slots[NPCRegistryReferenceSlot].Items[0].Content[0] = '!'
}
}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
expectedAssessments := 1
if first == invalidCitation {
expectedAssessments = 2
}
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
len(client.requests) != 2 || len(prepared) != 2 || assessments != expectedAssessments || len(result.Value.Occurrences) != 1 {
t.Fatalf("corrected occurrences=%#v %v, assessments=%d native=%d/%d", result, err, assessments, len(client.requests), len(prepared))
}
occurrence := result.Value.Occurrences[0]
if occurrence.NPCID != expectedID || occurrence.Name != "Mira Thorn" || len(occurrence.SourceRefs) != 1 ||
occurrence.SourceRefs[0].SourceID != request.Source.ID || occurrence.SourceRefs[0].StartUnitID != 10 || occurrence.SourceRefs[0].EndUnitID != 10 {
t.Fatalf("identity/evidence=%#v", occurrence)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatalf("durable occurrence=%v", err)
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "occurrence-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native selection lost")
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != first {
t.Fatal("fixed native context/history changed")
}
input := string(renderedInput(t, client.requests[1], NPCRegistryReferenceSlot))
if strings.Contains(input, expectedID) || strings.Contains(input, request.Source.ID) {
t.Fatal("registry projection exposed an opaque identity")
}
if !reflect.DeepEqual(renderedInput(t, client.requests[0], NPCRegistryReferenceSlot), renderedInput(t, client.requests[1], NPCRegistryReferenceSlot)) {
t.Fatal("registry projection was regenerated")
}
}
}
func TestNativeOccurrenceFallbackIsAnExplicitEmptyList(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
response string
errors []error
wantAssessments, wantRuns int
wantError bool
}{
{"strict unresolved", contracts.ModeStrict, unresolvedOccurrence, nil, 0, 2, true},
{"permissive unresolved", contracts.ModePermissive, unresolvedOccurrence, nil, 0, 2, false},
{"rejected citation", contracts.ModePermissive, invalidCitation, nil, 2, 2, false},
{"provider unavailable", contracts.ModePermissive, "", []error{failure}, 0, 1, false},
{"failed continuation", contracts.ModePermissive, invalidCitation, []error{nil, failure}, 1, 2, false},
{"structural failure", contracts.ModePermissive, `{"occurrences":null}`, nil, 0, 1, false},
} {
t.Run(test.name, func(t *testing.T) {
request := extractionRequest()
request.References, _ = operationReferences(t)
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
assessments, prepared := 0, 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
request.Operation.Assess = func(ctx context.Context, value dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
assessments++
return sourceAssessment(ctx, request, value)
}
client := &fakeOccurrencesLLMClient{raw: test.response, errors: test.errors}
result, err := newExtractor(t, client, request.References).Extract(context.Background(), request)
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
t.Fatalf("outcome=%#v %v, assessments/runs=%d/%d", result, err, assessments, prepared)
}
if test.wantError {
if result.Outcome.Disposition != "" {
t.Fatal("strict failure published an outcome")
}
return
}
if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Value.Occurrences == nil || len(result.Value.Occurrences) != 0 {
t.Fatalf("unqualified fallback=%#v", result)
}
if _, err := occurrencecodec.New().Encode(result.Value); err != nil {
t.Fatalf("empty occurrence shape=%v", err)
}
})
}
}
func TestCanceledOccurrenceDeliveryCannotConstructFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
request := extractionRequest()
request.References, _ = operationReferences(t)
request.Operation.Controls.Mode = contracts.ModePermissive
client := &fakeOccurrencesLLMClient{raw: knownOccurrence, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newExtractor(t, client, request.References).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
}

View File

@@ -1,75 +0,0 @@
package npcoccurrences
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: "dnd.npc_occurrences",
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-transcript-chunk.md",
"common-dnd-references.md",
"common-dnd-npc-registry.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/npc-occurrences")
if err != nil {
return nil, fmt.Errorf("scope NPC-occurrence assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare NPC-occurrence prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope NPC-occurrence schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,70 +0,0 @@
package npcoccurrences
import (
"context"
"io/fs"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsAndPrepareOccurrencePrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatal(err)
}
schemaFS, err := registry.SchemaFS()
if err != nil {
t.Fatal(err)
}
if _, err := fs.ReadFile(schemaFS, "dnd_npc_occurrences_llm.v1.json"); err != nil {
t.Fatalf("response schema asset: %v", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"npc-occurrences-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "npc-occurrences-test-model"),
})
transcript := `{"units":[{"sentinel":"occurrence-transcript"}]}`
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "npc-occurrences-test-profile",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", transcript),
"players": promptkit.Inline("occurrence-player"),
"party": promptkit.Inline("Mira: ranger"),
"glossary": promptkit.Inline("Greencloak: title"),
"npc_registry": promptkit.Inline(`{"npcs":[{"name":"occurrence-npc"}]}`),
},
})
if err != nil {
t.Fatal(err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_npc_occurrences_llm.v1.json" {
t.Fatalf("prepared prompt = %#v", prepared)
}
}
func TestPromptMetadataDoesNotExposeAssetContent(t *testing.T) {
hash, err := promptAssetMetadata()
if err != nil || !strings.HasPrefix(hash, "sha256:") {
t.Fatalf("promptAssetMetadata() = %q, %v", hash, err)
}
metadata := newExtractor(t, &fakeOccurrencesLLMClient{}).ManifestMetadata()
for _, forbidden := range []string{"common-dnd-system", "dnd_npc_occurrences_llm.v1.json"} {
if strings.Contains(strings.Join(mapValues(metadata), " "), forbidden) {
t.Fatalf("metadata leaked prompt or schema content %q: %#v", forbidden, metadata)
}
}
}
func mapValues(values map[string]any) []string {
out := make([]string, 0, len(values))
for _, value := range values {
if text, ok := value.(string); ok {
out = append(out, text)
}
}
return out
}

View File

@@ -1,25 +0,0 @@
package npcoccurrences
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.npc_occurrences"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_npc_occurrences_llm")
ResponseSchemaID = "notarius.dnd.npc_occurrences.llm"
ResponseSchemaName = "notarius_dnd_npc_occurrences_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_npc_occurrences_llm.v1.json",
})
}

View File

@@ -1,127 +0,0 @@
package npcoccurrences
import (
"bytes"
"encoding/json"
"strings"
"testing"
"github.com/santhosh-tekuri/jsonschema/v6"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestResponseSchemaOwnsOnlyPrivateStructuralContract(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v", schema)
}
valid := validOccurrenceResponse()
content, err := json.Marshal(valid)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("valid response rejected: %v", err)
}
semanticCandidate := validOccurrenceResponse()
occurrence := semanticCandidate["occurrences"].([]any)[0].(map[string]any)
occurrence["name"] = ""
ref := occurrence["source_refs"].([]any)[0].(map[string]any)
ref["start_unit_id"] = 0
ref["end_unit_id"] = -1
content, err = json.Marshal(semanticCandidate)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("schema rejected non-categorical validator-owned semantics: %v", err)
}
for _, mutate := range []func(map[string]any){
func(record map[string]any) { delete(record, "name") },
func(record map[string]any) { record["kind"] = 1 },
func(record map[string]any) { record["kind"] = "unsupported" },
func(record map[string]any) { record["npc_id"] = "npc:sha256:opaque" },
func(record map[string]any) { record["unexpected"] = true },
func(record map[string]any) {
record["source_refs"].([]any)[0].(map[string]any)["source_id"] = "assigned later"
},
} {
candidate := validOccurrenceResponse()
mutate(candidate["occurrences"].([]any)[0].(map[string]any))
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err == nil {
t.Fatal("schema accepted structurally invalid response")
}
}
}
func TestResponseSchemaIsDefensiveAndContentSafe(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
t.Fatalf("second schema = %s, %v", second.JSONSchema, err)
}
if diagnostics := second.DiagnosticsMap(); diagnostics["key"] != ResponseSchemaKey || diagnostics["id"] != ResponseSchemaID {
t.Fatalf("diagnostics = %#v", diagnostics)
} else if _, ok := diagnostics["json_schema"]; ok {
t.Fatalf("diagnostics leak schema content: %#v", diagnostics)
}
}
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
for _, kind := range []string{string(dnd.NPCOccurrenceKindMentioned), string(dnd.NPCOccurrenceKindNoncombatPresence), string(dnd.NPCOccurrenceKindDialogue), string(dnd.NPCOccurrenceKindCombatAlly), string(dnd.NPCOccurrenceKindCombatOpponent), string(dnd.NPCOccurrenceKindOther)} {
candidate := validOccurrenceResponse()
candidate["occurrences"].([]any)[0].(map[string]any)["kind"] = kind
content, err := json.Marshal(candidate)
if err != nil {
t.Fatal(err)
}
if err := validateJSONSchema(content, schema.JSONSchema); err != nil {
t.Fatalf("supported NPC-occurrence kind %q was rejected: %v", kind, err)
}
}
}
func validOccurrenceResponse() map[string]any {
return map[string]any{"occurrences": []any{map[string]any{
"name": "Mira Thorn", "kind": "dialogue",
"source_refs": []any{map[string]any{"start_unit_id": 1, "end_unit_id": 2}},
}}}
}
func validateJSONSchema(instanceContent, schemaContent []byte) error {
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
if err != nil {
return err
}
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
if err != nil {
return err
}
compiler := jsonschema.NewCompiler()
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
return err
}
schema, err := compiler.Compile("schema.json")
if err != nil {
return err
}
return schema.Validate(instance)
}

View File

@@ -0,0 +1,184 @@
package npcoccurrences
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
)
func extractionRequest() contracts.TypedExtractionRequest[dnd.NPCOccurrenceList] {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0,
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 30},
Content: []byte(`{"units":[10,2,7,5,20,30]}`), MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]{
Source: doc, Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
SessionID: "session-123", LLMProfile: "profile-npc-occurrences",
Operation: contracts.ProducerOperation[dnd.NPCOccurrenceList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.NPCOccurrenceList) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}},
}
}
func sourceDocument() *source.SourceDocument {
return &source.SourceDocument{
ID: "session-alpha", Kind: "transcript", Format: "application/vnd.seriatim.minimal+json", Digest: "sha256:source",
Units: []source.SourceUnit{
{ID: 10, Kind: "transcript_segment", Text: "Mira Thorn is mentioned."},
{ID: 2, Kind: "transcript_segment", Text: "Mira Thorn speaks to the party."},
{ID: 7, Kind: "transcript_segment", Text: "Mira Thorn watches nearby."},
{ID: 5, Kind: "transcript_segment", Text: "Mira Thorn joins the party in combat."},
{ID: 20, Kind: "transcript_segment", Text: "Mira Thorn attacks the party."},
{ID: 30, Kind: "transcript_segment", Text: "Mira Thorn performs a ritual."},
},
}
}
func requiredRegistryReferences(t *testing.T, names ...string) contracts.ReferenceSet {
t.Helper()
npcs := make([]dnd.NPC, len(names))
for index, name := range names {
npcs[index] = dnd.NPC{
ID: identity.DeriveID(name), Name: name,
SourceRefs: []source.SourceRef{{SourceID: "other-session", StartUnitID: index + 1, EndUnitID: index + 1}},
}
}
content, err := npccodec.New().Encode(dnd.NPCRegistry{NPCs: npcs})
if err != nil {
t.Fatal(err)
}
return contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
NPCRegistryReferenceSlot: {
Slot: contracts.ReferenceSlot{Name: NPCRegistryReferenceSlot},
Items: []contracts.ReferenceItem{{SlotName: NPCRegistryReferenceSlot, MediaType: npccodec.MediaType, Content: content, Origin: contracts.ReferenceOrigin{Type: "generated"}}},
},
}}
}
func occurrenceRefs(start, end int) []occurrenceSourceRefResponse {
return []occurrenceSourceRefResponse{{StartUnitID: start, EndUnitID: end}}
}
func occurrenceNames(value dnd.NPCOccurrenceList) []string {
names := make([]string, len(value.Occurrences))
for index, occurrence := range value.Occurrences {
names[index] = occurrence.Name
}
return names
}
func occurrenceKinds(value dnd.NPCOccurrenceList) []dnd.NPCOccurrenceKind {
kinds := make([]dnd.NPCOccurrenceKind, len(value.Occurrences))
for index, occurrence := range value.Occurrences {
kinds[index] = occurrence.Kind
}
return kinds
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.NPCOccurrenceList]) contracts.TypedExtractionRequest[dnd.NPCOccurrenceList] {
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "")
return req
}
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
profiles:
- id: profile-npc-occurrences
endpoint: https://example.invalid/v1
model: occurrence-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
extractor, err := New(engine, Options{}, references...)
if err != nil {
t.Fatal(err)
}
return extractor
}
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
t.Helper()
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name]
for _, message := range request.Prompt.Messages {
start := strings.Index(message.Content, "{")
if start < 0 {
continue
}
content := []byte(message.Content[start:])
var fields map[string]json.RawMessage
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
return content
}
}
t.Fatalf("missing rendered %s input", name)
return nil
}
type fakeOccurrencesLLMClient struct {
response extractionResponse
responses []string
raw string
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeOccurrencesLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if client.err != nil {
return nil, client.err
}
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
content := client.raw
if index < len(client.responses) {
content = client.responses[index]
}
if content == "" {
encoded, err := json.Marshal(client.response)
if err != nil {
return nil, err
}
content = string(encoded)
}
return &promptkit.GenerateResponse{Content: content}, nil
}