Migrate registry extraction to native correction and explicit empty fallbacks

This commit is contained in:
2026-11-22 00:50:54 +00:00
parent 5d3696d4a0
commit 303d6589c5
36 changed files with 1107 additions and 1121 deletions

View File

@@ -3,19 +3,21 @@ package locationregistry
import (
"context"
"encoding/json"
"fmt"
"strings"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const (
Key = "dnd/location-registry"
mappingPolicy = "dnd.location_registry.extract_mapping.v2"
)
const Key = "dnd/location-registry"
const PromptID = "notarius/dnd.location_registry"
const CorrectionPromptID = "notarius/dnd.location_registry.repair"
var requiredCapabilities = []string{"chunks", "source.transcript"}
var providedCapabilities = []string{"dnd.location_registry"}
@@ -32,75 +34,31 @@ func referenceSlots() []contracts.ReferenceSlot {
}
var _ contracts.Extractor[dnd.LocationRegistry] = (*Extractor)(nil)
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
type Options struct{}
type Extractor struct {
llm contracts.StructuredLLMClient
promptSHA string
responseSchemaSHA string
}
type Extractor struct{ engine *promptkit.Engine }
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if llmClient == nil {
return nil, extractorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
if engine == nil {
return nil, extractorErrorf("native engine must not be nil")
}
if len(references) > 1 {
return nil, extractorErrorf("at most one reference set may be supplied")
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, extractorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, extractorErrorf("load response schema: %w", err)
}
return &Extractor{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
return &Extractor{engine: engine}, nil
}
func (e *Extractor) Key() string { return Key }
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
func (e *Extractor) ManifestMetadata() map[string]any {
if e == nil {
return nil
}
return map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": e.promptSHA,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": e.responseSchemaSHA,
"identity_policy": identity.Policy,
"mapping_policy": mappingPolicy,
}
}
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if e == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: e.promptSHA},
{Name: "response_schema", Value: e.responseSchemaSHA},
{Name: "identity_policy", Value: identity.Policy},
{Name: "mapping_policy", Value: mappingPolicy},
}
}
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.LocationRegistry], error) {
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.LocationRegistry]) (contracts.TypedExtractionResult[dnd.LocationRegistry], error) {
if e == nil {
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("extractor must not be nil")
}
if e.llm == nil {
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("LLM client must not be nil")
if e.engine == nil {
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("native engine must not be nil")
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
if err != nil {
@@ -108,29 +66,41 @@ func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRe
}
order := shared.NewSourceRefOrder(req.Source)
var response extractionResponse
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key, PromptID: PromptID, PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Correction: req.Correction,
Inputs: shared.PromptInputs(sourceInput, req.References),
}, &response)
if err != nil {
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("complete structured output: %w", err)
sourceID := req.Source.ID
materials := shared.PromptInputs(sourceInput, req.References)
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
for name, material := range materials {
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
}
candidate, err := shared.ModelCandidateFromResponse(completion)
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationRegistry, []string, error) {
var response extractionResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return nil, nil, extractorErrorf("decode native output: %w", err)
}
canonicalizeResponse(&response, order, sourceID)
value := canonicalLocationRegistry(response, sourceID)
return &value, nil, nil
},
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
},
func(context.Context, *dnd.LocationRegistry) (dnd.LocationRegistry, error) {
return dnd.LocationRegistry{Locations: []dnd.Location{}}, nil
})
if err != nil {
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("capture model candidate: %w", err)
return contracts.TypedExtractionResult[dnd.LocationRegistry]{}, extractorErrorf("produce registry: %w", err)
}
canonicalizeResponse(&response, order, req.Source.ID)
return contracts.TypedExtractionResult[dnd.LocationRegistry]{Value: canonicalLocationRegistry(response, req.Source.ID), ModelCandidate: candidate}, nil
return contracts.TypedExtractionResult[dnd.LocationRegistry]{Value: value, Outcome: outcome}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
return pipeline.ModuleSpec{
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked,
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
Prompts: []string{PromptID, CorrectionPromptID},
Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...),
ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots(),
}
}
@@ -141,7 +111,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options, request.References)
return New(request.Dependencies.Engine, options, request.References)
})
}

View File

@@ -75,10 +75,11 @@ func TestExtractPassesReferencesWithoutTreatingThemAsEvidence(t *testing.T) {
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v", err)
}
inputs := client.requests[0].Inputs
if string(inputs["glossary"].Content) != "Old Mill: abandoned granary" || strings.Contains(string(inputs["transcript"].Content), "abandoned granary") {
t.Fatalf("prompt inputs = %#v, want separated reference material", inputs)
messageContaining(t, client.requests[0], "Old Mill: abandoned granary")
if strings.Contains(messageContaining(t, client.requests[0], string(req.Chunk.Content)).Content, "abandoned granary") {
t.Fatal("references became source evidence")
}
}
func TestExtractDoesNotMutateRequestMaterials(t *testing.T) {
@@ -105,11 +106,11 @@ func TestExtractHandlesEmptyOutputAndLocalFailures(t *testing.T) {
for _, test := range []struct {
name string
extractor *Extractor
req contracts.TypedExtractionRequest
req contracts.TypedExtractionRequest[dnd.LocationRegistry]
want string
}{
{name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"},
{name: "nil client", extractor: &Extractor{}, req: request, want: "LLM client"},
{name: "nil client", extractor: &Extractor{}, req: request, want: "native engine"},
{name: "preflight", extractor: newExtractor(t, &fakeLocationsLLMClient{}), req: mismatchedSourceInputRequest(request), want: "must match chunk"},
} {
t.Run(test.name, func(t *testing.T) {

View File

@@ -0,0 +1,156 @@
package locationregistry
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
shape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/shape"
refs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/locationregistry/source_relatedness"
"go.maximumdirect.net/promptkit"
)
const ungroundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":999,"end_unit_id":999}]}]}`
const groundedRegistry = `{"locations":[{"name":"Old Mill","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
const emptyRegistry = `{"locations":[]}`
func assessRegistry(ctx context.Context, request contracts.TypedExtractionRequest[dnd.LocationRegistry], value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
assessment := contracts.CandidateAssessment{}
for _, validator := range []contracts.TypedValidator[dnd.LocationRegistry]{
shape.New(shape.Options{}), refs.New(refs.Options{}), relatedness.New(relatedness.Options{}),
} {
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.LocationRegistry]{Stage: "extract", Source: request.Source, Chunk: request.Chunk, Value: value})
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if result.Approved {
decision = contracts.AssessmentApproved
}
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
}
return assessment, nil
}
func TestNativeExtractionCorrectsGroundingWithFixedInputs(t *testing.T) {
client := &fakeLocationsLLMClient{responses: []string{ungroundedRegistry, groundedRegistry}}
request := extractionRequest()
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"glossary": {Items: []contracts.ReferenceItem{{Content: []byte("fixed-reference")}}}}}
request.Operation.Controls.MaxAdditionalTurns = 1
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "registry-override"}
var prepared []promptkit.PreparedRun
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
assessments := 0
request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
assessments++
assessment, err := assessRegistry(ctx, request, value)
if assessments == 1 {
if assessment.Accepted() {
t.Fatal("unresolvable citation was accepted")
}
// Changes after capture must not cause source/reference preparation again.
request.SourceInput.Content[0] = '!'
request.References.Slots["glossary"].Items[0].Content[0] = '!'
}
return assessment, err
}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || !result.Outcome.ReuseEligible ||
assessments != 2 || len(client.requests) != 2 || len(prepared) != 2 || len(result.Value.Locations) != 1 {
t.Fatalf("corrected registry=%#v %v, assessments/provider/prepared=%d/%d/%d", result, err, assessments, len(client.requests), len(prepared))
}
record := result.Value.Locations[0]
if record.Name != "Old Mill" || len(record.SourceRefs) != 1 || record.SourceRefs[0].SourceID != request.Source.ID ||
record.SourceRefs[0].StartUnitID != 1 || record.SourceRefs[0].EndUnitID != 1 || record.ID != identity.DeriveID(record.Name, record.SourceRefs) {
t.Fatalf("canonical record identity/evidence=%#v", record)
}
if _, err := registrycodec.New().Encode(result.Value); err != nil {
t.Fatalf("durable registry encoding: %v", err)
}
for i, call := range client.requests {
id := PromptID
if i > 0 {
id = CorrectionPromptID
}
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != request.LLMProfile || call.Target.Model != "registry-override" || call.Prompt.SessionID != request.SessionID {
t.Fatal("native selection/overrides lost")
}
}
initial, revised := client.requests[0].Prompt.Messages, client.requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) ||
revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != ungroundedRegistry {
t.Fatal("native correction lost fixed inputs or prior response")
}
messageContaining(t, client.requests[1], "fixed-reference")
messageContaining(t, client.requests[1], string(request.Chunk.Content))
}
func TestNativeExtractionDistinguishesEmptyAcceptanceFromFallback(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name string
mode contracts.ExecutionMode
responses []string
providerErrors []error
wantAssessments, wantRuns int
wantError, wantFallback bool
}{
{"ordinary empty", contracts.ModeStrict, []string{emptyRegistry}, nil, 1, 1, false, false},
{"strict rejection", contracts.ModeStrict, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, true, false},
{"permissive rejection", contracts.ModePermissive, []string{ungroundedRegistry, ungroundedRegistry}, nil, 2, 2, false, true},
{"provider failure", contracts.ModePermissive, nil, []error{failure}, 0, 1, false, true},
{"failed correction", contracts.ModePermissive, []string{ungroundedRegistry}, []error{nil, failure}, 1, 2, false, true},
{"structural failure", contracts.ModePermissive, []string{`{"locations":null}`}, nil, 0, 1, false, true},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeLocationsLLMClient{responses: test.responses, errors: test.providerErrors}
if test.name == "structural failure" {
client.content = []byte(test.responses[0])
}
request := extractionRequest()
request.Operation.Controls.Mode = test.mode
request.Operation.Controls.MaxAdditionalTurns = 1
assessments, prepared := 0, 0
request.Operation.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
request.Operation.Assess = func(ctx context.Context, value dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
assessments++
return assessRegistry(ctx, request, value)
}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if (err != nil) != test.wantError || assessments != test.wantAssessments || prepared != test.wantRuns {
t.Fatalf("registry=%#v %v, assessments=%d runs=%d", result, err, assessments, prepared)
}
if test.wantError {
if result.Outcome.Disposition != "" {
t.Fatal("strict rejection produced an outcome")
}
return
}
if result.Value.Locations == nil || len(result.Value.Locations) != 0 || (result.Outcome.Disposition == contracts.ProducerFallback) != test.wantFallback ||
result.Outcome.ReuseEligible == test.wantFallback {
t.Fatalf("unqualified empty registry=%#v", result)
}
if _, err := registrycodec.New().Encode(result.Value); err != nil {
t.Fatalf("empty durable registry invalid: %v", err)
}
})
}
}
func TestCanceledNativeExtractionCannotDeliverFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := &fakeLocationsLLMClient{content: []byte(groundedRegistry), onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
request := extractionRequest()
request.Operation.Controls.Mode = contracts.ModePermissive
result, err := newExtractor(t, client).Extract(ctx, request)
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(client.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
}

View File

@@ -1,74 +0,0 @@
package locationregistry
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: PromptID,
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-extraction-evidence.md",
"common-dnd-identity.md",
"common-dnd-transcript-chunk.md",
"common-dnd-references.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/location-registry/extract")
if err != nil {
return nil, fmt.Errorf("scope location extraction assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare location prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope location extraction schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,39 +0,0 @@
package locationregistry
import (
"context"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesLocationPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v", err)
}
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"location-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "location-test-model"),
})
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "location-test-profile", Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline(`{"units":[{"sentinel":"location-transcript"}]}`), "players": promptkit.Inline("location-player"), "party": promptkit.Inline(" "), "glossary": promptkit.Inline(" "),
}})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_location_registry_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want location prompt identity and schema wiring", prepared)
}
content := make([]string, len(prepared.Messages))
for index, message := range prepared.Messages {
content[index] = message.Content
}
rendered := strings.Join(content, "\n")
if !strings.Contains(rendered, "stable proper name or unique in-world designation") || !strings.Contains(rendered, "the room") || !strings.Contains(rendered, "Do not use capitalization as an eligibility test") {
t.Fatalf("rendered prompt = %q, want named-or-unique location eligibility rules", rendered)
}
}

View File

@@ -8,17 +8,16 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
)
func TestModuleRegistrationAndMetadata(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
func TestModuleRegistrationAndNativeDependencies(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v, want client rejection", err)
}
if _, err := New(&fakeLocationsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
if _, err := New(newExtractor(t, &fakeLocationsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one") {
t.Fatalf("New() error = %v, want reference-set rejection", err)
}
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.location_registry"}, ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots()}
want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.location_registry"}, ArtifactKind: dnd.LocationRegistryKind, ReferenceSlots: referenceSlots()}
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
}
@@ -35,19 +34,5 @@ func TestModuleRegistrationAndMetadata(t *testing.T) {
if _, err := DecodeOptions(map[string]any{"unknown": true}); err == nil {
t.Fatal("DecodeOptions() accepted unknown option")
}
extractor := newExtractor(t, &fakeLocationsLLMClient{})
metadata := extractor.ManifestMetadata()
for key, value := range map[string]string{
"prompt_id": PromptID, "prompt_version": SchemaVersion,
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
"identity_policy": identity.Policy, "mapping_policy": mappingPolicy,
} {
if metadata[key] != value {
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], value)
}
}
if got := extractor.CheckpointFingerprints(); len(got) != 4 || got[0].Name != "prompt" || got[1].Name != "response_schema" || got[2].Value != identity.Policy || got[3].Value != mappingPolicy {
t.Fatalf("CheckpointFingerprints() = %#v", got)
}
}

View File

@@ -1,25 +0,0 @@
package locationregistry
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.location_registry"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_location_registry_llm")
ResponseSchemaID = "notarius.dnd.location_registry.llm"
ResponseSchemaName = "notarius_dnd_location_registry_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_location_registry_llm.v1.json",
})
}

View File

@@ -1,25 +0,0 @@
package locationregistry
import (
"encoding/json"
"strings"
"testing"
)
func TestLocationResponseSchemaIsPrivateAndStructural(t *testing.T) {
schema, err := loadResponseSchema()
if err != nil {
t.Fatalf("loadResponseSchema() error = %v", err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Name != ResponseSchemaName || schema.Version != SchemaVersion || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v, want location response schema", schema)
}
var document map[string]any
if err := json.Unmarshal(schema.JSONSchema, &document); err != nil {
t.Fatal(err)
}
encoded, err := json.Marshal(document)
if err != nil || strings.Contains(string(encoded), `"id"`) {
t.Fatalf("schema = %s, want no durable ID field", encoded)
}
}

View File

@@ -3,14 +3,18 @@ package locationregistry
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func extractionRequest() contracts.TypedExtractionRequest {
func extractionRequest() contracts.TypedExtractionRequest[dnd.LocationRegistry] {
doc := sourceDocument()
chunk := &source.Chunk{
ID: "session-locations:chunk:0", SourceID: doc.ID, Index: 0,
@@ -18,10 +22,11 @@ func extractionRequest() contracts.TypedExtractionRequest {
Content: []byte(`{"units":[1,2,3]}`), MediaType: "application/json",
Units: append([]source.SourceUnit(nil), doc.Units...),
}
return contracts.TypedExtractionRequest{
return contracts.TypedExtractionRequest[dnd.LocationRegistry]{
Source: doc, Chunk: chunk,
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-locations.json"),
SessionID: "location-session", LLMProfile: "location-profile",
Operation: registryOperation(),
}
}
@@ -37,58 +42,97 @@ func responseSourceRefs(startUnitID, endUnitID int) []locationSourceRefResponse
return []locationSourceRefResponse{{StartUnitID: startUnitID, EndUnitID: endUnitID}}
}
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
func registryOperation() contracts.ProducerOperation[dnd.LocationRegistry] {
return contracts.ProducerOperation[dnd.LocationRegistry]{
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true,
Assess: func(context.Context, dnd.LocationRegistry) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
},
}
}
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
t.Helper()
extractor, err := New(client, Options{}, references...)
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("registry fixture", []byte(`promptkit:
profiles:
- id: location-profile
endpoint: https://example.invalid/v1
model: registry-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatalf("New() error = %v, want nil", err)
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
extractor, err := New(engine, Options{}, references...)
if err != nil {
t.Fatal(err)
}
return extractor
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
func messageContaining(t *testing.T, request promptkit.GenerateRequest, text string) promptkit.RenderedMessage {
t.Helper()
for _, message := range request.Prompt.Messages {
if strings.Contains(message.Content, text) {
return message
}
}
t.Fatalf("missing rendered input %q", text)
return promptkit.RenderedMessage{}
}
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.LocationRegistry]) contracts.TypedExtractionRequest[dnd.LocationRegistry] {
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
return req
}
type fakeLocationsLLMClient struct {
response extractionResponse
content []byte
err error
requests []contracts.StructuredCompletionRequest
response extractionResponse
content []byte
responses []string
err error
errors []error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeLocationsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
func (client *fakeLocationsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
index := len(client.requests)
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
target, ok := out.(*extractionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
if err := ctx.Err(); err != nil {
return nil, err
}
if client.err != nil {
return nil, client.err
}
if index < len(client.errors) && client.errors[index] != nil {
return nil, client.errors[index]
}
content := append([]byte(nil), client.content...)
if len(content) != 0 {
if err := json.Unmarshal(content, target); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
} else {
*target = client.response
if index < len(client.responses) {
content = []byte(client.responses[index])
}
if len(content) == 0 {
var err error
content, err = json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
}
return contracts.StructuredCompletionResponse{Content: content}, nil
}
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
req.Inputs = req.Inputs.Clone()
correction, err := contracts.CloneSemanticCorrection(req.Correction)
if err != nil {
panic(err)
}
req.Correction = correction
return req
return &promptkit.GenerateResponse{Content: string(content)}, nil
}