Migrate ordered NPC and location handoff tests to native execution

This commit is contained in:
2026-11-22 04:45:39 +00:00
parent 0ad73ff575
commit 562dfad5cd
9 changed files with 191 additions and 202 deletions

View File

@@ -26,11 +26,9 @@ func loadIntegrationConfigText(t *testing.T, content string) config.Config {
return loadIntegrationConfig(t, path)
}
func configFromYAML(content string) (config.Config, error) {
directory, err := os.MkdirTemp("", "notarius-integration-config-")
if err != nil {
return config.Config{}, err
}
func configFromYAML(t *testing.T, content string) (config.Config, error) {
t.Helper()
directory := t.TempDir()
path := filepath.Join(directory, "config.yml")
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
return config.Config{}, err

View File

@@ -9,8 +9,8 @@ import (
func TestItemOccurrenceConfigurationRequiresEarlierItemRegistry(t *testing.T) {
registries := productionNPCRegistries(t)
cfg, err := configFromYAML(`
version: 5
cfg, err := configFromYAML(t, `
version: 6
pipelines:
items:
input: {module: seriatim}
@@ -41,8 +41,8 @@ pipelines:
t.Fatalf("Resolve() error = %v", err)
}
broken, err := configFromYAML(strings.Replace(`
version: 5
broken, err := configFromYAML(t, strings.Replace(`
version: 6
pipelines:
items:
input: {module: seriatim}

View File

@@ -10,15 +10,11 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
locationoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences"
locationcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
locationoccurrences "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences"
locations "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry"
locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry"
"go.maximumdirect.net/promptkit"
)
func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
@@ -29,15 +25,12 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 {
t.Fatalf("run outputs = %#v rejected = %#v, want accepted registry and occurrences", output.NormalizeOutputs, output.Rejected)
if len(output.OutputFiles) != 3 {
t.Fatalf("files=%#v, want two lanes and evidence", output.OutputFiles)
}
registryOutput := normalizedLane(t, output, "locations")
if registryOutput.Artifact.Kind != dnd.LocationRegistryKind || registryOutput.Artifact.Schema.ID != locationcodec.SchemaID {
t.Fatalf("registry envelope = %#v, want durable location artifact", registryOutput)
}
registry, err := locationcodec.New().Decode(registryOutput.Artifact.Content)
registryOutput := publishedLane(t, output, "locations")
registry, err := locationcodec.New().Decode(registryOutput.Bytes)
if err != nil {
t.Fatalf("Decode(registry) error = %v", err)
}
@@ -45,11 +38,8 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
t.Fatalf("registry = %#v, want same-name locations distinguished by source-derived IDs", registry)
}
occurrenceOutput := normalizedLane(t, output, "occurrences")
if occurrenceOutput.Artifact.Kind != dnd.LocationOccurrenceListKind || occurrenceOutput.Artifact.Schema.ID != locationoccurrencecodec.SchemaID {
t.Fatalf("occurrence envelope = %#v, want durable occurrence artifact", occurrenceOutput)
}
occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Artifact.Content)
occurrenceOutput := publishedLane(t, output, "occurrences")
occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Bytes)
if err != nil {
t.Fatalf("Decode(occurrences) error = %v", err)
}
@@ -62,11 +52,16 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
}
}
request := client.requestFor(t, locationoccurrences.PromptID)
registryInput := request.Inputs["location_registry"]
if registryInput.MediaType != locationcodec.MediaType || strings.Contains(string(registryInput.Content), "source_refs") || strings.Contains(string(registryInput.Content), registry.Locations[0].ID) || strings.Contains(string(registryInput.Content), registry.Locations[1].ID) || !strings.Contains(string(registryInput.Content), "registry_refs") {
t.Fatalf("occurrence registry input = %#v, want contextual selector projection", registryInput)
request := client.requestFor(t, "occurrences")
registryInput, err := nativeJSONInput(request, "locations")
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(registryInput), "source_refs") || strings.Contains(string(registryInput), registry.Locations[0].ID) || strings.Contains(string(registryInput), registry.Locations[1].ID) || !strings.Contains(string(registryInput), "registry_refs") {
t.Fatalf("registry input=%s, want contextual source-free selectors", registryInput)
}
contextArtifact := outputFileContent(t, output.OutputFiles, "evidence-context.json")
evidence, err := evidencecontext.New().Decode(contextArtifact)
if err != nil {
@@ -92,8 +87,8 @@ func TestLocationOccurrenceConsumerDoesNotRunAfterRejectedRegistry(t *testing.T)
if err == nil {
t.Fatal("Run() error = nil, want missing accepted location registry")
}
if client.requestCount(locationoccurrences.PromptID) != 0 {
t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount(locationoccurrences.PromptID))
if client.requestCount("occurrences") != 0 {
t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount("occurrences"))
}
}
@@ -106,24 +101,6 @@ func TestLocationOccurrenceConfigurationRequiresEarlierRegistry(t *testing.T) {
}
}
func TestLocationOccurrenceCheckpointTracksGeneratedRegistry(t *testing.T) {
registries := productionNPCRegistries(t)
checkpoint := newGeneratedReferenceCheckpointLoader()
_, err := runPreparedPipeline(t, registries, resolveLocationPipeline(t, registries), &locationHandoffLLMClient{}, pipeline.RunInput{
RawInput: readNPCFixture(t),
Checkpoint: checkpoint,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
for _, dependency := range checkpoint.extractDependencies("occurrences") {
if dependency.Name == "generated-reference:location_registry:0" && strings.HasPrefix(dependency.Value, "sha256:") {
return
}
}
t.Fatalf("occurrence checkpoint dependencies = %#v, want generated location registry fingerprint", checkpoint.extractDependencies("occurrences"))
}
func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipeline.ResolvedPipeline {
t.Helper()
cfg := loadIntegrationConfigText(t, locationPipelineConfig)
@@ -138,11 +115,7 @@ func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipel
return resolved
}
const locationPipelineConfig = `version: 5
cache:
chunk_plans:
mode: bypass
checkpoints: {}
const locationPipelineConfig = `version: 6
pipelines:
dnd-locations-fixture:
input:
@@ -175,28 +148,28 @@ pipelines:
type locationHandoffLLMClient struct {
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
requests []promptkit.GenerateRequest
rejectRegistry bool
}
func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
func (client *locationHandoffLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
client.mu.Lock()
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
client.requests = append(client.requests, request)
client.mu.Unlock()
var payload any
switch request.PromptID {
case locations.PromptID:
switch {
case nativeSchemaHasProperty(request, "locations"):
name := "Moon Gate"
if client.rejectRegistry {
name = ""
}
payload = map[string]any{"locations": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}}
case locationnormalize.PromptID:
case nativeSchemaHasProperty(request, "duplicate_groups"):
payload = map[string]any{"duplicate_groups": []any{}}
case locationoccurrences.PromptID:
case nativeSchemaHasProperty(request, "occurrences"):
var projection struct {
Locations []struct {
Name string `json:"name"`
@@ -206,48 +179,49 @@ func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context,
} `json:"registry_refs"`
} `json:"locations"`
}
if err := json.Unmarshal(request.Inputs["location_registry"].Content, &projection); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("decode location registry: %w", err)
content, err := nativeJSONInput(request, "locations")
if err != nil {
return nil, err
}
if err := json.Unmarshal(content, &projection); err != nil {
return nil, fmt.Errorf("decode location registry: %w", err)
}
if len(projection.Locations) != 2 {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations))
return nil, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations))
}
payload = map[string]any{"occurrences": []any{map[string]any{"name": projection.Locations[0].Name, "registry_refs": projection.Locations[0].RegistryRefs, "kind": "visited", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": projection.Locations[1].Name, "registry_refs": projection.Locations[1].RegistryRefs, "kind": "mentioned", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}}
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", request.PromptID)
return nil, fmt.Errorf("unexpected native location output contract")
}
content, err := json.Marshal(payload)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
if err := json.Unmarshal(content, output); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "location-handoff"}, nil
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func (client *locationHandoffLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest {
func (client *locationHandoffLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
t.Helper()
client.mu.Lock()
defer client.mu.Unlock()
for _, request := range client.requests {
if request.PromptID == promptID {
if nativeSchemaHasProperty(request, property) {
return request
}
}
t.Fatalf("requests = %#v, missing %q", client.requests, promptID)
return contracts.StructuredCompletionRequest{}
t.Fatalf("requests = %#v, missing %q", client.requests, property)
return promptkit.GenerateRequest{}
}
func (client *locationHandoffLLMClient) requestCount(promptID string) int {
func (client *locationHandoffLLMClient) requestCount(property string) int {
client.mu.Lock()
defer client.mu.Unlock()
count := 0
for _, request := range client.requests {
if request.PromptID == promptID {
if nativeSchemaHasProperty(request, property) {
count++
}
}
return count
}
var _ contracts.StructuredLLMClient = (*locationHandoffLLMClient)(nil)
var _ promptkit.LLMClient = (*locationHandoffLLMClient)(nil)

View File

@@ -482,17 +482,6 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr
return output
}
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
t.Helper()
for _, file := range files {
if file.Name == name {
return append([]byte(nil), file.Bytes...)
}
}
t.Fatalf("output files = %#v, missing %q", files, name)
return nil
}
func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
t.Helper()
for _, serialized := range output.NormalizeOutputs {

View File

@@ -5,20 +5,16 @@ import (
"encoding/json"
"fmt"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
occurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcoccurrences"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
"go.maximumdirect.net/promptkit"
)
func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.T) {
@@ -30,24 +26,22 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 {
t.Fatalf("run outputs = %#v rejected = %#v, want NPC and occurrence artifacts", output.NormalizeOutputs, output.Rejected)
if len(output.OutputFiles) != 2 {
t.Fatalf("output files = %#v, want registry and occurrences", output.OutputFiles)
}
request := client.requestFor(t, occurrenceextract.PromptID)
request := client.requestFor(t, "occurrences")
wantRegistry := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`
if got := string(request.Inputs["npc_registry"].Content); got != wantRegistry {
t.Fatalf("occurrence registry input = %s, want names-only projection %s", got, wantRegistry)
registryInput, err := nativeJSONInput(request, "npcs")
if err != nil {
t.Fatal(err)
}
if request.Inputs["npc_registry"].MediaType != npccodec.MediaType {
t.Fatalf("occurrence registry media type = %q, want %q", request.Inputs["npc_registry"].MediaType, npccodec.MediaType)
if string(registryInput) != wantRegistry {
t.Fatalf("occurrence registry = %s, want names-only projection %s", registryInput, wantRegistry)
}
serialized := normalizedLane(t, output, "occurrences")
if serialized.Artifact.Schema.ID != occurrencecodec.SchemaID || serialized.Artifact.Schema.Version != occurrencecodec.SchemaVersion {
t.Fatalf("occurrence artifact schema = %#v", serialized.Artifact.Schema)
}
occurrences, err := occurrencecodec.New().Decode(serialized.Artifact.Content)
serialized := publishedLane(t, output, "occurrences")
occurrences, err := occurrencecodec.New().Decode(serialized.Bytes)
if err != nil {
t.Fatalf("Decode(occurrence output) error = %v", err)
}
@@ -68,7 +62,7 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.
}
var durable map[string]json.RawMessage
if err := json.Unmarshal(serialized.Artifact.Content, &durable); err != nil {
if err := json.Unmarshal(serialized.Bytes, &durable); err != nil {
t.Fatalf("unmarshal durable occurrence payload: %v", err)
}
if len(durable) != 1 || durable["occurrences"] == nil {
@@ -90,46 +84,39 @@ func TestSemanticNPCNormalizationCrossesOrderedRegistryHandoff(t *testing.T) {
if err != nil || len(warnings) != 0 {
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
}
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
resolved.Steps[stepIndex].ArtifactLanes[laneIndex].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
}
}
client := &semanticNPCOccurrenceClient{}
raw := []byte(`{"metadata":{"id":"semantic-session","title":"Semantic NPC session"},"segments":[{"id":1,"start":0,"end":1,"speaker":"DM","text":"Mira Thorn enters."},{"id":2,"start":1,"end":2,"speaker":"DM","text":"Mira Thorn, the Greencloak, waves."}]}`)
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: raw})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if client.requestCount(npcregistry.PromptID) != 2 || client.requestCount(npcnormalize.PromptID) != 1 {
if client.requestCount("npcs") != 2 || client.requestCount("duplicate_groups") != 1 {
t.Fatalf("prompt requests = %#v, want two extraction calls and one document normalization call", client.requests)
}
npcOutput := normalizedLane(t, output, "npc_registry")
npcsValue, err := npccodec.New().Decode(npcOutput.Artifact.Content)
npcOutput := publishedLane(t, output, "npc_registry")
npcsValue, err := npccodec.New().Decode(npcOutput.Bytes)
if err != nil {
t.Fatal(err)
}
if npcOutput.StepID != "identify-npcs" || len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") {
if len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") {
t.Fatalf("NPC output = %#v / %#v, want canonical ordered producer artifact", npcOutput, npcsValue)
}
if refs := npcsValue.NPCs[0].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "semantic-session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "semantic-session", StartUnitID: 2, EndUnitID: 2}}) {
t.Fatalf("NPC evidence = %#v, want original extraction evidence union", refs)
}
occurrenceOutput := normalizedLane(t, output, "occurrences")
if occurrenceOutput.StepID != "extract-occurrences" {
t.Fatalf("occurrence output step = %q, want ordered downstream step", occurrenceOutput.StepID)
registryRequest := client.requestFor(t, "occurrences")
registryInput, err := nativeJSONInput(registryRequest, "npcs")
if err != nil || string(registryInput) != `{"npcs":[{"name":"Mira Thorn"}]}` {
t.Fatalf("downstream registry = %s, error=%v", registryInput, err)
}
registryRequest := client.requestFor(t, occurrenceextract.PromptID)
if got := string(registryRequest.Inputs["npc_registry"].Content); got != `{"npcs":[{"name":"Mira Thorn"}]}` {
t.Fatalf("downstream registry = %s, want one canonical NPC name", got)
occurrences, err := occurrencecodec.New().Decode(publishedLane(t, output, "occurrences").Bytes)
if err != nil || len(occurrences.Occurrences) != 2 {
t.Fatalf("downstream occurrences=%#v error=%v, want both chunks", occurrences, err)
}
manifestContent, err := json.Marshal(output.Manifest)
if err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"npc:sha256:", "Mira Thorn enters.", "Mira Thorn, the Greencloak, waves."} {
if strings.Contains(string(manifestContent), forbidden) {
t.Fatalf("manifest leaked private identity or transcript content %q: %s", forbidden, manifestContent)
for index, occurrence := range occurrences.Occurrences {
wantRef := source.SourceRef{SourceID: "semantic-session", StartUnitID: index + 1, EndUnitID: index + 1}
if occurrence.Name != "Mira Thorn" || occurrence.NPCID != npcsValue.NPCs[0].ID || !reflect.DeepEqual(occurrence.SourceRefs, []source.SourceRef{wantRef}) {
t.Fatalf("occurrence=%#v, want canonical identity and current chunk evidence %#v", occurrence, wantRef)
}
}
}
@@ -138,16 +125,17 @@ func TestNPCOccurrencePipelineSkipsConsumerWhenNPCProducerIsRejected(t *testing.
registries := productionNPCRegistries(t)
resolved := resolveNPCOccurrencePipeline(t, registries)
client := &npcOccurrenceLLMClient{rejectNPCs: true}
resolved.Steps[0].ArtifactLanes[0].ExtractControls.MaxAdditionalTurns = 0
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
if err == nil {
t.Fatalf("Run() output = %#v, want missing generated NPC producer error", output)
}
if client.requestCount(npcregistry.PromptID) != 1 {
t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount(npcregistry.PromptID))
if client.requestCount("npcs") != 1 {
t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount("npcs"))
}
if client.requestCount(occurrenceextract.PromptID) != 0 {
t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount(occurrenceextract.PromptID))
if client.requestCount("occurrences") != 0 {
t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount("occurrences"))
}
}
@@ -170,17 +158,6 @@ func loadNPCOccurrencePipelineConfig(t *testing.T) config.Config {
return loadIntegrationConfig(t, "testdata/dnd_npc_occurrences_pipeline.yml")
}
func normalizedLane(t *testing.T, output pipeline.RunOutput, laneID string) contracts.SerializedOutput {
t.Helper()
for _, serialized := range output.NormalizeOutputs {
if serialized.LaneID == laneID {
return serialized
}
}
t.Fatalf("normalized lanes = %#v, missing %q", output.NormalizeOutputs, laneID)
return contracts.SerializedOutput{}
}
func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) {
t.Helper()
if len(references) == 0 {
@@ -195,21 +172,21 @@ func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) {
type npcOccurrenceLLMClient struct {
mu sync.Mutex
requests []contracts.StructuredCompletionRequest
requests []promptkit.GenerateRequest
rejectNPCs bool
}
func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
func (client *npcOccurrenceLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
client.mu.Lock()
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
client.requests = append(client.requests, request)
client.mu.Unlock()
var payload any
switch request.PromptID {
case npcregistry.PromptID:
switch {
case nativeSchemaHasProperty(request, "npcs"):
if client.rejectNPCs {
payload = map[string]any{"npcs": []any{map[string]any{
"name": "", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
@@ -220,109 +197,123 @@ func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, re
map[string]any{"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
}}
}
case npcnormalize.PromptID:
case nativeSchemaHasProperty(request, "duplicate_groups"):
payload = map[string]any{"duplicate_groups": []any{}}
case occurrenceextract.PromptID:
case nativeSchemaHasProperty(request, "occurrences"):
payload = map[string]any{"occurrences": []any{
map[string]any{"npc_id": identity.DeriveID("Hooded Guard"), "name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}},
map[string]any{"name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}},
}}
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected occurrence prompt %q", request.PromptID)
return nil, fmt.Errorf("unexpected native occurrence response contract")
}
content, err := json.Marshal(payload)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate occurrence response: %w", err)
}
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "occurrence-fake"}, nil
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest {
func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
t.Helper()
client.mu.Lock()
defer client.mu.Unlock()
for _, request := range client.requests {
if request.PromptID == promptID {
if nativeSchemaHasProperty(request, property) {
return request
}
}
t.Fatalf("requests = %#v, missing prompt %q", client.requests, promptID)
return contracts.StructuredCompletionRequest{}
t.Fatalf("requests = %#v, missing prompt %q", client.requests, property)
return promptkit.GenerateRequest{}
}
func (client *npcOccurrenceLLMClient) requestCount(promptID string) int {
func (client *npcOccurrenceLLMClient) requestCount(property string) int {
client.mu.Lock()
defer client.mu.Unlock()
count := 0
for _, request := range client.requests {
if request.PromptID == promptID {
if nativeSchemaHasProperty(request, property) {
count++
}
}
return count
}
var _ contracts.StructuredLLMClient = (*npcOccurrenceLLMClient)(nil)
var _ promptkit.LLMClient = (*npcOccurrenceLLMClient)(nil)
type semanticNPCOccurrenceClient struct {
requests []contracts.StructuredCompletionRequest
npcCalls int
mu sync.Mutex
requests []promptkit.GenerateRequest
}
func (client *semanticNPCOccurrenceClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
func (client *semanticNPCOccurrenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.mu.Lock()
defer client.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
client.requests = append(client.requests, request)
var payload any
switch request.PromptID {
case npcregistry.PromptID:
client.npcCalls++
var unitID int
if nativeSchemaHasProperty(request, "npcs") || nativeSchemaHasProperty(request, "occurrences") {
content, err := nativeJSONInput(request, "units")
if err != nil {
return nil, err
}
var document struct {
Units []struct {
ID int `json:"id"`
} `json:"units"`
}
if err := json.Unmarshal(content, &document); err != nil {
return nil, err
}
if len(document.Units) != 1 || document.Units[0].ID <= 0 {
return nil, fmt.Errorf("semantic fixture requires one source-addressed unit: %s", content)
}
unitID = document.Units[0].ID
}
switch {
case nativeSchemaHasProperty(request, "npcs"):
name := "Mira Thorn"
if client.npcCalls == 2 {
if unitID == 2 {
name = "Mira Thorn, the Greencloak"
}
payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": client.npcCalls, "end_unit_id": client.npcCalls}}}}}
case npcnormalize.PromptID:
payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}}
case nativeSchemaHasProperty(request, "duplicate_groups"):
content := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`)
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
case occurrenceextract.PromptID:
payload = map[string]any{"occurrences": []any{map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}}}
return &promptkit.GenerateResponse{Content: string(content)}, nil
case nativeSchemaHasProperty(request, "occurrences"):
payload = map[string]any{"occurrences": []any{map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}}
default:
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected semantic pipeline prompt %q", request.PromptID)
return nil, fmt.Errorf("unexpected native occurrence response contract")
}
content, err := json.Marshal(payload)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
if err := json.Unmarshal(content, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
return &promptkit.GenerateResponse{Content: string(content)}, nil
}
func (client *semanticNPCOccurrenceClient) requestCount(promptID string) int {
func (client *semanticNPCOccurrenceClient) requestCount(property string) int {
count := 0
for _, request := range client.requests {
if request.PromptID == promptID {
if nativeSchemaHasProperty(request, property) {
count++
}
}
return count
}
func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest {
func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
t.Helper()
for _, request := range client.requests {
if request.PromptID == promptID {
if nativeSchemaHasProperty(request, property) {
return request
}
}
t.Fatalf("requests = %#v, missing %q", client.requests, promptID)
return contracts.StructuredCompletionRequest{}
t.Fatalf("requests = %#v, missing %q", client.requests, property)
return promptkit.GenerateRequest{}
}
var _ contracts.StructuredLLMClient = (*semanticNPCOccurrenceClient)(nil)
var _ promptkit.LLMClient = (*semanticNPCOccurrenceClient)(nil)

View File

@@ -95,3 +95,19 @@ func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string)
_, found := schema.Properties[property]
return found
}
func nativeJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) {
for _, message := range request.Prompt.Messages {
for offset := range len(message.Content) {
if message.Content[offset] != '{' {
continue
}
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
var object map[string]json.RawMessage
if decoder.Decode(&object) == nil && object[field] != nil {
return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil
}
}
}
return nil, fmt.Errorf("native request has no JSON input with field %q", field)
}

View File

@@ -57,3 +57,14 @@ func publishedLane(t *testing.T, output pipeline.RunOutput, laneID string) contr
t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID)
return contracts.OutputFile{}
}
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
t.Helper()
for _, file := range files {
if file.Name == name {
return append([]byte(nil), file.Bytes...)
}
}
t.Fatalf("output files = %#v, missing %q", files, name)
return nil
}

View File

@@ -1,10 +1,6 @@
version: 5
version: 6
output:
directory: ./notarius-output
cache:
chunk_plans:
mode: bypass
checkpoints: {}
debug:
directory: ./notarius-debug
pipelines: