Migrate ordered NPC and location handoff tests to native execution
This commit is contained in:
@@ -1711,6 +1711,20 @@ These selected integration tests, vet/race checks, all framework/configuration
|
||||
tests, and import-boundary tests pass. Remaining combat, location, occurrence,
|
||||
and grounded-pipeline fixtures still block full integration compilation.
|
||||
|
||||
NPC-occurrence and location handoff tests now run through native providers and
|
||||
published lane/evidence files. Retained contextual same-name location selection,
|
||||
names-only NPC projection, deterministic durable IDs, source chronology, earlier-
|
||||
producer requirements, and no consumer execution after strict producer rejection.
|
||||
The multi-chunk semantic NPC fixture now derives evidence from the actual rendered
|
||||
chunk rather than completion order, and verifies both downstream chunks with the
|
||||
canonical registry identity under ordinary parallel workers. Removed only the old
|
||||
generated-reference checkpoint assertion; existing native cache/eligibility tests
|
||||
own its replacement behavior. Updated occurrence test configurations to version 6
|
||||
and made temporary config cleanup test-owned. These selected integration tests,
|
||||
vet/repeated race checks, framework/configuration tests, and import-boundary tests
|
||||
pass. Only combat and grounded-pipeline legacy fixtures remain as integration
|
||||
compile blockers; CLI/dependency cleanup and complete validation are still pending.
|
||||
|
||||
**Goal:** delete the old architecture rather than leave dormant alternatives.
|
||||
|
||||
**Inspect/change:** all production/test callers of the feature roadmap's removal
|
||||
|
||||
@@ -26,11 +26,9 @@ func loadIntegrationConfigText(t *testing.T, content string) config.Config {
|
||||
return loadIntegrationConfig(t, path)
|
||||
}
|
||||
|
||||
func configFromYAML(content string) (config.Config, error) {
|
||||
directory, err := os.MkdirTemp("", "notarius-integration-config-")
|
||||
if err != nil {
|
||||
return config.Config{}, err
|
||||
}
|
||||
func configFromYAML(t *testing.T, content string) (config.Config, error) {
|
||||
t.Helper()
|
||||
directory := t.TempDir()
|
||||
path := filepath.Join(directory, "config.yml")
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
return config.Config{}, err
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
|
||||
func TestItemOccurrenceConfigurationRequiresEarlierItemRegistry(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
cfg, err := configFromYAML(`
|
||||
version: 5
|
||||
cfg, err := configFromYAML(t, `
|
||||
version: 6
|
||||
pipelines:
|
||||
items:
|
||||
input: {module: seriatim}
|
||||
@@ -41,8 +41,8 @@ pipelines:
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
}
|
||||
|
||||
broken, err := configFromYAML(strings.Replace(`
|
||||
version: 5
|
||||
broken, err := configFromYAML(t, strings.Replace(`
|
||||
version: 6
|
||||
pipelines:
|
||||
items:
|
||||
input: {module: seriatim}
|
||||
|
||||
@@ -10,15 +10,11 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
locationoccurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationoccurrences"
|
||||
locationcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
|
||||
locationoccurrences "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationoccurrences"
|
||||
locations "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/locationregistry"
|
||||
locationnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/locationregistry"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
|
||||
@@ -29,15 +25,12 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("run outputs = %#v rejected = %#v, want accepted registry and occurrences", output.NormalizeOutputs, output.Rejected)
|
||||
if len(output.OutputFiles) != 3 {
|
||||
t.Fatalf("files=%#v, want two lanes and evidence", output.OutputFiles)
|
||||
}
|
||||
|
||||
registryOutput := normalizedLane(t, output, "locations")
|
||||
if registryOutput.Artifact.Kind != dnd.LocationRegistryKind || registryOutput.Artifact.Schema.ID != locationcodec.SchemaID {
|
||||
t.Fatalf("registry envelope = %#v, want durable location artifact", registryOutput)
|
||||
}
|
||||
registry, err := locationcodec.New().Decode(registryOutput.Artifact.Content)
|
||||
registryOutput := publishedLane(t, output, "locations")
|
||||
registry, err := locationcodec.New().Decode(registryOutput.Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode(registry) error = %v", err)
|
||||
}
|
||||
@@ -45,11 +38,8 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
|
||||
t.Fatalf("registry = %#v, want same-name locations distinguished by source-derived IDs", registry)
|
||||
}
|
||||
|
||||
occurrenceOutput := normalizedLane(t, output, "occurrences")
|
||||
if occurrenceOutput.Artifact.Kind != dnd.LocationOccurrenceListKind || occurrenceOutput.Artifact.Schema.ID != locationoccurrencecodec.SchemaID {
|
||||
t.Fatalf("occurrence envelope = %#v, want durable occurrence artifact", occurrenceOutput)
|
||||
}
|
||||
occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Artifact.Content)
|
||||
occurrenceOutput := publishedLane(t, output, "occurrences")
|
||||
occurrences, err := locationoccurrencecodec.New().Decode(occurrenceOutput.Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode(occurrences) error = %v", err)
|
||||
}
|
||||
@@ -62,11 +52,16 @@ func TestLocationRegistryHandoffProducesOccurrencesAndEvidence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
request := client.requestFor(t, locationoccurrences.PromptID)
|
||||
registryInput := request.Inputs["location_registry"]
|
||||
if registryInput.MediaType != locationcodec.MediaType || strings.Contains(string(registryInput.Content), "source_refs") || strings.Contains(string(registryInput.Content), registry.Locations[0].ID) || strings.Contains(string(registryInput.Content), registry.Locations[1].ID) || !strings.Contains(string(registryInput.Content), "registry_refs") {
|
||||
t.Fatalf("occurrence registry input = %#v, want contextual selector projection", registryInput)
|
||||
request := client.requestFor(t, "occurrences")
|
||||
|
||||
registryInput, err := nativeJSONInput(request, "locations")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(registryInput), "source_refs") || strings.Contains(string(registryInput), registry.Locations[0].ID) || strings.Contains(string(registryInput), registry.Locations[1].ID) || !strings.Contains(string(registryInput), "registry_refs") {
|
||||
t.Fatalf("registry input=%s, want contextual source-free selectors", registryInput)
|
||||
}
|
||||
|
||||
contextArtifact := outputFileContent(t, output.OutputFiles, "evidence-context.json")
|
||||
evidence, err := evidencecontext.New().Decode(contextArtifact)
|
||||
if err != nil {
|
||||
@@ -92,8 +87,8 @@ func TestLocationOccurrenceConsumerDoesNotRunAfterRejectedRegistry(t *testing.T)
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want missing accepted location registry")
|
||||
}
|
||||
if client.requestCount(locationoccurrences.PromptID) != 0 {
|
||||
t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount(locationoccurrences.PromptID))
|
||||
if client.requestCount("occurrences") != 0 {
|
||||
t.Fatalf("occurrence requests = %d, want none after rejected registry", client.requestCount("occurrences"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,24 +101,6 @@ func TestLocationOccurrenceConfigurationRequiresEarlierRegistry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocationOccurrenceCheckpointTracksGeneratedRegistry(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
checkpoint := newGeneratedReferenceCheckpointLoader()
|
||||
_, err := runPreparedPipeline(t, registries, resolveLocationPipeline(t, registries), &locationHandoffLLMClient{}, pipeline.RunInput{
|
||||
RawInput: readNPCFixture(t),
|
||||
Checkpoint: checkpoint,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
for _, dependency := range checkpoint.extractDependencies("occurrences") {
|
||||
if dependency.Name == "generated-reference:location_registry:0" && strings.HasPrefix(dependency.Value, "sha256:") {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("occurrence checkpoint dependencies = %#v, want generated location registry fingerprint", checkpoint.extractDependencies("occurrences"))
|
||||
}
|
||||
|
||||
func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipeline.ResolvedPipeline {
|
||||
t.Helper()
|
||||
cfg := loadIntegrationConfigText(t, locationPipelineConfig)
|
||||
@@ -138,11 +115,7 @@ func resolveLocationPipeline(t *testing.T, registries pipeline.Registries) pipel
|
||||
return resolved
|
||||
}
|
||||
|
||||
const locationPipelineConfig = `version: 5
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: bypass
|
||||
checkpoints: {}
|
||||
const locationPipelineConfig = `version: 6
|
||||
pipelines:
|
||||
dnd-locations-fixture:
|
||||
input:
|
||||
@@ -175,28 +148,28 @@ pipelines:
|
||||
|
||||
type locationHandoffLLMClient struct {
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
requests []promptkit.GenerateRequest
|
||||
rejectRegistry bool
|
||||
}
|
||||
|
||||
func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
|
||||
func (client *locationHandoffLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
|
||||
client.requests = append(client.requests, request)
|
||||
client.mu.Unlock()
|
||||
var payload any
|
||||
switch request.PromptID {
|
||||
case locations.PromptID:
|
||||
switch {
|
||||
case nativeSchemaHasProperty(request, "locations"):
|
||||
name := "Moon Gate"
|
||||
if client.rejectRegistry {
|
||||
name = ""
|
||||
}
|
||||
payload = map[string]any{"locations": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}}
|
||||
case locationnormalize.PromptID:
|
||||
case nativeSchemaHasProperty(request, "duplicate_groups"):
|
||||
payload = map[string]any{"duplicate_groups": []any{}}
|
||||
case locationoccurrences.PromptID:
|
||||
case nativeSchemaHasProperty(request, "occurrences"):
|
||||
var projection struct {
|
||||
Locations []struct {
|
||||
Name string `json:"name"`
|
||||
@@ -206,48 +179,49 @@ func (client *locationHandoffLLMClient) CompleteStructured(ctx context.Context,
|
||||
} `json:"registry_refs"`
|
||||
} `json:"locations"`
|
||||
}
|
||||
if err := json.Unmarshal(request.Inputs["location_registry"].Content, &projection); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("decode location registry: %w", err)
|
||||
content, err := nativeJSONInput(request, "locations")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(content, &projection); err != nil {
|
||||
return nil, fmt.Errorf("decode location registry: %w", err)
|
||||
}
|
||||
if len(projection.Locations) != 2 {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations))
|
||||
return nil, fmt.Errorf("generated location registry has %d locations, want 2", len(projection.Locations))
|
||||
}
|
||||
payload = map[string]any{"occurrences": []any{map[string]any{"name": projection.Locations[0].Name, "registry_refs": projection.Locations[0].RegistryRefs, "kind": "visited", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}, map[string]any{"name": projection.Locations[1].Name, "registry_refs": projection.Locations[1].RegistryRefs, "kind": "mentioned", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}}}}
|
||||
default:
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", request.PromptID)
|
||||
return nil, fmt.Errorf("unexpected native location output contract")
|
||||
}
|
||||
content, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(content, output); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "location-handoff"}, nil
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
func (client *locationHandoffLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest {
|
||||
func (client *locationHandoffLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
|
||||
t.Helper()
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
if nativeSchemaHasProperty(request, property) {
|
||||
return request
|
||||
}
|
||||
}
|
||||
t.Fatalf("requests = %#v, missing %q", client.requests, promptID)
|
||||
return contracts.StructuredCompletionRequest{}
|
||||
t.Fatalf("requests = %#v, missing %q", client.requests, property)
|
||||
return promptkit.GenerateRequest{}
|
||||
}
|
||||
func (client *locationHandoffLLMClient) requestCount(promptID string) int {
|
||||
func (client *locationHandoffLLMClient) requestCount(property string) int {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
count := 0
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
if nativeSchemaHasProperty(request, property) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*locationHandoffLLMClient)(nil)
|
||||
var _ promptkit.LLMClient = (*locationHandoffLLMClient)(nil)
|
||||
|
||||
@@ -482,17 +482,6 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr
|
||||
return output
|
||||
}
|
||||
|
||||
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
|
||||
t.Helper()
|
||||
for _, file := range files {
|
||||
if file.Name == name {
|
||||
return append([]byte(nil), file.Bytes...)
|
||||
}
|
||||
}
|
||||
t.Fatalf("output files = %#v, missing %q", files, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
|
||||
t.Helper()
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
|
||||
@@ -5,20 +5,16 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
occurrenceextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcoccurrences"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.T) {
|
||||
@@ -30,24 +26,22 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("run outputs = %#v rejected = %#v, want NPC and occurrence artifacts", output.NormalizeOutputs, output.Rejected)
|
||||
if len(output.OutputFiles) != 2 {
|
||||
t.Fatalf("output files = %#v, want registry and occurrences", output.OutputFiles)
|
||||
}
|
||||
|
||||
request := client.requestFor(t, occurrenceextract.PromptID)
|
||||
request := client.requestFor(t, "occurrences")
|
||||
wantRegistry := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`
|
||||
if got := string(request.Inputs["npc_registry"].Content); got != wantRegistry {
|
||||
t.Fatalf("occurrence registry input = %s, want names-only projection %s", got, wantRegistry)
|
||||
registryInput, err := nativeJSONInput(request, "npcs")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if request.Inputs["npc_registry"].MediaType != npccodec.MediaType {
|
||||
t.Fatalf("occurrence registry media type = %q, want %q", request.Inputs["npc_registry"].MediaType, npccodec.MediaType)
|
||||
if string(registryInput) != wantRegistry {
|
||||
t.Fatalf("occurrence registry = %s, want names-only projection %s", registryInput, wantRegistry)
|
||||
}
|
||||
|
||||
serialized := normalizedLane(t, output, "occurrences")
|
||||
if serialized.Artifact.Schema.ID != occurrencecodec.SchemaID || serialized.Artifact.Schema.Version != occurrencecodec.SchemaVersion {
|
||||
t.Fatalf("occurrence artifact schema = %#v", serialized.Artifact.Schema)
|
||||
}
|
||||
occurrences, err := occurrencecodec.New().Decode(serialized.Artifact.Content)
|
||||
serialized := publishedLane(t, output, "occurrences")
|
||||
occurrences, err := occurrencecodec.New().Decode(serialized.Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode(occurrence output) error = %v", err)
|
||||
}
|
||||
@@ -68,7 +62,7 @@ func TestNPCOccurrencePipelineUsesAcceptedRegistryAndCurrentEvidence(t *testing.
|
||||
}
|
||||
|
||||
var durable map[string]json.RawMessage
|
||||
if err := json.Unmarshal(serialized.Artifact.Content, &durable); err != nil {
|
||||
if err := json.Unmarshal(serialized.Bytes, &durable); err != nil {
|
||||
t.Fatalf("unmarshal durable occurrence payload: %v", err)
|
||||
}
|
||||
if len(durable) != 1 || durable["occurrences"] == nil {
|
||||
@@ -90,46 +84,39 @@ func TestSemanticNPCNormalizationCrossesOrderedRegistryHandoff(t *testing.T) {
|
||||
if err != nil || len(warnings) != 0 {
|
||||
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
||||
}
|
||||
for stepIndex := range resolved.Steps {
|
||||
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
|
||||
resolved.Steps[stepIndex].ArtifactLanes[laneIndex].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
}
|
||||
}
|
||||
client := &semanticNPCOccurrenceClient{}
|
||||
raw := []byte(`{"metadata":{"id":"semantic-session","title":"Semantic NPC session"},"segments":[{"id":1,"start":0,"end":1,"speaker":"DM","text":"Mira Thorn enters."},{"id":2,"start":1,"end":2,"speaker":"DM","text":"Mira Thorn, the Greencloak, waves."}]}`)
|
||||
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if client.requestCount(npcregistry.PromptID) != 2 || client.requestCount(npcnormalize.PromptID) != 1 {
|
||||
if client.requestCount("npcs") != 2 || client.requestCount("duplicate_groups") != 1 {
|
||||
t.Fatalf("prompt requests = %#v, want two extraction calls and one document normalization call", client.requests)
|
||||
}
|
||||
npcOutput := normalizedLane(t, output, "npc_registry")
|
||||
npcsValue, err := npccodec.New().Decode(npcOutput.Artifact.Content)
|
||||
npcOutput := publishedLane(t, output, "npc_registry")
|
||||
npcsValue, err := npccodec.New().Decode(npcOutput.Bytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if npcOutput.StepID != "identify-npcs" || len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") {
|
||||
if len(npcsValue.NPCs) != 1 || npcsValue.NPCs[0].Name != "Mira Thorn" || npcsValue.NPCs[0].ID != identity.DeriveID("Mira Thorn") {
|
||||
t.Fatalf("NPC output = %#v / %#v, want canonical ordered producer artifact", npcOutput, npcsValue)
|
||||
}
|
||||
if refs := npcsValue.NPCs[0].SourceRefs; !reflect.DeepEqual(refs, []source.SourceRef{{SourceID: "semantic-session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "semantic-session", StartUnitID: 2, EndUnitID: 2}}) {
|
||||
t.Fatalf("NPC evidence = %#v, want original extraction evidence union", refs)
|
||||
}
|
||||
occurrenceOutput := normalizedLane(t, output, "occurrences")
|
||||
if occurrenceOutput.StepID != "extract-occurrences" {
|
||||
t.Fatalf("occurrence output step = %q, want ordered downstream step", occurrenceOutput.StepID)
|
||||
registryRequest := client.requestFor(t, "occurrences")
|
||||
registryInput, err := nativeJSONInput(registryRequest, "npcs")
|
||||
if err != nil || string(registryInput) != `{"npcs":[{"name":"Mira Thorn"}]}` {
|
||||
t.Fatalf("downstream registry = %s, error=%v", registryInput, err)
|
||||
}
|
||||
registryRequest := client.requestFor(t, occurrenceextract.PromptID)
|
||||
if got := string(registryRequest.Inputs["npc_registry"].Content); got != `{"npcs":[{"name":"Mira Thorn"}]}` {
|
||||
t.Fatalf("downstream registry = %s, want one canonical NPC name", got)
|
||||
occurrences, err := occurrencecodec.New().Decode(publishedLane(t, output, "occurrences").Bytes)
|
||||
if err != nil || len(occurrences.Occurrences) != 2 {
|
||||
t.Fatalf("downstream occurrences=%#v error=%v, want both chunks", occurrences, err)
|
||||
}
|
||||
manifestContent, err := json.Marshal(output.Manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"npc:sha256:", "Mira Thorn enters.", "Mira Thorn, the Greencloak, waves."} {
|
||||
if strings.Contains(string(manifestContent), forbidden) {
|
||||
t.Fatalf("manifest leaked private identity or transcript content %q: %s", forbidden, manifestContent)
|
||||
for index, occurrence := range occurrences.Occurrences {
|
||||
wantRef := source.SourceRef{SourceID: "semantic-session", StartUnitID: index + 1, EndUnitID: index + 1}
|
||||
if occurrence.Name != "Mira Thorn" || occurrence.NPCID != npcsValue.NPCs[0].ID || !reflect.DeepEqual(occurrence.SourceRefs, []source.SourceRef{wantRef}) {
|
||||
t.Fatalf("occurrence=%#v, want canonical identity and current chunk evidence %#v", occurrence, wantRef)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -138,16 +125,17 @@ func TestNPCOccurrencePipelineSkipsConsumerWhenNPCProducerIsRejected(t *testing.
|
||||
registries := productionNPCRegistries(t)
|
||||
resolved := resolveNPCOccurrencePipeline(t, registries)
|
||||
client := &npcOccurrenceLLMClient{rejectNPCs: true}
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractControls.MaxAdditionalTurns = 0
|
||||
|
||||
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
||||
if err == nil {
|
||||
t.Fatalf("Run() output = %#v, want missing generated NPC producer error", output)
|
||||
}
|
||||
if client.requestCount(npcregistry.PromptID) != 1 {
|
||||
t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount(npcregistry.PromptID))
|
||||
if client.requestCount("npcs") != 1 {
|
||||
t.Fatalf("NPC requests = %d, want rejected producer", client.requestCount("npcs"))
|
||||
}
|
||||
if client.requestCount(occurrenceextract.PromptID) != 0 {
|
||||
t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount(occurrenceextract.PromptID))
|
||||
if client.requestCount("occurrences") != 0 {
|
||||
t.Fatalf("occurrence requests = %d, want none after rejected producer", client.requestCount("occurrences"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,17 +158,6 @@ func loadNPCOccurrencePipelineConfig(t *testing.T) config.Config {
|
||||
return loadIntegrationConfig(t, "testdata/dnd_npc_occurrences_pipeline.yml")
|
||||
}
|
||||
|
||||
func normalizedLane(t *testing.T, output pipeline.RunOutput, laneID string) contracts.SerializedOutput {
|
||||
t.Helper()
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
if serialized.LaneID == laneID {
|
||||
return serialized
|
||||
}
|
||||
}
|
||||
t.Fatalf("normalized lanes = %#v, missing %q", output.NormalizeOutputs, laneID)
|
||||
return contracts.SerializedOutput{}
|
||||
}
|
||||
|
||||
func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) {
|
||||
t.Helper()
|
||||
if len(references) == 0 {
|
||||
@@ -195,21 +172,21 @@ func assertOccurrenceEvidence(t *testing.T, references []source.SourceRef) {
|
||||
|
||||
type npcOccurrenceLLMClient struct {
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
requests []promptkit.GenerateRequest
|
||||
rejectNPCs bool
|
||||
}
|
||||
|
||||
func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
func (client *npcOccurrenceLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
|
||||
client.requests = append(client.requests, request)
|
||||
client.mu.Unlock()
|
||||
|
||||
var payload any
|
||||
switch request.PromptID {
|
||||
case npcregistry.PromptID:
|
||||
switch {
|
||||
case nativeSchemaHasProperty(request, "npcs"):
|
||||
if client.rejectNPCs {
|
||||
payload = map[string]any{"npcs": []any{map[string]any{
|
||||
"name": "", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
@@ -220,109 +197,123 @@ func (client *npcOccurrenceLLMClient) CompleteStructured(ctx context.Context, re
|
||||
map[string]any{"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
|
||||
}}
|
||||
}
|
||||
case npcnormalize.PromptID:
|
||||
case nativeSchemaHasProperty(request, "duplicate_groups"):
|
||||
payload = map[string]any{"duplicate_groups": []any{}}
|
||||
case occurrenceextract.PromptID:
|
||||
case nativeSchemaHasProperty(request, "occurrences"):
|
||||
payload = map[string]any{"occurrences": []any{
|
||||
map[string]any{"npc_id": identity.DeriveID("Hooded Guard"), "name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
|
||||
map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}},
|
||||
map[string]any{"name": "Hooded Guard", "kind": "noncombat_presence", "source_refs": []any{map[string]int{"start_unit_id": 3, "end_unit_id": 3}}},
|
||||
map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 2, "end_unit_id": 2}}},
|
||||
}}
|
||||
default:
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected occurrence prompt %q", request.PromptID)
|
||||
return nil, fmt.Errorf("unexpected native occurrence response contract")
|
||||
}
|
||||
content, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate occurrence response: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "occurrence-fake"}, nil
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest {
|
||||
func (client *npcOccurrenceLLMClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
|
||||
t.Helper()
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
if nativeSchemaHasProperty(request, property) {
|
||||
return request
|
||||
}
|
||||
}
|
||||
t.Fatalf("requests = %#v, missing prompt %q", client.requests, promptID)
|
||||
return contracts.StructuredCompletionRequest{}
|
||||
t.Fatalf("requests = %#v, missing prompt %q", client.requests, property)
|
||||
return promptkit.GenerateRequest{}
|
||||
}
|
||||
|
||||
func (client *npcOccurrenceLLMClient) requestCount(promptID string) int {
|
||||
func (client *npcOccurrenceLLMClient) requestCount(property string) int {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
count := 0
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
if nativeSchemaHasProperty(request, property) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*npcOccurrenceLLMClient)(nil)
|
||||
var _ promptkit.LLMClient = (*npcOccurrenceLLMClient)(nil)
|
||||
|
||||
type semanticNPCOccurrenceClient struct {
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
npcCalls int
|
||||
mu sync.Mutex
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (client *semanticNPCOccurrenceClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
|
||||
func (client *semanticNPCOccurrenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.requests = append(client.requests, request)
|
||||
var payload any
|
||||
switch request.PromptID {
|
||||
case npcregistry.PromptID:
|
||||
client.npcCalls++
|
||||
var unitID int
|
||||
if nativeSchemaHasProperty(request, "npcs") || nativeSchemaHasProperty(request, "occurrences") {
|
||||
content, err := nativeJSONInput(request, "units")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var document struct {
|
||||
Units []struct {
|
||||
ID int `json:"id"`
|
||||
} `json:"units"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &document); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(document.Units) != 1 || document.Units[0].ID <= 0 {
|
||||
return nil, fmt.Errorf("semantic fixture requires one source-addressed unit: %s", content)
|
||||
}
|
||||
unitID = document.Units[0].ID
|
||||
}
|
||||
switch {
|
||||
case nativeSchemaHasProperty(request, "npcs"):
|
||||
name := "Mira Thorn"
|
||||
if client.npcCalls == 2 {
|
||||
if unitID == 2 {
|
||||
name = "Mira Thorn, the Greencloak"
|
||||
}
|
||||
payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": client.npcCalls, "end_unit_id": client.npcCalls}}}}}
|
||||
case npcnormalize.PromptID:
|
||||
payload = map[string]any{"npcs": []any{map[string]any{"name": name, "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}}
|
||||
case nativeSchemaHasProperty(request, "duplicate_groups"):
|
||||
content := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`)
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
case occurrenceextract.PromptID:
|
||||
payload = map[string]any{"occurrences": []any{map[string]any{"npc_id": identity.DeriveID("Mira Thorn"), "name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}}}}}
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
case nativeSchemaHasProperty(request, "occurrences"):
|
||||
payload = map[string]any{"occurrences": []any{map[string]any{"name": "Mira Thorn", "kind": "dialogue", "source_refs": []any{map[string]int{"start_unit_id": unitID, "end_unit_id": unitID}}}}}
|
||||
default:
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected semantic pipeline prompt %q", request.PromptID)
|
||||
return nil, fmt.Errorf("unexpected native occurrence response contract")
|
||||
}
|
||||
content, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
func (client *semanticNPCOccurrenceClient) requestCount(promptID string) int {
|
||||
func (client *semanticNPCOccurrenceClient) requestCount(property string) int {
|
||||
count := 0
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
if nativeSchemaHasProperty(request, property) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, promptID string) contracts.StructuredCompletionRequest {
|
||||
func (client *semanticNPCOccurrenceClient) requestFor(t *testing.T, property string) promptkit.GenerateRequest {
|
||||
t.Helper()
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
if nativeSchemaHasProperty(request, property) {
|
||||
return request
|
||||
}
|
||||
}
|
||||
t.Fatalf("requests = %#v, missing %q", client.requests, promptID)
|
||||
return contracts.StructuredCompletionRequest{}
|
||||
t.Fatalf("requests = %#v, missing %q", client.requests, property)
|
||||
return promptkit.GenerateRequest{}
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*semanticNPCOccurrenceClient)(nil)
|
||||
var _ promptkit.LLMClient = (*semanticNPCOccurrenceClient)(nil)
|
||||
|
||||
@@ -95,3 +95,19 @@ func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string)
|
||||
_, found := schema.Properties[property]
|
||||
return found
|
||||
}
|
||||
|
||||
func nativeJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) {
|
||||
for _, message := range request.Prompt.Messages {
|
||||
for offset := range len(message.Content) {
|
||||
if message.Content[offset] != '{' {
|
||||
continue
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
|
||||
var object map[string]json.RawMessage
|
||||
if decoder.Decode(&object) == nil && object[field] != nil {
|
||||
return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("native request has no JSON input with field %q", field)
|
||||
}
|
||||
|
||||
@@ -57,3 +57,14 @@ func publishedLane(t *testing.T, output pipeline.RunOutput, laneID string) contr
|
||||
t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID)
|
||||
return contracts.OutputFile{}
|
||||
}
|
||||
|
||||
func outputFileContent(t *testing.T, files []contracts.OutputFile, name string) []byte {
|
||||
t.Helper()
|
||||
for _, file := range files {
|
||||
if file.Name == name {
|
||||
return append([]byte(nil), file.Bytes...)
|
||||
}
|
||||
}
|
||||
t.Fatalf("output files = %#v, missing %q", files, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
version: 5
|
||||
version: 6
|
||||
output:
|
||||
directory: ./notarius-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: bypass
|
||||
checkpoints: {}
|
||||
debug:
|
||||
directory: ./notarius-debug
|
||||
pipelines:
|
||||
|
||||
Reference in New Issue
Block a user