Exercise spell and NPC integration through native providers and published artifacts
This commit is contained in:
@@ -6,22 +6,19 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
|
||||
npcshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/shape"
|
||||
npcregistrysourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/npcregistry/source_refs"
|
||||
genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T) {
|
||||
@@ -57,17 +54,15 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalize outputs = %d, want one NPC output; rejected=%#v", len(output.NormalizeOutputs), output.Rejected)
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("output files = %#v, want NPC registry", output.OutputFiles)
|
||||
}
|
||||
serialized := output.NormalizeOutputs[0]
|
||||
if serialized.LaneID != "npc_registry" || serialized.NormalizerKey != npcregistry.Key || serialized.Artifact.Schema.ID != npccodec.SchemaID || serialized.Artifact.Schema.Version != npccodec.SchemaVersion {
|
||||
t.Fatalf("serialized output = %#v, want durable NPC lane schema", serialized)
|
||||
}
|
||||
value, err := npccodec.New().Decode(serialized.Artifact.Content)
|
||||
file := publishedLane(t, output, "npc_registry")
|
||||
value, err := npccodec.New().Decode(file.Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode(output) error = %v, want durable NPC payload", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(value.NPCs) != 2 {
|
||||
t.Fatalf("NPC output = %#v, want repeated name consolidated and group/PC omitted", value.NPCs)
|
||||
}
|
||||
@@ -88,29 +83,12 @@ func TestRunnerProcessesSeriatimInputWithProductionDNDNPCPipeline(t *testing.T)
|
||||
if !hasNPCWarning(output.Diagnostics, "duplicate_npc_collapsed") {
|
||||
t.Fatalf("diagnostics = %#v, want name-only consolidation advisory", output.Diagnostics)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" || len(output.Manifest.ArtifactLanes) != 1 {
|
||||
t.Fatalf("manifest = %#v, want approved NPC lane", output.Manifest)
|
||||
if file.ContentType != npccodec.MediaType {
|
||||
t.Fatalf("NPC content type = %q, want %q", file.ContentType, npccodec.MediaType)
|
||||
}
|
||||
lane := output.Manifest.ArtifactLanes[0]
|
||||
if lane.ID != "npc_registry" || lane.Extractor != npcregistry.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != npcregistry.Key {
|
||||
t.Fatalf("manifest lane = %#v, want NPC production composition", lane)
|
||||
}
|
||||
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
|
||||
if !ok || normalizerMetadata["identity_policy"] != identity.Policy || normalizerMetadata["normalization_policy"] != npcnormalize.NormalizationPolicy || normalizerMetadata["prompt_id"] != npcnormalize.PromptID || normalizerMetadata["response_schema_id"] != semanticreconcile.ResponseSchemaID {
|
||||
t.Fatalf("normalizer metadata = %#v, want identity and normalization policies", lane.Metadata)
|
||||
}
|
||||
var npcOutputFile *contracts.OutputFile
|
||||
for index := range output.OutputFiles {
|
||||
if output.OutputFiles[index].Name == "lanes/npc_registry.json" {
|
||||
npcOutputFile = &output.OutputFiles[index]
|
||||
break
|
||||
}
|
||||
}
|
||||
if npcOutputFile == nil || npcOutputFile.ContentType != npccodec.MediaType {
|
||||
t.Fatalf("output files = %#v, want JSON NPC lane file", output.OutputFiles)
|
||||
}
|
||||
if len(client.requests) != 2 || client.requests[0].PromptID != npcregistry.PromptID || client.requests[1].PromptID != npcnormalize.PromptID {
|
||||
t.Fatalf("LLM requests = %#v, want extraction and normalization prompt requests", client.requests)
|
||||
|
||||
if len(client.requests) != 2 || client.extractions != 1 || client.normalizations != 1 {
|
||||
t.Fatalf("native requests=%d extraction=%d normalization=%d", len(client.requests), client.extractions, client.normalizations)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,54 +104,45 @@ func TestProductionNPCPipelineRoutesSemanticCandidatesToDeterministicValidators(
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response []byte
|
||||
reasonCode string
|
||||
validatorName string
|
||||
}{
|
||||
{
|
||||
name: "blank string",
|
||||
response: []byte(`{"npcs":[{"name":"","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
||||
reasonCode: npcshape.ReasonCode,
|
||||
validatorName: npcshape.Key,
|
||||
},
|
||||
{
|
||||
name: "empty evidence",
|
||||
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[]}]}`),
|
||||
reasonCode: npcshape.ReasonCode,
|
||||
validatorName: npcshape.Key,
|
||||
},
|
||||
{
|
||||
name: "nonpositive unit candidate",
|
||||
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`),
|
||||
reasonCode: npcregistrysourcerefs.ReasonCode,
|
||||
validatorName: npcregistrysourcerefs.Key,
|
||||
},
|
||||
{
|
||||
name: "unknown unit candidate",
|
||||
response: []byte(`{"npcs":[{"name":"Mira Thorn","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`),
|
||||
reasonCode: npcregistrysourcerefs.ReasonCode,
|
||||
validatorName: npcregistrysourcerefs.Key,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &fakeNPCProductionLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}}
|
||||
resolved := effective.ResolvedPipeline
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), test.validatorName) {
|
||||
t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName)
|
||||
}
|
||||
if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted rejection", len(client.requests), output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
rejection := output.Rejected[0]
|
||||
if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 {
|
||||
t.Fatalf("rejection = %#v, want exhausted %s rejection", rejection, test.validatorName)
|
||||
if client.extractions != 3 || len(output.OutputFiles) != 0 {
|
||||
t.Fatalf("native extraction calls=%d files=%#v, want exhausted rejection", client.extractions, output.OutputFiles)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) {
|
||||
func TestProductionNPCNormalizationUsesFinalSafeProposal(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
baseResponse := npcProductionResponse{NPCs: []npcProductionRecord{
|
||||
{Name: "Mira", SourceRefs: []npcProductionSourceRef{{StartUnitID: 1, EndUnitID: 1}}},
|
||||
@@ -184,20 +153,24 @@ func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) {
|
||||
safe := []byte(`{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
retries int
|
||||
responses [][]byte
|
||||
wantCalls int
|
||||
wantExhaustion bool
|
||||
name string
|
||||
turns int
|
||||
mode contracts.ExecutionMode
|
||||
responses [][]byte
|
||||
wantCalls int
|
||||
wantFallback bool
|
||||
wantError bool
|
||||
}{
|
||||
{name: "default retry budget", responses: [][]byte{partial}, wantCalls: 1, wantExhaustion: true},
|
||||
{name: "later complete proposal", retries: 1, responses: [][]byte{partial, safe}, wantCalls: 2},
|
||||
{name: "permissive final safe groups", mode: contracts.ModePermissive, responses: [][]byte{partial}, wantCalls: 1, wantFallback: true},
|
||||
{name: "later complete proposal", mode: contracts.ModeStrict, turns: 1, responses: [][]byte{partial, safe}, wantCalls: 2},
|
||||
{name: "strict final rejection", mode: contracts.ModeStrict, responses: [][]byte{partial}, wantCalls: 1, wantError: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := loadNPCPipelineConfig(t)
|
||||
profile := cfg.Pipelines["dnd-npc-registry-fixture"]
|
||||
lane := profile.Artifacts["npc_registry"]
|
||||
lane.Normalize.Retries = test.retries
|
||||
lane.Normalize.MaxAdditionalTurns = &test.turns
|
||||
lane.Normalize.Mode = &test.mode
|
||||
profile.Artifacts["npc_registry"] = lane
|
||||
cfg.Pipelines["dnd-npc-registry-fixture"] = profile
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-npc-registry-fixture", Catalog: moduleCatalog(registries)})
|
||||
@@ -206,14 +179,28 @@ func TestProductionNPCNormalizationRetryUsesFinalSafeProposal(t *testing.T) {
|
||||
}
|
||||
client := &fakeNPCProductionLLMClient{response: baseResponse, normalizeResponses: test.responses}
|
||||
output, err := runPreparedPipeline(t, registries, effective.ResolvedPipeline, client, pipeline.RunInput{RawInput: readNPCFixture(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
if (err != nil) != test.wantError {
|
||||
t.Fatalf("Run() error=%v, want error %t", err, test.wantError)
|
||||
}
|
||||
if client.requestCount(npcnormalize.PromptID) != test.wantCalls || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalization calls = %d output = %#v, want %d accepted result", client.requestCount(npcnormalize.PromptID), output.NormalizeOutputs, test.wantCalls)
|
||||
if client.normalizations != test.wantCalls {
|
||||
t.Fatalf("normalization calls=%d, want %d", client.normalizations, test.wantCalls)
|
||||
}
|
||||
if got := hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted); got != test.wantExhaustion {
|
||||
t.Fatalf("diagnostics = %#v, exhaustion = %t, want %t", output.Diagnostics, got, test.wantExhaustion)
|
||||
if test.wantError {
|
||||
if len(output.OutputFiles) != 0 {
|
||||
t.Fatal("strict failure published files")
|
||||
}
|
||||
return
|
||||
}
|
||||
wantFallbacks := 0
|
||||
if test.wantFallback {
|
||||
wantFallbacks = 1
|
||||
}
|
||||
if output.FallbackCount != wantFallbacks || hasWarningReason(output.Diagnostics, npcnormalize.ReasonCodeNPCSemanticReconciliationExhausted) != test.wantFallback {
|
||||
t.Fatalf("outcome=%#v, want fallback %t", output, test.wantFallback)
|
||||
}
|
||||
value, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes)
|
||||
if err != nil || len(value.NPCs) != 2 || value.NPCs[0].Name != "Mira Thorn" || len(value.NPCs[0].SourceRefs) != 2 || value.NPCs[1].Name != "Hooded Guard" {
|
||||
t.Fatalf("final safe registry=%#v error=%v", value, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -237,94 +224,49 @@ type fakeNPCProductionLLMClient struct {
|
||||
response npcProductionResponse
|
||||
rawResponses [][]byte
|
||||
normalizeResponses [][]byte
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
requests []promptkit.GenerateRequest
|
||||
mu sync.Mutex
|
||||
extractions int
|
||||
normalizations int
|
||||
}
|
||||
|
||||
func (client *fakeNPCProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||
func (client *fakeNPCProductionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.requests = append(client.requests, snapshot)
|
||||
client.requests = append(client.requests, request)
|
||||
var content []byte
|
||||
switch req.PromptID {
|
||||
case npcregistry.PromptID:
|
||||
if client.rawResponses != nil {
|
||||
index := client.requestCount(npcregistry.PromptID) - 1
|
||||
if index >= len(client.rawResponses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC response %d", index)
|
||||
if nativeSchemaHasProperty(request, "duplicate_groups") {
|
||||
index := client.normalizations
|
||||
client.normalizations++
|
||||
content = []byte(`{"duplicate_groups":[]}`)
|
||||
if client.normalizeResponses != nil {
|
||||
if index >= len(client.normalizeResponses) {
|
||||
return nil, fmt.Errorf("missing NPC normalization response %d", index)
|
||||
}
|
||||
content = append([]byte(nil), client.rawResponses[index]...)
|
||||
content = client.normalizeResponses[index]
|
||||
}
|
||||
} else if nativeSchemaHasProperty(request, "npcs") {
|
||||
index := client.extractions
|
||||
client.extractions++
|
||||
if client.rawResponses != nil {
|
||||
if index >= len(client.rawResponses) {
|
||||
return nil, fmt.Errorf("missing NPC extraction response %d", index)
|
||||
}
|
||||
content = client.rawResponses[index]
|
||||
} else {
|
||||
var err error
|
||||
content, err = json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
case npcnormalize.PromptID:
|
||||
content = []byte(`{"duplicate_groups":[]}`)
|
||||
if client.normalizeResponses != nil {
|
||||
index := client.requestCount(npcnormalize.PromptID) - 1
|
||||
if index >= len(client.normalizeResponses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake NPC normalization response %d", index)
|
||||
}
|
||||
content = append([]byte(nil), client.normalizeResponses[index]...)
|
||||
}
|
||||
default:
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected fake NPC prompt %q", req.PromptID)
|
||||
}
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate NPC structured target: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
func (client *fakeNPCProductionLLMClient) requestCount(promptID string) int {
|
||||
count := 0
|
||||
for _, request := range client.requests {
|
||||
if request.PromptID == promptID {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func productionNPCRegistries(t *testing.T) pipeline.Registries {
|
||||
t.Helper()
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
for _, registration := range []struct {
|
||||
name string
|
||||
fn func(pipeline.Registries) error
|
||||
}{
|
||||
{name: "generic", fn: genericregister.Register},
|
||||
{name: "seriatim", fn: seriatimregister.Register},
|
||||
{name: "dnd", fn: dndregister.Register},
|
||||
} {
|
||||
if err := registration.fn(registries); err != nil {
|
||||
t.Fatalf("register %s modules: %v", registration.name, err)
|
||||
}
|
||||
}
|
||||
return registries
|
||||
}
|
||||
|
||||
func moduleCatalog(registries pipeline.Registries) pipeline.ModuleCatalog {
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, ArtifactEvidence: registries.ArtifactEvidence,
|
||||
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
|
||||
} else {
|
||||
return nil, fmt.Errorf("unexpected NPC provider output contract")
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
func loadNPCPipelineConfig(t *testing.T) config.Config {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
@@ -107,7 +108,7 @@ func dndCapabilityCatalog(t *testing.T, inputSpec, extractorSpec pipeline.Module
|
||||
}
|
||||
|
||||
outputs := pipeline.NewOutputEncoderRegistry()
|
||||
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) { return dndSpellsOutput{}, nil }); err != nil {
|
||||
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) { return jsonoutput.New(), nil }); err != nil {
|
||||
t.Fatalf("register capability output: %v", err)
|
||||
}
|
||||
|
||||
@@ -204,7 +205,7 @@ func dndSpellsTestCatalog(t *testing.T, specs dndSpellsCatalogSpecs) pipeline.Mo
|
||||
Stage: pipeline.StageOutput,
|
||||
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||
}, func() (contracts.OutputEncoder, error) {
|
||||
return dndSpellsOutput{}, nil
|
||||
return jsonoutput.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
@@ -226,7 +227,7 @@ type configExtractor struct{ key string }
|
||||
|
||||
func (extractor configExtractor) Key() string { return extractor.key }
|
||||
func (configExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
func (configExtractor) Extract(context.Context, contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, nil
|
||||
}
|
||||
|
||||
@@ -264,20 +265,6 @@ func (dndSpellsChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (c
|
||||
}, nil
|
||||
}
|
||||
|
||||
type dndSpellsOutput struct{}
|
||||
|
||||
func (dndSpellsOutput) Key() string {
|
||||
return pipeline.DefaultOutputModule
|
||||
}
|
||||
|
||||
func (dndSpellsOutput) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{
|
||||
Files: []contracts.OutputFile{
|
||||
{Name: "output.json", ContentType: "application/json", Bytes: []byte(`{"encoded":true}`)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func withoutCapability(capabilities []string, capability string) []string {
|
||||
filtered := make([]string, 0, len(capabilities))
|
||||
for _, candidate := range capabilities {
|
||||
@@ -290,5 +277,5 @@ func withoutCapability(capabilities []string, capability string) []string {
|
||||
|
||||
var (
|
||||
_ contracts.Chunker = dndSpellsChunker{}
|
||||
_ contracts.OutputEncoder = dndSpellsOutput{}
|
||||
_ contracts.OutputEncoder = jsonoutput.New()
|
||||
)
|
||||
|
||||
@@ -4,8 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type extractionResponse struct {
|
||||
@@ -27,16 +28,22 @@ type fakeSpellsLLMClient struct {
|
||||
response extractionResponse
|
||||
responses []extractionResponse
|
||||
rawResponses [][]byte
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
requests []promptkit.GenerateRequest
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
func (client *fakeSpellsLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.requests = append(client.requests, req)
|
||||
var content []byte
|
||||
if client.rawResponses != nil {
|
||||
index := len(client.requests) - 1
|
||||
if index >= len(client.rawResponses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
|
||||
return nil, fmt.Errorf("missing fake response %d", index)
|
||||
}
|
||||
content = append([]byte(nil), client.rawResponses[index]...)
|
||||
} else {
|
||||
@@ -44,20 +51,17 @@ func (client *fakeSpellsLLMClient) CompleteStructured(_ context.Context, req con
|
||||
if client.responses != nil {
|
||||
index := len(client.requests) - 1
|
||||
if index >= len(client.responses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
|
||||
return nil, fmt.Errorf("missing fake response %d", index)
|
||||
}
|
||||
response = client.responses[index]
|
||||
}
|
||||
var err error
|
||||
content, err = json.Marshal(response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured target: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
func responseSourceRefs(startUnitID int, endUnitID int) []spellSourceRefResponse {
|
||||
@@ -68,22 +72,3 @@ func responseSourceRefs(startUnitID int, endUnitID int) []spellSourceRefResponse
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
if len(req.Vars) == 0 {
|
||||
req.Vars = nil
|
||||
return req
|
||||
}
|
||||
vars := make(map[string]any, len(req.Vars))
|
||||
for key, value := range req.Vars {
|
||||
vars[key] = value
|
||||
}
|
||||
req.Vars = vars
|
||||
return req
|
||||
}
|
||||
|
||||
@@ -17,11 +17,12 @@ import (
|
||||
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
|
||||
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient contracts.StructuredLLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) {
|
||||
func runPreparedPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, llmClient promptkit.LLMClient, input pipeline.RunInput) (pipeline.RunOutput, error) {
|
||||
t.Helper()
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, llmClient)})
|
||||
if err != nil {
|
||||
return pipeline.RunOutput{}, err
|
||||
}
|
||||
@@ -57,14 +58,13 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||
}
|
||||
serializedOutput := output.NormalizeOutputs[0]
|
||||
if serializedOutput.LaneID != "spells" || serializedOutput.Artifact.Schema.ID != spellcodec.SchemaID || serializedOutput.Artifact.Schema.Version != spellcodec.SchemaVersion {
|
||||
t.Fatalf("serialized output envelope = %#v, want durable spell artifact schema on spells lane", serializedOutput)
|
||||
if output.OutputFiles[0].Name != "lanes/spells.json" {
|
||||
t.Fatalf("spell output = %#v", output.OutputFiles)
|
||||
}
|
||||
response := decodeRunnerSpellResponse(t, serializedOutput.Artifact.Content)
|
||||
response := decodeRunnerSpellResponse(t, output.OutputFiles[0].Bytes)
|
||||
if len(response.SpellCasts) != 2 {
|
||||
t.Fatalf("len(spell_casts) = %d, want 2", len(response.SpellCasts))
|
||||
}
|
||||
@@ -84,31 +84,6 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
if output.Manifest.InputModule != transcript.Key {
|
||||
t.Fatalf("manifest input module = %q, want %q", output.Manifest.InputModule, transcript.Key)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" {
|
||||
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
|
||||
}
|
||||
if len(output.Manifest.ArtifactLanes) != 1 {
|
||||
t.Fatalf("len(ArtifactLanes) = %d, want 1", len(output.Manifest.ArtifactLanes))
|
||||
}
|
||||
lane := output.Manifest.ArtifactLanes[0]
|
||||
if lane.ID != "spells" || lane.Extractor != spells.Key {
|
||||
t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane)
|
||||
}
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("manifest lane metadata = %#v, want extractor metadata", lane.Metadata)
|
||||
}
|
||||
if extractorMetadata["prompt_id"] != spells.PromptID ||
|
||||
extractorMetadata["response_schema_key"] != string(spells.ResponseSchemaKey) ||
|
||||
extractorMetadata["response_schema_name"] != spells.ResponseSchemaName {
|
||||
t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata)
|
||||
}
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||
}
|
||||
if output.OutputFiles[0].ContentType != "application/json" {
|
||||
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
||||
}
|
||||
@@ -140,24 +115,17 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
|
||||
}
|
||||
if len(output.Manifest.References) != 2 {
|
||||
t.Fatalf("manifest references = %#v, want party and glossary provenance", output.Manifest.References)
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||
}
|
||||
if len(llmClient.requests) != 1 {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
|
||||
}
|
||||
request := llmClient.requests[0]
|
||||
if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion {
|
||||
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion)
|
||||
}
|
||||
if got := string(request.Inputs["party"].Content); got != "Aria: party cleric\nBorin: fighter" {
|
||||
t.Fatalf("party input = %q, want reference text", got)
|
||||
}
|
||||
if got := string(request.Inputs["glossary"].Content); got != "Fire Bolt: evocation cantrip" {
|
||||
t.Fatalf("glossary input = %q, want reference text", got)
|
||||
for _, value := range []string{"Aria: party cleric\nBorin: fighter", "Fire Bolt: evocation cantrip"} {
|
||||
if !strings.Contains(nativePromptText(request), value) {
|
||||
t.Fatalf("rendered prompt omitted %q", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,7 +137,7 @@ func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidator
|
||||
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 100}},
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"spells": {
|
||||
Extract: pipeline.ModuleBinding{Module: spells.Key, Retries: 2},
|
||||
Extract: pipeline.ModuleBinding{Module: spells.Key},
|
||||
Normalize: pipeline.Binding(spellnormalize.Key),
|
||||
},
|
||||
},
|
||||
@@ -184,48 +152,38 @@ func TestProductionSpellPipelineRoutesSemanticCandidatesToDeterministicValidator
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response []byte
|
||||
reasonCode string
|
||||
validatorName string
|
||||
}{
|
||||
{
|
||||
name: "blank string",
|
||||
response: []byte(`{"spell_casts":[{"caster":"","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`),
|
||||
reasonCode: spellshape.ReasonCode,
|
||||
validatorName: spellshape.Key,
|
||||
},
|
||||
{
|
||||
name: "empty evidence",
|
||||
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[]}]}`),
|
||||
reasonCode: spellshape.ReasonCode,
|
||||
validatorName: spellshape.Key,
|
||||
},
|
||||
{
|
||||
name: "nonpositive unit candidate",
|
||||
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":0,"end_unit_id":1}]}]}`),
|
||||
reasonCode: spellsourcerefs.ReasonCode,
|
||||
validatorName: spellsourcerefs.Key,
|
||||
},
|
||||
{
|
||||
name: "unknown unit candidate",
|
||||
response: []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":99,"end_unit_id":99}]}]}`),
|
||||
reasonCode: spellsourcerefs.ReasonCode,
|
||||
validatorName: spellsourcerefs.Key,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &fakeSpellsLLMClient{rawResponses: [][]byte{test.response, test.response, test.response}}
|
||||
resolved := effective.ResolvedPipeline
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
output, err := runPreparedPipeline(t, registries, resolved, client, pipeline.RunInput{RawInput: readDNDSpellsFixture(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
|
||||
if err == nil || !strings.Contains(err.Error(), test.validatorName) {
|
||||
t.Fatalf("Run() error = %v, want scoped %s rejection", err, test.validatorName)
|
||||
}
|
||||
if len(client.requests) != 3 || len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("LLM requests = %d rejected = %#v normalized = %#v, want exhausted rejection", len(client.requests), output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
rejection := output.Rejected[0]
|
||||
if rejection.ReasonCode != test.reasonCode || rejection.ValidatorName != test.validatorName || rejection.AttemptCount != 3 {
|
||||
t.Fatalf("rejection = %#v, want exhausted %s rejection", rejection, test.validatorName)
|
||||
if len(client.requests) != 3 || len(output.OutputFiles) != 0 {
|
||||
t.Fatalf("requests=%d files=%#v, want exhausted rejection", len(client.requests), output.OutputFiles)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -22,7 +22,12 @@ func nativeTestEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...st
|
||||
for index, id := range profileIDs {
|
||||
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false}
|
||||
}
|
||||
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{"profiles": profiles}})
|
||||
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{
|
||||
"profiles": profiles,
|
||||
"collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{
|
||||
{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false},
|
||||
}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -72,3 +77,21 @@ func nativePromptText(request promptkit.GenerateRequest) string {
|
||||
}
|
||||
return strings.Join(content, "\n")
|
||||
}
|
||||
|
||||
func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string) bool {
|
||||
if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil {
|
||||
return false
|
||||
}
|
||||
content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var schema struct {
|
||||
Properties map[string]json.RawMessage `json:"properties"`
|
||||
}
|
||||
if json.Unmarshal(content, &schema) != nil {
|
||||
return false
|
||||
}
|
||||
_, found := schema.Properties[property]
|
||||
return found
|
||||
}
|
||||
|
||||
59
internal/modules/integration/registries_test.go
Normal file
59
internal/modules/integration/registries_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
dndregister "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/register"
|
||||
genericregister "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/register"
|
||||
seriatimregister "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/register"
|
||||
)
|
||||
|
||||
func productionNPCRegistries(t *testing.T) pipeline.Registries {
|
||||
t.Helper()
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(),
|
||||
ArtifactEvidence: pipeline.NewArtifactEvidenceRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
for _, registration := range []struct {
|
||||
name string
|
||||
fn func(pipeline.Registries) error
|
||||
}{
|
||||
{name: "generic", fn: genericregister.Register},
|
||||
{name: "seriatim", fn: seriatimregister.Register},
|
||||
{name: "dnd", fn: dndregister.Register},
|
||||
} {
|
||||
if err := registration.fn(registries); err != nil {
|
||||
t.Fatalf("register %s modules: %v", registration.name, err)
|
||||
}
|
||||
}
|
||||
return registries
|
||||
}
|
||||
|
||||
func moduleCatalog(registries pipeline.Registries) pipeline.ModuleCatalog {
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, ArtifactEvidence: registries.ArtifactEvidence,
|
||||
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func publishedLane(t *testing.T, output pipeline.RunOutput, laneID string) contracts.OutputFile {
|
||||
t.Helper()
|
||||
for _, file := range output.OutputFiles {
|
||||
if file.Name == "lanes/"+laneID+".json" {
|
||||
return file
|
||||
}
|
||||
}
|
||||
t.Fatalf("published files = %#v, missing lane %q", output.OutputFiles, laneID)
|
||||
return contracts.OutputFile{}
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
version: 5
|
||||
version: 6
|
||||
output:
|
||||
directory: ./notarius-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: bypass
|
||||
checkpoints: {}
|
||||
debug:
|
||||
directory: ./notarius-debug
|
||||
pipelines:
|
||||
@@ -15,6 +11,6 @@ pipelines:
|
||||
npc_registry:
|
||||
extract:
|
||||
module: dnd/npc-registry
|
||||
retries: 2
|
||||
max_additional_turns: 2
|
||||
normalize: {module: dnd/npc-registry}
|
||||
output: {module: json}
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
version: 5
|
||||
version: 6
|
||||
output:
|
||||
directory: ./notarius-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: bypass
|
||||
checkpoints: {}
|
||||
debug:
|
||||
directory: ./notarius-debug
|
||||
pipelines:
|
||||
|
||||
Reference in New Issue
Block a user