Move cross-family runner contracts into module integration tests

This commit is contained in:
2026-11-22 04:33:28 +00:00
parent 93d94fdd0a
commit 20c0975d9d
7 changed files with 175 additions and 53 deletions

View File

@@ -0,0 +1,124 @@
package integration_test
import (
"context"
"os"
"path/filepath"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape"
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_refs"
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_relatedness"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
func TestNativeCombatPipelineHonorsSceneEligibilityWithoutSyntheticRequests(t *testing.T) {
for _, test := range []struct {
name string
kind dnd.SceneKind
mismatched bool
wantCalls int
}{
{"combat", dnd.SceneKindCombat, false, 1},
{"narrative", dnd.SceneKindNarrative, false, 0},
{"recap", dnd.SceneKindRecap, false, 0},
{"meta", dnd.SceneKindMeta, false, 0},
{"mismatched combat", dnd.SceneKindCombat, true, 0},
} {
t.Run(test.name, func(t *testing.T) {
client := &nativeSequenceClient{responses: []string{`{"combat_turns":[{"actor":"Aria","turn_kind":"turn","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`}}
engine := nativeTestEngine(t, client, "profile-combat")
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
check(transcript.Register(registries.Inputs))
check(units.Register(registries.Chunkers))
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, combatcodec.New()))
check(combatextract.Register(registries.Extractors))
check(appendorder.RegisterTyped(registries.Mergers, dnd.CombatTurnListKind, func(values []dnd.CombatTurnList) (dnd.CombatTurnList, error) {
result := dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}
for _, value := range values {
result.CombatTurns = append(result.CombatTurns, value.CombatTurns...)
}
return result, nil
}))
check(noop.RegisterTyped[dnd.CombatTurnList](registries.Normalizers, dnd.CombatTurnListKind))
check(jsonoutput.Register(registries.Outputs))
check(shape.Register(registries.Validators))
check(sourcerefs.Register(registries.Validators))
check(relatedness.Register(registries.Validators))
scene := dnd.SceneDescription{
ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "combat-session", StartUnitID: 1, EndUnitID: 1},
Kind: test.kind, Title: "Eligibility only", Summary: "This prose must not become combat evidence.",
}
if test.mismatched {
scene.SourceRef.EndUnitID = 2
}
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{scene}})
check(err)
path := filepath.Join(t.TempDir(), "scenes.json")
check(os.WriteFile(path, content, 0600))
profile := pipeline.PipelineProfile{
ID: "native-combat", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
LLMProfile: "profile-combat",
Artifacts: map[string]pipeline.ArtifactLaneProfile{"combat": {
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
pipeline.Binding(shape.Key), pipeline.Binding(sourcerefs.Key), pipeline.Binding(relatedness.Key),
}}, References: map[string]pipeline.ReferenceSource{combatextract.SceneDescriptionReferenceSlot: pipeline.ExternalReference(path)}},
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
}},
}
catalog := pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
check(err)
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
check(err)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
check(err)
preparations := 0
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "combat-session", Workers: 1,
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Aria strikes the ogre."}]}`),
Execution: contracts.ExecutionOptions{OnPrepared: func(context.Context, promptkit.PreparedRun) { preparations++ }},
})
check(err)
if len(client.requests) != test.wantCalls || preparations != test.wantCalls || result.FallbackCount != 0 || len(result.OutputFiles) != 1 {
t.Fatalf("pipeline=%#v native=%d/%d, want %d", result, len(client.requests), preparations, test.wantCalls)
}
value, err := combatcodec.New().Decode(result.OutputFiles[0].Bytes)
check(err)
if value.CombatTurns == nil || len(value.CombatTurns) != test.wantCalls {
t.Fatalf("combat artifact=%#v", value)
}
if test.wantCalls > 0 && (value.CombatTurns[0].Actor != "Aria" || value.CombatTurns[0].SourceRefs[0].SourceID != "combat-session") {
t.Fatalf("combat evidence=%#v", value)
}
})
}
}

View File

@@ -0,0 +1,143 @@
package integration_test
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
enemyextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/enemyevents"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
)
func enemyPipelineReferenceSources(t *testing.T) map[string]pipeline.ReferenceSource {
t.Helper()
ref := func(unit int) source.SourceRef {
return source.SourceRef{SourceID: "combat-session", StartUnitID: unit, EndUnitID: unit}
}
encode := func(content []byte, err error) []byte {
t.Helper()
if err != nil {
t.Fatal(err)
}
return content
}
contents := map[string][]byte{
enemyextract.NPCRegistryReferenceSlot: encode(npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{{
ID: identity.DeriveID("Ashfang"), Name: "Ashfang", SourceRefs: []source.SourceRef{ref(1)},
}}})),
enemyextract.SceneDescriptionReferenceSlot: encode(scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{
ID: "chunk-000001", SourceRef: ref(1), Kind: dnd.SceneKindCombat, Title: "Combat", Summary: "Ashfang opposes the party.",
}}})),
enemyextract.CombatTurnReferenceSlot: encode(combatcodec.New().Encode(dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{
{Actor: "Ashfang", TurnKind: dnd.CombatTurnKindTurn, SourceRefs: []source.SourceRef{ref(1)}},
{Actor: "Aria", TurnKind: dnd.CombatTurnKindReaction, SourceRefs: []source.SourceRef{ref(2)}},
}})),
enemyextract.NPCOccurrenceReferenceSlot: encode(occurrencecodec.New().Encode(dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{
{NPCID: identity.DeriveID("Ashfang"), Name: "Ashfang", Kind: dnd.NPCOccurrenceKindCombatOpponent, SourceRefs: []source.SourceRef{ref(1)}},
{NPCID: identity.DeriveID("Aria"), Name: "Aria", Kind: dnd.NPCOccurrenceKindCombatAlly, SourceRefs: []source.SourceRef{ref(2)}},
}})),
}
root := t.TempDir()
references := make(map[string]pipeline.ReferenceSource, len(contents))
for slot, content := range contents {
path := filepath.Join(root, slot+".json")
if err := os.WriteFile(path, content, 0600); err != nil {
t.Fatal(err)
}
references[slot] = pipeline.ExternalReference(path)
}
return references
}
func TestNativeEnemyPipelineRetainsIntrinsicCodecBoundaryWithEmptyValidators(t *testing.T) {
for _, name := range []string{"Ashfang", " "} {
t.Run(name, func(t *testing.T) {
client := &nativeSequenceClient{responses: []string{fmt.Sprintf(`{"events":[{"name":%q,"kind":"engaged","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name)}}
engine := nativeTestEngine(t, client, "enemy-profile")
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
check(transcript.Register(registries.Inputs))
check(units.Register(registries.Chunkers))
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, eventcodec.New()))
check(enemyextract.Register(registries.Extractors))
check(appendorder.RegisterTyped(registries.Mergers, dnd.EnemyEventListKind, func(values []dnd.EnemyEventList) (dnd.EnemyEventList, error) {
result := dnd.EnemyEventList{Events: []dnd.EnemyEvent{}}
for _, value := range values {
result.Events = append(result.Events, value.Events...)
}
return result, nil
}))
check(noop.RegisterTyped[dnd.EnemyEventList](registries.Normalizers, dnd.EnemyEventListKind))
check(jsonoutput.Register(registries.Outputs))
referenceSources := enemyPipelineReferenceSources(t)
profile := pipeline.PipelineProfile{
ID: "native-enemies", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
LLMProfile: "enemy-profile",
Artifacts: map[string]pipeline.ArtifactLaneProfile{"enemies": {
Extract: pipeline.ModuleBinding{Module: enemyextract.Key, References: referenceSources, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{}}},
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
}},
}
catalog := pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
check(err)
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
check(err)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
check(err)
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "combat-session", Workers: 1,
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"GM","start":0,"end":1,"text":"Ashfang attacks the party."}]}`),
})
if len(client.requests) != 1 {
t.Fatalf("native calls=%d", len(client.requests))
}
if strings.TrimSpace(name) == "" {
if err == nil || !strings.Contains(err.Error(), "encode extract lane") || !strings.Contains(err.Error(), "name must not be empty") {
t.Fatalf("intrinsic codec error=%v", err)
}
return
}
check(err)
if result.FallbackCount != 0 || len(result.OutputFiles) != 1 {
t.Fatalf("assembled outcome=%#v", result)
}
value, err := eventcodec.New().Decode(result.OutputFiles[0].Bytes)
check(err)
if len(value.Events) != 1 || value.Events[0].Name != name || value.Events[0].SourceRefs[0].SourceID != "combat-session" {
t.Fatalf("durable events=%#v", value)
}
})
}
}

View File

@@ -0,0 +1,74 @@
package integration_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func nativeTestEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine {
t.Helper()
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
profiles := make([]map[string]any, len(profileIDs))
for index, id := range profileIDs {
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false}
}
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{"profiles": profiles}})
if err != nil {
t.Fatal(err)
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("native provider fixture", content)}, configkit.LayeredOptions[settings]{
Definition: definition,
Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
},
})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
return engine
}
type nativeSequenceClient struct {
mu sync.Mutex
responses []string
requests []promptkit.GenerateRequest
}
func (client *nativeSequenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.mu.Lock()
defer client.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
index := len(client.requests)
client.requests = append(client.requests, request)
if index >= len(client.responses) {
return nil, fmt.Errorf("unexpected provider request %d", index+1)
}
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
}
func nativePromptText(request promptkit.GenerateRequest) string {
var content []string
for _, message := range request.Prompt.Messages {
content = append(content, message.Content)
}
return strings.Join(content, "\n")
}

View File

@@ -0,0 +1,197 @@
package integration_test
import (
"context"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
spellcatalogvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
spellsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
// Count the real validator boundary, not a second implementation of its rules.
type observedSpellValidator struct {
contracts.TypedValidator[dnd.SpellList]
record func(string)
}
func (v observedSpellValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
v.record(v.Name())
return v.TypedValidator.Validate(ctx, req)
}
func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) {
response := func(name string) string {
return fmt.Sprintf(`{"spell_casts":[{"caster":"Aria","spell":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name)
}
for _, test := range []struct {
name string
mode contracts.ExecutionMode
responses []string
wantError bool
fallbacks int
}{
{"initial acceptance", contracts.ModeStrict, []string{response("Cure Wounds")}, false, 0},
{"final correction accepted", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Cure Wounds")}, false, 0},
{"strict exhaustion", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, true, 0},
{"permissive empty fallback", contracts.ModePermissive, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, false, 1},
} {
t.Run(test.name, func(t *testing.T) {
client := &nativeSequenceClient{responses: append([]string(nil), test.responses...)}
engine := nativeTestEngine(t, client, "profile-spells")
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
check(transcript.Register(registries.Inputs))
check(units.Register(registries.Chunkers))
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, spellcodec.New()))
check(spells.Register(registries.Extractors))
check(appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, func(values []dnd.SpellList) (dnd.SpellList, error) {
result := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
for _, value := range values {
result.SpellCasts = append(result.SpellCasts, value.SpellCasts...)
}
return result, nil
}))
check(noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind))
check(jsonoutput.Register(registries.Outputs))
catalogValidator, err := spellcatalogvalidator.New(spellcatalogvalidator.Options{})
check(err)
validators := []contracts.TypedValidator[dnd.SpellList]{
spellshape.New(spellshape.Options{}), spellsourcerefs.New(spellsourcerefs.Options{}),
catalogValidator, spellsourcerelatedness.New(spellsourcerelatedness.Options{}),
}
var mu sync.Mutex
var calls []string
record := func(name string) { mu.Lock(); defer mu.Unlock(); calls = append(calls, name) }
var bindings []pipeline.ModuleBinding
var names []string
for _, validator := range validators {
check(pipeline.RegisterTypedValidator(registries.Validators, dnd.SpellListKind,
pipeline.ValidatorSpec{Key: validator.Name(), ExecutionClass: contracts.ExecutionClassDeterministic},
func() (contracts.TypedValidator[dnd.SpellList], error) {
return observedSpellValidator{validator, record}, nil
}))
bindings = append(bindings, pipeline.Binding(validator.Name()))
names = append(names, validator.Name())
}
partyFile := filepath.Join(t.TempDir(), "party.txt")
check(os.WriteFile(partyFile, []byte("party-grounding-sentinel: Aria"), 0600))
turns := 2
profile := pipeline.PipelineProfile{
ID: "native-spells", Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
LLMProfile: "profile-spells",
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {
Extract: pipeline.ModuleBinding{Module: spells.Key, Mode: &test.mode, MaxAdditionalTurns: &turns,
Validators: pipeline.ValidatorOverride{Set: true, Validators: bindings},
References: map[string]pipeline.ReferenceSource{"party": pipeline.ExternalReference(partyFile)}},
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
}},
}
catalog := pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
check(err)
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
check(err)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
check(err)
model := "run-wide-spell-model"
var preparations []promptkit.PreparedRun
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", Workers: 1,
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`),
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model},
OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }},
})
if (err != nil) != test.wantError {
t.Fatalf("pipeline error = %v, want error %t", err, test.wantError)
}
wantCalls := len(test.responses)
if len(client.requests) != wantCalls || len(preparations) != wantCalls {
t.Fatalf("provider/preparation calls = %d/%d, want %d", len(client.requests), len(preparations), wantCalls)
}
var expected []string
for range wantCalls {
expected = append(expected, names...)
}
if !reflect.DeepEqual(calls, expected) {
t.Fatalf("ordered assessments = %v, want %v (no outer assessment)", calls, expected)
}
for i, request := range client.requests {
if request.Target.Model != model || request.Prompt.SessionID != "spell-session-override" || preparations[i].SelectedProfileID != "profile-spells" {
t.Fatal("native profile/session/execution override lost")
}
wantID := spells.PromptID
if i > 0 {
wantID = spells.CorrectionPromptID
}
if preparations[i].PromptID != wantID {
t.Fatalf("prompt = %q, want %q", preparations[i].PromptID, wantID)
}
if i == 0 {
continue
}
previous := client.requests[i-1].Prompt.Messages
current := request.Prompt.Messages
if len(current) != len(previous)+2 || !reflect.DeepEqual(current[:len(previous)], previous) {
t.Fatal("native continuation did not retain exact ordered history")
}
if current[len(previous)].Role != "assistant" || current[len(previous)].Content != test.responses[i-1] ||
!strings.Contains(current[len(previous)+1].Content, "recognized D&D spell") {
t.Fatal("correction lost the actual candidate or catalog guidance")
}
if strings.Count(nativePromptText(request), "party-grounding-sentinel") != 1 {
t.Fatal("reference bytes were regenerated in correction")
}
}
if test.wantError {
return
}
if output.FallbackCount != test.fallbacks || len(output.OutputFiles) != 1 {
t.Fatalf("outcome = %#v", output)
}
value, err := spellcodec.New().Decode(output.OutputFiles[0].Bytes)
check(err)
if test.fallbacks > 0 {
if value.SpellCasts == nil || len(value.SpellCasts) != 0 {
t.Fatalf("invalid empty fallback: %#v", value)
}
} else if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" || value.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("durable spell artifact = %#v", value)
}
})
}
}

View File

@@ -0,0 +1,171 @@
package integration_test
import (
"context"
"encoding/json"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
)
func loadPipelineConfig(t *testing.T) config.Config {
t.Helper()
cfg := config.Default()
cfg.Pipelines["seriatim-fixture"] = pipeline.PipelineProfile{
ID: "seriatim-fixture",
Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.Binding("fake/chunk"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"events": {Extract: pipeline.Binding("fake/extract")},
},
}
return cfg
}
func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.ModuleCatalog {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry()
extractors := pipeline.NewExtractorRegistry()
mergers := pipeline.NewMergerRegistry()
normalizers := pipeline.NewNormalizerRegistry()
outputs := pipeline.NewOutputEncoderRegistry()
if reflect.DeepEqual(inputSpec, transcript.ModuleSpec()) {
if err := transcript.Register(inputs); err != nil {
t.Fatalf("register seriatim input: %v", err)
}
} else if err := inputs.RegisterWithSpec(inputSpec, func() (contracts.InputAdapter, error) {
return transcript.New(), nil
}); err != nil {
t.Fatalf("register seriatim input override: %v", err)
}
mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{Key: "fake/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source.transcript"}, Provides: []string{"chunks"}})
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
Key: "fake/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind,
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"},
})
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind, Requires: []string{"fake.artifacts"}})
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind})
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic})
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.ModuleCatalog{Inputs: inputs, Chunkers: chunkers, ArtifactCodecs: codecs, Extractors: extractors, Mergers: mergers, Normalizers: normalizers, ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: outputs}
}
func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) { return fakeChunker{}, nil }); err != nil {
t.Fatalf("register chunker: %v", err)
}
}
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) { return fakeExtractor{}, nil }); err != nil {
t.Fatalf("register extractor: %v", err)
}
}
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) { return fakeMerger{}, nil }); err != nil {
t.Fatalf("register merger: %v", err)
}
}
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) { return fakeNormalizer{}, nil }); err != nil {
t.Fatalf("register normalizer: %v", err)
}
}
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
t.Helper()
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return fakeOutput{}, nil }); err != nil {
t.Fatalf("register output: %v", err)
}
}
type fakeChunker struct{}
func (fakeChunker) Key() string { return "fake/chunk" }
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
return contracts.ChunkPlanResult{}, nil
}
type fakeExtractor struct{}
func (fakeExtractor) Key() string { return "fake/extract" }
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeExtractor) Extract(context.Context, contracts.TypedExtractionRequest[seriatimArtifact]) (contracts.TypedExtractionResult[seriatimArtifact], error) {
return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
}
type fakeMerger struct{}
func (fakeMerger) Key() string { return pipeline.DefaultMergeModule }
func (fakeMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[seriatimArtifact]) (contracts.TypedMergeResult[seriatimArtifact], error) {
if len(req.ExtractOutputs) == 0 {
return contracts.TypedMergeResult[seriatimArtifact]{}, nil
}
return contracts.TypedMergeResult[seriatimArtifact]{Value: req.ExtractOutputs[0].Value}, nil
}
type fakeNormalizer struct{}
func (fakeNormalizer) Key() string { return pipeline.DefaultNormalizeModule }
func (fakeNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (fakeNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[seriatimArtifact]) (contracts.TypedNormalizeResult[seriatimArtifact], error) {
return contracts.TypedNormalizeResult[seriatimArtifact]{Value: req.MergeOutput.Value}, nil
}
type fakeOutput struct{}
func (fakeOutput) Key() string { return pipeline.DefaultOutputModule }
func (fakeOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
return contracts.OutputResult{}, nil
}
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
type seriatimArtifact struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
type seriatimArtifactCodec struct{}
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
}
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
func (seriatimArtifactCodec) EncodeCandidate(value seriatimArtifact) ([]byte, error) {
return json.Marshal(value)
}
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
return json.Marshal(value)
}
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
var value seriatimArtifact
err := json.Unmarshal(content, &value)
return value, err
}
func (codec seriatimArtifactCodec) DecodeCandidate(content []byte) (seriatimArtifact, error) {
return codec.Decode(content)
}

View File

@@ -0,0 +1,254 @@
package integration_test
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
)
func runSeriatimPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, input pipeline.RunInput) (pipeline.RunOutput, error) {
t.Helper()
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
return pipeline.RunOutput{}, err
}
input.Prepared = prepared
return pipeline.New().Run(context.Background(), input)
}
func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
raw, err := os.ReadFile("../seriatim/input/transcript/testdata/valid_minimal.json")
if err != nil {
t.Fatal(err)
}
expectedDoc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
if err != nil {
t.Fatalf("Parse() error = %v, want nil", err)
}
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
extractor := &runnerSeriatimExtractor{}
output, err := runSeriatimPipeline(t, seriatimRunnerRegistries(t, extractor), resolved.ResolvedPipeline, pipeline.RunInput{
RawInput: raw,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.ArtifactCount() != 1 || output.OutputFiles[0].Name != "lanes/events.json" {
t.Fatalf("output files=%#v, want the event domain artifact", output.OutputFiles)
}
var payload struct {
Value string `json:"value"`
SourceRefs []source.SourceRef `json:"source_refs"`
}
if err := json.Unmarshal(output.OutputFiles[0].Bytes, &payload); err != nil {
t.Fatalf("Unmarshal(serialized output) error = %v, want nil", err)
}
if len(payload.SourceRefs) != 1 {
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
}
if err := source.ValidateRef(expectedDoc, payload.SourceRefs[0]); err != nil {
t.Fatalf("ValidateRef() error = %v, want nil", err)
}
if payload.SourceRefs[0].StartUnitID != 1 || payload.SourceRefs[0].EndUnitID != 2 {
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", payload.SourceRefs[0])
}
if extractor.calls != 1 {
t.Fatalf("extractor calls = %d, want 1", extractor.calls)
}
if len(output.OutputFiles) != 1 {
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
}
if output.OutputFiles[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
}
}
func TestRunnerFailsOnInvalidSeriatimInput(t *testing.T) {
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
if err != nil {
t.Fatalf("Resolve() error = %v, want nil", err)
}
output, err := runSeriatimPipeline(t, seriatimRunnerRegistries(t, &runnerSeriatimExtractor{}), resolved.ResolvedPipeline, pipeline.RunInput{
RawInput: []byte(`{"metadata":{},"segments":[]}`),
SourceID: "invalid-source",
})
if err == nil {
t.Fatal("Run() error = nil, want invalid input error")
}
if !strings.Contains(err.Error(), "parse input with adapter") || !strings.Contains(err.Error(), "seriatim input") {
t.Fatalf("Run() error = %q, want Seriatim parse context", err.Error())
}
if output.ArtifactCount() != 0 {
t.Fatalf("invalid input produced files: %#v", output.OutputFiles)
}
}
func configResolveInput(t *testing.T) config.ResolveInput {
t.Helper()
return config.ResolveInput{
PipelineID: "seriatim-fixture",
Catalog: seriatimTestCatalog(t, transcript.ModuleSpec()),
}
}
func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor[seriatimArtifact]) pipeline.Registries {
t.Helper()
inputs := pipeline.NewInputAdapterRegistry()
chunkers := pipeline.NewChunkerRegistry()
extractors := pipeline.NewExtractorRegistry()
mergers := pipeline.NewMergerRegistry()
normalizers := pipeline.NewNormalizerRegistry()
outputs := pipeline.NewOutputEncoderRegistry()
if err := transcript.Register(inputs); err != nil {
t.Fatalf("register seriatim input: %v", err)
}
if err := chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "fake/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.Chunker, error) {
return runnerSeriatimChunker{}, nil
}); err != nil {
t.Fatalf("register chunker: %v", err)
}
if err := pipeline.RegisterExtractor[seriatimArtifact](extractors, pipeline.ModuleSpec{Key: "fake/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind, Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"}}, func() (contracts.Extractor[seriatimArtifact], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register extractor: %v", err)
}
if err := pipeline.RegisterMerger[seriatimArtifact](mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind}, func() (contracts.Merger[seriatimArtifact], error) {
return fakeMerger{}, nil
}); err != nil {
t.Fatalf("register merger: %v", err)
}
if err := pipeline.RegisterNormalizer[seriatimArtifact](normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind}, func() (contracts.Normalizer[seriatimArtifact], error) {
return fakeNormalizer{}, nil
}); err != nil {
t.Fatalf("register normalizer: %v", err)
}
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) {
return jsonoutput.New(), nil
}); err != nil {
t.Fatalf("register output: %v", err)
}
codecs := pipeline.NewArtifactCodecRegistry()
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
t.Fatalf("register artifact codec: %v", err)
}
return pipeline.Registries{
Inputs: inputs,
Chunkers: chunkers,
ArtifactCodecs: codecs,
Extractors: extractors,
Mergers: mergers,
Normalizers: normalizers,
Outputs: outputs,
}
}
type runnerSeriatimChunker struct{}
func (runnerSeriatimChunker) Key() string {
return "fake/chunk"
}
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (runnerSeriatimChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
return contracts.ChunkPlanResult{
Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID}}},
}, nil
}
type runnerSeriatimExtractor struct {
calls int
}
func (e *runnerSeriatimExtractor) Key() string {
return "fake/extract"
}
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[seriatimArtifact]) (contracts.TypedExtractionResult[seriatimArtifact], error) {
e.calls++
if req.Source == nil {
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source must not be nil")
}
if req.Chunk == nil {
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk must not be nil")
}
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
}
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
}
for _, unit := range req.Chunk.Units {
if speaker, ok := transcript.Speaker(unit); !ok || speaker == "" {
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
}
if _, ok := transcript.Start(unit); !ok {
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
}
if _, ok := transcript.End(unit); !ok {
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
}
}
return contracts.TypedExtractionResult[seriatimArtifact]{Value: seriatimArtifact{
Value: "seriatim-source-ref",
SourceRefs: []source.SourceRef{
{
SourceID: req.Source.ID,
StartUnitID: req.Chunk.Units[0].ID,
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
},
},
}}, nil
}
func unitIDs(units []source.SourceUnit) []int {
ids := make([]int, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}
return ids
}
func equalInts(a, b []int) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
var (
_ contracts.Chunker = runnerSeriatimChunker{}
_ contracts.Extractor[seriatimArtifact] = (*runnerSeriatimExtractor)(nil)
)