Move cross-family runner contracts into module integration tests
This commit is contained in:
124
internal/modules/integration/native_combat_pipeline_test.go
Normal file
124
internal/modules/integration/native_combat_pipeline_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape"
|
||||
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/source_relatedness"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNativeCombatPipelineHonorsSceneEligibilityWithoutSyntheticRequests(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
kind dnd.SceneKind
|
||||
mismatched bool
|
||||
wantCalls int
|
||||
}{
|
||||
{"combat", dnd.SceneKindCombat, false, 1},
|
||||
{"narrative", dnd.SceneKindNarrative, false, 0},
|
||||
{"recap", dnd.SceneKindRecap, false, 0},
|
||||
{"meta", dnd.SceneKindMeta, false, 0},
|
||||
{"mismatched combat", dnd.SceneKindCombat, true, 0},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &nativeSequenceClient{responses: []string{`{"combat_turns":[{"actor":"Aria","turn_kind":"turn","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`}}
|
||||
engine := nativeTestEngine(t, client, "profile-combat")
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
check := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
check(transcript.Register(registries.Inputs))
|
||||
check(units.Register(registries.Chunkers))
|
||||
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, combatcodec.New()))
|
||||
check(combatextract.Register(registries.Extractors))
|
||||
check(appendorder.RegisterTyped(registries.Mergers, dnd.CombatTurnListKind, func(values []dnd.CombatTurnList) (dnd.CombatTurnList, error) {
|
||||
result := dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{}}
|
||||
for _, value := range values {
|
||||
result.CombatTurns = append(result.CombatTurns, value.CombatTurns...)
|
||||
}
|
||||
return result, nil
|
||||
}))
|
||||
check(noop.RegisterTyped[dnd.CombatTurnList](registries.Normalizers, dnd.CombatTurnListKind))
|
||||
check(jsonoutput.Register(registries.Outputs))
|
||||
check(shape.Register(registries.Validators))
|
||||
check(sourcerefs.Register(registries.Validators))
|
||||
check(relatedness.Register(registries.Validators))
|
||||
scene := dnd.SceneDescription{
|
||||
ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "combat-session", StartUnitID: 1, EndUnitID: 1},
|
||||
Kind: test.kind, Title: "Eligibility only", Summary: "This prose must not become combat evidence.",
|
||||
}
|
||||
if test.mismatched {
|
||||
scene.SourceRef.EndUnitID = 2
|
||||
}
|
||||
content, err := scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{scene}})
|
||||
check(err)
|
||||
path := filepath.Join(t.TempDir(), "scenes.json")
|
||||
check(os.WriteFile(path, content, 0600))
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: "native-combat", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
|
||||
LLMProfile: "profile-combat",
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{"combat": {
|
||||
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding(shape.Key), pipeline.Binding(sourcerefs.Key), pipeline.Binding(relatedness.Key),
|
||||
}}, References: map[string]pipeline.ReferenceSource{combatextract.SceneDescriptionReferenceSlot: pipeline.ExternalReference(path)}},
|
||||
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
|
||||
}},
|
||||
}
|
||||
catalog := pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
|
||||
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
|
||||
}
|
||||
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
|
||||
check(err)
|
||||
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
|
||||
check(err)
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
|
||||
check(err)
|
||||
preparations := 0
|
||||
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared, SourceID: "combat-session", Workers: 1,
|
||||
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Aria strikes the ogre."}]}`),
|
||||
Execution: contracts.ExecutionOptions{OnPrepared: func(context.Context, promptkit.PreparedRun) { preparations++ }},
|
||||
})
|
||||
check(err)
|
||||
if len(client.requests) != test.wantCalls || preparations != test.wantCalls || result.FallbackCount != 0 || len(result.OutputFiles) != 1 {
|
||||
t.Fatalf("pipeline=%#v native=%d/%d, want %d", result, len(client.requests), preparations, test.wantCalls)
|
||||
}
|
||||
value, err := combatcodec.New().Decode(result.OutputFiles[0].Bytes)
|
||||
check(err)
|
||||
if value.CombatTurns == nil || len(value.CombatTurns) != test.wantCalls {
|
||||
t.Fatalf("combat artifact=%#v", value)
|
||||
}
|
||||
if test.wantCalls > 0 && (value.CombatTurns[0].Actor != "Aria" || value.CombatTurns[0].SourceRefs[0].SourceID != "combat-session") {
|
||||
t.Fatalf("combat evidence=%#v", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
143
internal/modules/integration/native_enemy_pipeline_test.go
Normal file
143
internal/modules/integration/native_enemy_pipeline_test.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
||||
eventcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/enemyevents"
|
||||
occurrencecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcoccurrences"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
enemyextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/enemyevents"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
func enemyPipelineReferenceSources(t *testing.T) map[string]pipeline.ReferenceSource {
|
||||
t.Helper()
|
||||
ref := func(unit int) source.SourceRef {
|
||||
return source.SourceRef{SourceID: "combat-session", StartUnitID: unit, EndUnitID: unit}
|
||||
}
|
||||
encode := func(content []byte, err error) []byte {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return content
|
||||
}
|
||||
contents := map[string][]byte{
|
||||
enemyextract.NPCRegistryReferenceSlot: encode(npccodec.New().Encode(dnd.NPCRegistry{NPCs: []dnd.NPC{{
|
||||
ID: identity.DeriveID("Ashfang"), Name: "Ashfang", SourceRefs: []source.SourceRef{ref(1)},
|
||||
}}})),
|
||||
enemyextract.SceneDescriptionReferenceSlot: encode(scenecodec.New().Encode(dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{
|
||||
ID: "chunk-000001", SourceRef: ref(1), Kind: dnd.SceneKindCombat, Title: "Combat", Summary: "Ashfang opposes the party.",
|
||||
}}})),
|
||||
enemyextract.CombatTurnReferenceSlot: encode(combatcodec.New().Encode(dnd.CombatTurnList{CombatTurns: []dnd.CombatTurn{
|
||||
{Actor: "Ashfang", TurnKind: dnd.CombatTurnKindTurn, SourceRefs: []source.SourceRef{ref(1)}},
|
||||
{Actor: "Aria", TurnKind: dnd.CombatTurnKindReaction, SourceRefs: []source.SourceRef{ref(2)}},
|
||||
}})),
|
||||
enemyextract.NPCOccurrenceReferenceSlot: encode(occurrencecodec.New().Encode(dnd.NPCOccurrenceList{Occurrences: []dnd.NPCOccurrence{
|
||||
{NPCID: identity.DeriveID("Ashfang"), Name: "Ashfang", Kind: dnd.NPCOccurrenceKindCombatOpponent, SourceRefs: []source.SourceRef{ref(1)}},
|
||||
{NPCID: identity.DeriveID("Aria"), Name: "Aria", Kind: dnd.NPCOccurrenceKindCombatAlly, SourceRefs: []source.SourceRef{ref(2)}},
|
||||
}})),
|
||||
}
|
||||
root := t.TempDir()
|
||||
references := make(map[string]pipeline.ReferenceSource, len(contents))
|
||||
for slot, content := range contents {
|
||||
path := filepath.Join(root, slot+".json")
|
||||
if err := os.WriteFile(path, content, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
references[slot] = pipeline.ExternalReference(path)
|
||||
}
|
||||
return references
|
||||
}
|
||||
|
||||
func TestNativeEnemyPipelineRetainsIntrinsicCodecBoundaryWithEmptyValidators(t *testing.T) {
|
||||
for _, name := range []string{"Ashfang", " "} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
client := &nativeSequenceClient{responses: []string{fmt.Sprintf(`{"events":[{"name":%q,"kind":"engaged","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name)}}
|
||||
engine := nativeTestEngine(t, client, "enemy-profile")
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
check := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
check(transcript.Register(registries.Inputs))
|
||||
check(units.Register(registries.Chunkers))
|
||||
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, eventcodec.New()))
|
||||
check(enemyextract.Register(registries.Extractors))
|
||||
check(appendorder.RegisterTyped(registries.Mergers, dnd.EnemyEventListKind, func(values []dnd.EnemyEventList) (dnd.EnemyEventList, error) {
|
||||
result := dnd.EnemyEventList{Events: []dnd.EnemyEvent{}}
|
||||
for _, value := range values {
|
||||
result.Events = append(result.Events, value.Events...)
|
||||
}
|
||||
return result, nil
|
||||
}))
|
||||
check(noop.RegisterTyped[dnd.EnemyEventList](registries.Normalizers, dnd.EnemyEventListKind))
|
||||
check(jsonoutput.Register(registries.Outputs))
|
||||
referenceSources := enemyPipelineReferenceSources(t)
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: "native-enemies", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
|
||||
LLMProfile: "enemy-profile",
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{"enemies": {
|
||||
Extract: pipeline.ModuleBinding{Module: enemyextract.Key, References: referenceSources, Validators: pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{}}},
|
||||
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
|
||||
}},
|
||||
}
|
||||
catalog := pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
|
||||
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
|
||||
}
|
||||
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
|
||||
check(err)
|
||||
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
|
||||
check(err)
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
|
||||
check(err)
|
||||
result, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared, SourceID: "combat-session", Workers: 1,
|
||||
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"GM","start":0,"end":1,"text":"Ashfang attacks the party."}]}`),
|
||||
})
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("native calls=%d", len(client.requests))
|
||||
}
|
||||
if strings.TrimSpace(name) == "" {
|
||||
if err == nil || !strings.Contains(err.Error(), "encode extract lane") || !strings.Contains(err.Error(), "name must not be empty") {
|
||||
t.Fatalf("intrinsic codec error=%v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
check(err)
|
||||
if result.FallbackCount != 0 || len(result.OutputFiles) != 1 {
|
||||
t.Fatalf("assembled outcome=%#v", result)
|
||||
}
|
||||
value, err := eventcodec.New().Decode(result.OutputFiles[0].Bytes)
|
||||
check(err)
|
||||
if len(value.Events) != 1 || value.Events[0].Name != name || value.Events[0].SourceRefs[0].SourceID != "combat-session" {
|
||||
t.Fatalf("durable events=%#v", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
74
internal/modules/integration/native_provider_test.go
Normal file
74
internal/modules/integration/native_provider_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func nativeTestEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
profiles := make([]map[string]any, len(profileIDs))
|
||||
for index, id := range profileIDs {
|
||||
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false}
|
||||
}
|
||||
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{"profiles": profiles}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("native provider fixture", content)}, configkit.LayeredOptions[settings]{
|
||||
Definition: definition,
|
||||
Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
type nativeSequenceClient struct {
|
||||
mu sync.Mutex
|
||||
responses []string
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (client *nativeSequenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if index >= len(client.responses) {
|
||||
return nil, fmt.Errorf("unexpected provider request %d", index+1)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
|
||||
}
|
||||
|
||||
func nativePromptText(request promptkit.GenerateRequest) string {
|
||||
var content []string
|
||||
for _, message := range request.Prompt.Messages {
|
||||
content = append(content, message.Content)
|
||||
}
|
||||
return strings.Join(content, "\n")
|
||||
}
|
||||
197
internal/modules/integration/native_spell_pipeline_test.go
Normal file
197
internal/modules/integration/native_spell_pipeline_test.go
Normal file
@@ -0,0 +1,197 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
||||
spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
spellcatalogvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
|
||||
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
|
||||
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
|
||||
spellsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// Count the real validator boundary, not a second implementation of its rules.
|
||||
type observedSpellValidator struct {
|
||||
contracts.TypedValidator[dnd.SpellList]
|
||||
record func(string)
|
||||
}
|
||||
|
||||
func (v observedSpellValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
|
||||
v.record(v.Name())
|
||||
return v.TypedValidator.Validate(ctx, req)
|
||||
}
|
||||
|
||||
func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) {
|
||||
response := func(name string) string {
|
||||
return fmt.Sprintf(`{"spell_casts":[{"caster":"Aria","spell":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mode contracts.ExecutionMode
|
||||
responses []string
|
||||
wantError bool
|
||||
fallbacks int
|
||||
}{
|
||||
{"initial acceptance", contracts.ModeStrict, []string{response("Cure Wounds")}, false, 0},
|
||||
{"final correction accepted", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Cure Wounds")}, false, 0},
|
||||
{"strict exhaustion", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, true, 0},
|
||||
{"permissive empty fallback", contracts.ModePermissive, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, false, 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
client := &nativeSequenceClient{responses: append([]string(nil), test.responses...)}
|
||||
engine := nativeTestEngine(t, client, "profile-spells")
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
|
||||
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
check := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
check(transcript.Register(registries.Inputs))
|
||||
check(units.Register(registries.Chunkers))
|
||||
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, spellcodec.New()))
|
||||
check(spells.Register(registries.Extractors))
|
||||
check(appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, func(values []dnd.SpellList) (dnd.SpellList, error) {
|
||||
result := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
||||
for _, value := range values {
|
||||
result.SpellCasts = append(result.SpellCasts, value.SpellCasts...)
|
||||
}
|
||||
return result, nil
|
||||
}))
|
||||
check(noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind))
|
||||
check(jsonoutput.Register(registries.Outputs))
|
||||
catalogValidator, err := spellcatalogvalidator.New(spellcatalogvalidator.Options{})
|
||||
check(err)
|
||||
validators := []contracts.TypedValidator[dnd.SpellList]{
|
||||
spellshape.New(spellshape.Options{}), spellsourcerefs.New(spellsourcerefs.Options{}),
|
||||
catalogValidator, spellsourcerelatedness.New(spellsourcerelatedness.Options{}),
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var calls []string
|
||||
record := func(name string) { mu.Lock(); defer mu.Unlock(); calls = append(calls, name) }
|
||||
var bindings []pipeline.ModuleBinding
|
||||
var names []string
|
||||
for _, validator := range validators {
|
||||
check(pipeline.RegisterTypedValidator(registries.Validators, dnd.SpellListKind,
|
||||
pipeline.ValidatorSpec{Key: validator.Name(), ExecutionClass: contracts.ExecutionClassDeterministic},
|
||||
func() (contracts.TypedValidator[dnd.SpellList], error) {
|
||||
return observedSpellValidator{validator, record}, nil
|
||||
}))
|
||||
bindings = append(bindings, pipeline.Binding(validator.Name()))
|
||||
names = append(names, validator.Name())
|
||||
}
|
||||
partyFile := filepath.Join(t.TempDir(), "party.txt")
|
||||
check(os.WriteFile(partyFile, []byte("party-grounding-sentinel: Aria"), 0600))
|
||||
turns := 2
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: "native-spells", Input: pipeline.Binding(transcript.Key),
|
||||
Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
|
||||
LLMProfile: "profile-spells",
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {
|
||||
Extract: pipeline.ModuleBinding{Module: spells.Key, Mode: &test.mode, MaxAdditionalTurns: &turns,
|
||||
Validators: pipeline.ValidatorOverride{Set: true, Validators: bindings},
|
||||
References: map[string]pipeline.ReferenceSource{"party": pipeline.ExternalReference(partyFile)}},
|
||||
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
|
||||
}},
|
||||
}
|
||||
catalog := pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
|
||||
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
|
||||
}
|
||||
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
|
||||
check(err)
|
||||
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
|
||||
check(err)
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
|
||||
check(err)
|
||||
model := "run-wide-spell-model"
|
||||
var preparations []promptkit.PreparedRun
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", Workers: 1,
|
||||
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`),
|
||||
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model},
|
||||
OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }},
|
||||
})
|
||||
if (err != nil) != test.wantError {
|
||||
t.Fatalf("pipeline error = %v, want error %t", err, test.wantError)
|
||||
}
|
||||
wantCalls := len(test.responses)
|
||||
if len(client.requests) != wantCalls || len(preparations) != wantCalls {
|
||||
t.Fatalf("provider/preparation calls = %d/%d, want %d", len(client.requests), len(preparations), wantCalls)
|
||||
}
|
||||
var expected []string
|
||||
for range wantCalls {
|
||||
expected = append(expected, names...)
|
||||
}
|
||||
if !reflect.DeepEqual(calls, expected) {
|
||||
t.Fatalf("ordered assessments = %v, want %v (no outer assessment)", calls, expected)
|
||||
}
|
||||
for i, request := range client.requests {
|
||||
if request.Target.Model != model || request.Prompt.SessionID != "spell-session-override" || preparations[i].SelectedProfileID != "profile-spells" {
|
||||
t.Fatal("native profile/session/execution override lost")
|
||||
}
|
||||
wantID := spells.PromptID
|
||||
if i > 0 {
|
||||
wantID = spells.CorrectionPromptID
|
||||
}
|
||||
if preparations[i].PromptID != wantID {
|
||||
t.Fatalf("prompt = %q, want %q", preparations[i].PromptID, wantID)
|
||||
}
|
||||
if i == 0 {
|
||||
continue
|
||||
}
|
||||
previous := client.requests[i-1].Prompt.Messages
|
||||
current := request.Prompt.Messages
|
||||
if len(current) != len(previous)+2 || !reflect.DeepEqual(current[:len(previous)], previous) {
|
||||
t.Fatal("native continuation did not retain exact ordered history")
|
||||
}
|
||||
if current[len(previous)].Role != "assistant" || current[len(previous)].Content != test.responses[i-1] ||
|
||||
!strings.Contains(current[len(previous)+1].Content, "recognized D&D spell") {
|
||||
t.Fatal("correction lost the actual candidate or catalog guidance")
|
||||
}
|
||||
if strings.Count(nativePromptText(request), "party-grounding-sentinel") != 1 {
|
||||
t.Fatal("reference bytes were regenerated in correction")
|
||||
}
|
||||
}
|
||||
if test.wantError {
|
||||
return
|
||||
}
|
||||
if output.FallbackCount != test.fallbacks || len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("outcome = %#v", output)
|
||||
}
|
||||
value, err := spellcodec.New().Decode(output.OutputFiles[0].Bytes)
|
||||
check(err)
|
||||
if test.fallbacks > 0 {
|
||||
if value.SpellCasts == nil || len(value.SpellCasts) != 0 {
|
||||
t.Fatalf("invalid empty fallback: %#v", value)
|
||||
}
|
||||
} else if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" || value.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
|
||||
t.Fatalf("durable spell artifact = %#v", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
171
internal/modules/integration/seriatim_runner_helpers_test.go
Normal file
171
internal/modules/integration/seriatim_runner_helpers_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
func loadPipelineConfig(t *testing.T) config.Config {
|
||||
t.Helper()
|
||||
|
||||
cfg := config.Default()
|
||||
cfg.Pipelines["seriatim-fixture"] = pipeline.PipelineProfile{
|
||||
ID: "seriatim-fixture",
|
||||
Input: pipeline.Binding(transcript.Key),
|
||||
Chunk: pipeline.Binding("fake/chunk"),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"events": {Extract: pipeline.Binding("fake/extract")},
|
||||
},
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func seriatimTestCatalog(t *testing.T, inputSpec pipeline.ModuleSpec) pipeline.ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
inputs := pipeline.NewInputAdapterRegistry()
|
||||
chunkers := pipeline.NewChunkerRegistry()
|
||||
extractors := pipeline.NewExtractorRegistry()
|
||||
mergers := pipeline.NewMergerRegistry()
|
||||
normalizers := pipeline.NewNormalizerRegistry()
|
||||
outputs := pipeline.NewOutputEncoderRegistry()
|
||||
|
||||
if reflect.DeepEqual(inputSpec, transcript.ModuleSpec()) {
|
||||
if err := transcript.Register(inputs); err != nil {
|
||||
t.Fatalf("register seriatim input: %v", err)
|
||||
}
|
||||
} else if err := inputs.RegisterWithSpec(inputSpec, func() (contracts.InputAdapter, error) {
|
||||
return transcript.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register seriatim input override: %v", err)
|
||||
}
|
||||
|
||||
mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{Key: "fake/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"source.transcript"}, Provides: []string{"chunks"}})
|
||||
mustRegisterExtractor(t, extractors, pipeline.ModuleSpec{
|
||||
Key: "fake/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind,
|
||||
Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"},
|
||||
})
|
||||
mustRegisterMerger(t, mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind, Requires: []string{"fake.artifacts"}})
|
||||
mustRegisterNormalizer(t, normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind})
|
||||
mustRegisterOutput(t, outputs, pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic})
|
||||
|
||||
codecs := pipeline.NewArtifactCodecRegistry()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
|
||||
t.Fatalf("register artifact codec: %v", err)
|
||||
}
|
||||
return pipeline.ModuleCatalog{Inputs: inputs, Chunkers: chunkers, ArtifactCodecs: codecs, Extractors: extractors, Mergers: mergers, Normalizers: normalizers, ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: outputs}
|
||||
}
|
||||
|
||||
func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) { return fakeChunker{}, nil }); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := pipeline.RegisterExtractor[seriatimArtifact](registry, spec, func() (contracts.Extractor[seriatimArtifact], error) { return fakeExtractor{}, nil }); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := pipeline.RegisterMerger[seriatimArtifact](registry, spec, func() (contracts.Merger[seriatimArtifact], error) { return fakeMerger{}, nil }); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := pipeline.RegisterNormalizer[seriatimArtifact](registry, spec, func() (contracts.Normalizer[seriatimArtifact], error) { return fakeNormalizer{}, nil }); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return fakeOutput{}, nil }); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeChunker struct{}
|
||||
|
||||
func (fakeChunker) Key() string { return "fake/chunk" }
|
||||
func (fakeChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (fakeChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
return contracts.ChunkPlanResult{}, nil
|
||||
}
|
||||
|
||||
type fakeExtractor struct{}
|
||||
|
||||
func (fakeExtractor) Key() string { return "fake/extract" }
|
||||
func (fakeExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (fakeExtractor) Extract(context.Context, contracts.TypedExtractionRequest[seriatimArtifact]) (contracts.TypedExtractionResult[seriatimArtifact], error) {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, nil
|
||||
}
|
||||
|
||||
type fakeMerger struct{}
|
||||
|
||||
func (fakeMerger) Key() string { return pipeline.DefaultMergeModule }
|
||||
func (fakeMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[seriatimArtifact]) (contracts.TypedMergeResult[seriatimArtifact], error) {
|
||||
if len(req.ExtractOutputs) == 0 {
|
||||
return contracts.TypedMergeResult[seriatimArtifact]{}, nil
|
||||
}
|
||||
return contracts.TypedMergeResult[seriatimArtifact]{Value: req.ExtractOutputs[0].Value}, nil
|
||||
}
|
||||
|
||||
type fakeNormalizer struct{}
|
||||
|
||||
func (fakeNormalizer) Key() string { return pipeline.DefaultNormalizeModule }
|
||||
func (fakeNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (fakeNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[seriatimArtifact]) (contracts.TypedNormalizeResult[seriatimArtifact], error) {
|
||||
return contracts.TypedNormalizeResult[seriatimArtifact]{Value: req.MergeOutput.Value}, nil
|
||||
}
|
||||
|
||||
type fakeOutput struct{}
|
||||
|
||||
func (fakeOutput) Key() string { return pipeline.DefaultOutputModule }
|
||||
func (fakeOutput) Encode(context.Context, contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
return contracts.OutputResult{}, nil
|
||||
}
|
||||
|
||||
const seriatimArtifactKind contracts.ArtifactKind = "test/seriatim-event"
|
||||
|
||||
type seriatimArtifact struct {
|
||||
Value string `json:"value"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
}
|
||||
|
||||
type seriatimArtifactCodec struct{}
|
||||
|
||||
func (seriatimArtifactCodec) Kind() contracts.ArtifactKind { return seriatimArtifactKind }
|
||||
func (seriatimArtifactCodec) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "fake.event", Name: "fake_event", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
||||
}
|
||||
func (seriatimArtifactCodec) MediaType() string { return "application/json" }
|
||||
func (seriatimArtifactCodec) EncodeCandidate(value seriatimArtifact) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
func (seriatimArtifactCodec) Encode(value seriatimArtifact) ([]byte, error) {
|
||||
return json.Marshal(value)
|
||||
}
|
||||
func (seriatimArtifactCodec) Decode(content []byte) (seriatimArtifact, error) {
|
||||
var value seriatimArtifact
|
||||
err := json.Unmarshal(content, &value)
|
||||
return value, err
|
||||
}
|
||||
|
||||
func (codec seriatimArtifactCodec) DecodeCandidate(content []byte) (seriatimArtifact, error) {
|
||||
return codec.Decode(content)
|
||||
}
|
||||
254
internal/modules/integration/seriatim_runner_test.go
Normal file
254
internal/modules/integration/seriatim_runner_test.go
Normal file
@@ -0,0 +1,254 @@
|
||||
package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
)
|
||||
|
||||
func runSeriatimPipeline(t *testing.T, registries pipeline.Registries, resolved pipeline.ResolvedPipeline, input pipeline.RunInput) (pipeline.RunOutput, error) {
|
||||
t.Helper()
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
return pipeline.RunOutput{}, err
|
||||
}
|
||||
input.Prepared = prepared
|
||||
return pipeline.New().Run(context.Background(), input)
|
||||
}
|
||||
|
||||
func TestRunnerProcessesSeriatimInputWithFakeModules(t *testing.T) {
|
||||
raw, err := os.ReadFile("../seriatim/input/transcript/testdata/valid_minimal.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectedDoc, err := transcript.New().Parse(context.Background(), contracts.ParseRequest{Raw: raw})
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
extractor := &runnerSeriatimExtractor{}
|
||||
output, err := runSeriatimPipeline(t, seriatimRunnerRegistries(t, extractor), resolved.ResolvedPipeline, pipeline.RunInput{
|
||||
RawInput: raw,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if output.ArtifactCount() != 1 || output.OutputFiles[0].Name != "lanes/events.json" {
|
||||
t.Fatalf("output files=%#v, want the event domain artifact", output.OutputFiles)
|
||||
}
|
||||
var payload struct {
|
||||
Value string `json:"value"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs"`
|
||||
}
|
||||
if err := json.Unmarshal(output.OutputFiles[0].Bytes, &payload); err != nil {
|
||||
t.Fatalf("Unmarshal(serialized output) error = %v, want nil", err)
|
||||
}
|
||||
if len(payload.SourceRefs) != 1 {
|
||||
t.Fatalf("len(SourceRefs) = %d, want 1", len(payload.SourceRefs))
|
||||
}
|
||||
if err := source.ValidateRef(expectedDoc, payload.SourceRefs[0]); err != nil {
|
||||
t.Fatalf("ValidateRef() error = %v, want nil", err)
|
||||
}
|
||||
if payload.SourceRefs[0].StartUnitID != 1 || payload.SourceRefs[0].EndUnitID != 2 {
|
||||
t.Fatalf("SourceRefs[0] = %#v, want Seriatim unit IDs", payload.SourceRefs[0])
|
||||
}
|
||||
if extractor.calls != 1 {
|
||||
t.Fatalf("extractor calls = %d, want 1", extractor.calls)
|
||||
}
|
||||
if len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("len(OutputFiles) = %d, want 1", len(output.OutputFiles))
|
||||
}
|
||||
if output.OutputFiles[0].ContentType != "application/json" {
|
||||
t.Fatalf("ContentType = %q, want application/json", output.OutputFiles[0].ContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerFailsOnInvalidSeriatimInput(t *testing.T) {
|
||||
resolved, err := loadPipelineConfig(t).Resolve(configResolveInput(t))
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
output, err := runSeriatimPipeline(t, seriatimRunnerRegistries(t, &runnerSeriatimExtractor{}), resolved.ResolvedPipeline, pipeline.RunInput{
|
||||
RawInput: []byte(`{"metadata":{},"segments":[]}`),
|
||||
SourceID: "invalid-source",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Run() error = nil, want invalid input error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "parse input with adapter") || !strings.Contains(err.Error(), "seriatim input") {
|
||||
t.Fatalf("Run() error = %q, want Seriatim parse context", err.Error())
|
||||
}
|
||||
if output.ArtifactCount() != 0 {
|
||||
t.Fatalf("invalid input produced files: %#v", output.OutputFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func configResolveInput(t *testing.T) config.ResolveInput {
|
||||
t.Helper()
|
||||
return config.ResolveInput{
|
||||
PipelineID: "seriatim-fixture",
|
||||
Catalog: seriatimTestCatalog(t, transcript.ModuleSpec()),
|
||||
}
|
||||
}
|
||||
|
||||
func seriatimRunnerRegistries(t *testing.T, extractor contracts.Extractor[seriatimArtifact]) pipeline.Registries {
|
||||
t.Helper()
|
||||
|
||||
inputs := pipeline.NewInputAdapterRegistry()
|
||||
chunkers := pipeline.NewChunkerRegistry()
|
||||
extractors := pipeline.NewExtractorRegistry()
|
||||
mergers := pipeline.NewMergerRegistry()
|
||||
normalizers := pipeline.NewNormalizerRegistry()
|
||||
outputs := pipeline.NewOutputEncoderRegistry()
|
||||
|
||||
if err := transcript.Register(inputs); err != nil {
|
||||
t.Fatalf("register seriatim input: %v", err)
|
||||
}
|
||||
if err := chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "fake/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.Chunker, error) {
|
||||
return runnerSeriatimChunker{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
if err := pipeline.RegisterExtractor[seriatimArtifact](extractors, pipeline.ModuleSpec{Key: "fake/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind, Requires: []string{"chunks", "transcript.speaker", "transcript.timestamps"}, Provides: []string{"fake.artifacts"}}, func() (contracts.Extractor[seriatimArtifact], error) {
|
||||
return extractor, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register extractor: %v", err)
|
||||
}
|
||||
if err := pipeline.RegisterMerger[seriatimArtifact](mergers, pipeline.ModuleSpec{Key: pipeline.DefaultMergeModule, Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind}, func() (contracts.Merger[seriatimArtifact], error) {
|
||||
return fakeMerger{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register merger: %v", err)
|
||||
}
|
||||
if err := pipeline.RegisterNormalizer[seriatimArtifact](normalizers, pipeline.ModuleSpec{Key: pipeline.DefaultNormalizeModule, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: seriatimArtifactKind}, func() (contracts.Normalizer[seriatimArtifact], error) {
|
||||
return fakeNormalizer{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register normalizer: %v", err)
|
||||
}
|
||||
if err := outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: pipeline.DefaultOutputModule, Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.OutputEncoder, error) {
|
||||
return jsonoutput.New(), nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register output: %v", err)
|
||||
}
|
||||
|
||||
codecs := pipeline.NewArtifactCodecRegistry()
|
||||
if err := pipeline.RegisterArtifactCodec(codecs, seriatimArtifactCodec{}); err != nil {
|
||||
t.Fatalf("register artifact codec: %v", err)
|
||||
}
|
||||
return pipeline.Registries{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
ArtifactCodecs: codecs,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
type runnerSeriatimChunker struct{}
|
||||
|
||||
func (runnerSeriatimChunker) Key() string {
|
||||
return "fake/chunk"
|
||||
}
|
||||
|
||||
func (runnerSeriatimChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runnerSeriatimChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
||||
return contracts.ChunkPlanResult{
|
||||
Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID}}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type runnerSeriatimExtractor struct {
|
||||
calls int
|
||||
}
|
||||
|
||||
func (e *runnerSeriatimExtractor) Key() string {
|
||||
return "fake/extract"
|
||||
}
|
||||
|
||||
func (e *runnerSeriatimExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *runnerSeriatimExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[seriatimArtifact]) (contracts.TypedExtractionResult[seriatimArtifact], error) {
|
||||
e.calls++
|
||||
if req.Source == nil {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source must not be nil")
|
||||
}
|
||||
if req.Chunk == nil {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk must not be nil")
|
||||
}
|
||||
if got := unitIDs(req.Source.Units); !equalInts(got, []int{1, 2}) {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("source unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
}
|
||||
if got := unitIDs(req.Chunk.Units); !equalInts(got, []int{1, 2}) {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("chunk unit IDs = %#v, want Seriatim segment IDs", got)
|
||||
}
|
||||
for _, unit := range req.Chunk.Units {
|
||||
if speaker, ok := transcript.Speaker(unit); !ok || speaker == "" {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing speaker metadata", unit.ID)
|
||||
}
|
||||
if _, ok := transcript.Start(unit); !ok {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing start metadata", unit.ID)
|
||||
}
|
||||
if _, ok := transcript.End(unit); !ok {
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{}, fmt.Errorf("unit %d missing end metadata", unit.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return contracts.TypedExtractionResult[seriatimArtifact]{Value: seriatimArtifact{
|
||||
Value: "seriatim-source-ref",
|
||||
SourceRefs: []source.SourceRef{
|
||||
{
|
||||
SourceID: req.Source.ID,
|
||||
StartUnitID: req.Chunk.Units[0].ID,
|
||||
EndUnitID: req.Chunk.Units[len(req.Chunk.Units)-1].ID,
|
||||
},
|
||||
},
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func unitIDs(units []source.SourceUnit) []int {
|
||||
ids := make([]int, 0, len(units))
|
||||
for _, unit := range units {
|
||||
ids = append(ids, unit.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func equalInts(a, b []int) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var (
|
||||
_ contracts.Chunker = runnerSeriatimChunker{}
|
||||
_ contracts.Extractor[seriatimArtifact] = (*runnerSeriatimExtractor)(nil)
|
||||
)
|
||||
Reference in New Issue
Block a user