Files
notarius/internal/modules/integration/native_spell_pipeline_test.go

198 lines
9.5 KiB
Go
Raw Normal View History

package integration_test
import (
"context"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
spellcatalogvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
spellsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
"go.maximumdirect.net/promptkit"
)
// Count the real validator boundary, not a second implementation of its rules.
type observedSpellValidator struct {
contracts.TypedValidator[dnd.SpellList]
record func(string)
}
func (v observedSpellValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
v.record(v.Name())
return v.TypedValidator.Validate(ctx, req)
}
func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) {
response := func(name string) string {
return fmt.Sprintf(`{"spell_casts":[{"caster":"Aria","spell":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name)
}
for _, test := range []struct {
name string
mode contracts.ExecutionMode
responses []string
wantError bool
fallbacks int
}{
{"initial acceptance", contracts.ModeStrict, []string{response("Cure Wounds")}, false, 0},
{"final correction accepted", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Cure Wounds")}, false, 0},
{"strict exhaustion", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, true, 0},
{"permissive empty fallback", contracts.ModePermissive, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, false, 1},
} {
t.Run(test.name, func(t *testing.T) {
client := &nativeSequenceClient{responses: append([]string(nil), test.responses...)}
engine := nativeTestEngine(t, client, "profile-spells")
registries := pipeline.Registries{
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
Outputs: pipeline.NewOutputEncoderRegistry(),
}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
check(transcript.Register(registries.Inputs))
check(units.Register(registries.Chunkers))
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, spellcodec.New()))
check(spells.Register(registries.Extractors))
check(appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, func(values []dnd.SpellList) (dnd.SpellList, error) {
result := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
for _, value := range values {
result.SpellCasts = append(result.SpellCasts, value.SpellCasts...)
}
return result, nil
}))
check(noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind))
check(jsonoutput.Register(registries.Outputs))
catalogValidator, err := spellcatalogvalidator.New(spellcatalogvalidator.Options{})
check(err)
validators := []contracts.TypedValidator[dnd.SpellList]{
spellshape.New(spellshape.Options{}), spellsourcerefs.New(spellsourcerefs.Options{}),
catalogValidator, spellsourcerelatedness.New(spellsourcerelatedness.Options{}),
}
var mu sync.Mutex
var calls []string
record := func(name string) { mu.Lock(); defer mu.Unlock(); calls = append(calls, name) }
var bindings []pipeline.ModuleBinding
var names []string
for _, validator := range validators {
check(pipeline.RegisterTypedValidator(registries.Validators, dnd.SpellListKind,
pipeline.ValidatorSpec{Key: validator.Name(), ExecutionClass: contracts.ExecutionClassDeterministic},
func() (contracts.TypedValidator[dnd.SpellList], error) {
return observedSpellValidator{validator, record}, nil
}))
bindings = append(bindings, pipeline.Binding(validator.Name()))
names = append(names, validator.Name())
}
partyFile := filepath.Join(t.TempDir(), "party.txt")
check(os.WriteFile(partyFile, []byte("party-grounding-sentinel: Aria"), 0600))
turns := 2
profile := pipeline.PipelineProfile{
ID: "native-spells", Input: pipeline.Binding(transcript.Key),
Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
LLMProfile: "profile-spells",
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {
Extract: pipeline.ModuleBinding{Module: spells.Key, Mode: &test.mode, MaxAdditionalTurns: &turns,
Validators: pipeline.ValidatorOverride{Set: true, Validators: bindings},
References: map[string]pipeline.ReferenceSource{"party": pipeline.ExternalReference(partyFile)}},
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
}},
}
catalog := pipeline.ModuleCatalog{
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
}
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
check(err)
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
check(err)
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
check(err)
model := "run-wide-spell-model"
var preparations []promptkit.PreparedRun
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", Workers: 1,
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`),
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model},
OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }},
})
if (err != nil) != test.wantError {
t.Fatalf("pipeline error = %v, want error %t", err, test.wantError)
}
wantCalls := len(test.responses)
if len(client.requests) != wantCalls || len(preparations) != wantCalls {
t.Fatalf("provider/preparation calls = %d/%d, want %d", len(client.requests), len(preparations), wantCalls)
}
var expected []string
for range wantCalls {
expected = append(expected, names...)
}
if !reflect.DeepEqual(calls, expected) {
t.Fatalf("ordered assessments = %v, want %v (no outer assessment)", calls, expected)
}
for i, request := range client.requests {
if request.Target.Model != model || request.Prompt.SessionID != "spell-session-override" || preparations[i].SelectedProfileID != "profile-spells" {
t.Fatal("native profile/session/execution override lost")
}
wantID := spells.PromptID
if i > 0 {
wantID = spells.CorrectionPromptID
}
if preparations[i].PromptID != wantID {
t.Fatalf("prompt = %q, want %q", preparations[i].PromptID, wantID)
}
if i == 0 {
continue
}
previous := client.requests[i-1].Prompt.Messages
current := request.Prompt.Messages
if len(current) != len(previous)+2 || !reflect.DeepEqual(current[:len(previous)], previous) {
t.Fatal("native continuation did not retain exact ordered history")
}
if current[len(previous)].Role != "assistant" || current[len(previous)].Content != test.responses[i-1] ||
!strings.Contains(current[len(previous)+1].Content, "recognized D&D spell") {
t.Fatal("correction lost the actual candidate or catalog guidance")
}
if strings.Count(nativePromptText(request), "party-grounding-sentinel") != 1 {
t.Fatal("reference bytes were regenerated in correction")
}
}
if test.wantError {
return
}
if output.FallbackCount != test.fallbacks || len(output.OutputFiles) != 1 {
t.Fatalf("outcome = %#v", output)
}
value, err := spellcodec.New().Decode(output.OutputFiles[0].Bytes)
check(err)
if test.fallbacks > 0 {
if value.SpellCasts == nil || len(value.SpellCasts) != 0 {
t.Fatalf("invalid empty fallback: %#v", value)
}
} else if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" || value.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("durable spell artifact = %#v", value)
}
})
}
}