2026-11-22 04:33:28 +00:00
|
|
|
package integration_test
|
2026-11-22 00:02:13 +00:00
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"fmt"
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"reflect"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
|
|
|
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
2026-11-22 04:33:28 +00:00
|
|
|
spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
2026-11-22 00:02:13 +00:00
|
|
|
spellcatalogvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog"
|
|
|
|
|
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
|
|
|
|
|
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
|
|
|
|
|
spellsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
2026-11-22 04:19:23 +00:00
|
|
|
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json"
|
2026-11-22 00:02:13 +00:00
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
|
|
|
|
"go.maximumdirect.net/promptkit"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// Count the real validator boundary, not a second implementation of its rules.
|
|
|
|
|
type observedSpellValidator struct {
|
|
|
|
|
contracts.TypedValidator[dnd.SpellList]
|
|
|
|
|
record func(string)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (v observedSpellValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
|
|
|
|
|
v.record(v.Name())
|
|
|
|
|
return v.TypedValidator.Validate(ctx, req)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) {
|
|
|
|
|
response := func(name string) string {
|
|
|
|
|
return fmt.Sprintf(`{"spell_casts":[{"caster":"Aria","spell":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name)
|
|
|
|
|
}
|
|
|
|
|
for _, test := range []struct {
|
|
|
|
|
name string
|
|
|
|
|
mode contracts.ExecutionMode
|
|
|
|
|
responses []string
|
|
|
|
|
wantError bool
|
|
|
|
|
fallbacks int
|
|
|
|
|
}{
|
|
|
|
|
{"initial acceptance", contracts.ModeStrict, []string{response("Cure Wounds")}, false, 0},
|
|
|
|
|
{"final correction accepted", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Cure Wounds")}, false, 0},
|
|
|
|
|
{"strict exhaustion", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, true, 0},
|
|
|
|
|
{"permissive empty fallback", contracts.ModePermissive, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, false, 1},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(test.name, func(t *testing.T) {
|
2026-11-22 04:33:28 +00:00
|
|
|
client := &nativeSequenceClient{responses: append([]string(nil), test.responses...)}
|
|
|
|
|
engine := nativeTestEngine(t, client, "profile-spells")
|
2026-11-22 00:02:13 +00:00
|
|
|
registries := pipeline.Registries{
|
|
|
|
|
Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(),
|
|
|
|
|
ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(),
|
|
|
|
|
Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(),
|
|
|
|
|
Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
|
|
|
|
Outputs: pipeline.NewOutputEncoderRegistry(),
|
|
|
|
|
}
|
|
|
|
|
check := func(err error) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
check(transcript.Register(registries.Inputs))
|
|
|
|
|
check(units.Register(registries.Chunkers))
|
|
|
|
|
check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, spellcodec.New()))
|
2026-11-22 04:33:28 +00:00
|
|
|
check(spells.Register(registries.Extractors))
|
2026-11-22 00:02:13 +00:00
|
|
|
check(appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, func(values []dnd.SpellList) (dnd.SpellList, error) {
|
|
|
|
|
result := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
|
|
|
|
for _, value := range values {
|
|
|
|
|
result.SpellCasts = append(result.SpellCasts, value.SpellCasts...)
|
|
|
|
|
}
|
|
|
|
|
return result, nil
|
|
|
|
|
}))
|
|
|
|
|
check(noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind))
|
2026-11-22 04:19:23 +00:00
|
|
|
check(jsonoutput.Register(registries.Outputs))
|
2026-11-22 00:02:13 +00:00
|
|
|
catalogValidator, err := spellcatalogvalidator.New(spellcatalogvalidator.Options{})
|
|
|
|
|
check(err)
|
|
|
|
|
validators := []contracts.TypedValidator[dnd.SpellList]{
|
|
|
|
|
spellshape.New(spellshape.Options{}), spellsourcerefs.New(spellsourcerefs.Options{}),
|
|
|
|
|
catalogValidator, spellsourcerelatedness.New(spellsourcerelatedness.Options{}),
|
|
|
|
|
}
|
|
|
|
|
var mu sync.Mutex
|
|
|
|
|
var calls []string
|
|
|
|
|
record := func(name string) { mu.Lock(); defer mu.Unlock(); calls = append(calls, name) }
|
|
|
|
|
var bindings []pipeline.ModuleBinding
|
|
|
|
|
var names []string
|
|
|
|
|
for _, validator := range validators {
|
|
|
|
|
check(pipeline.RegisterTypedValidator(registries.Validators, dnd.SpellListKind,
|
|
|
|
|
pipeline.ValidatorSpec{Key: validator.Name(), ExecutionClass: contracts.ExecutionClassDeterministic},
|
|
|
|
|
func() (contracts.TypedValidator[dnd.SpellList], error) {
|
|
|
|
|
return observedSpellValidator{validator, record}, nil
|
|
|
|
|
}))
|
|
|
|
|
bindings = append(bindings, pipeline.Binding(validator.Name()))
|
|
|
|
|
names = append(names, validator.Name())
|
|
|
|
|
}
|
|
|
|
|
partyFile := filepath.Join(t.TempDir(), "party.txt")
|
|
|
|
|
check(os.WriteFile(partyFile, []byte("party-grounding-sentinel: Aria"), 0600))
|
|
|
|
|
turns := 2
|
|
|
|
|
profile := pipeline.PipelineProfile{
|
|
|
|
|
ID: "native-spells", Input: pipeline.Binding(transcript.Key),
|
2026-11-22 04:19:23 +00:00
|
|
|
Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key),
|
2026-11-22 00:02:13 +00:00
|
|
|
LLMProfile: "profile-spells",
|
|
|
|
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {
|
2026-11-22 04:33:28 +00:00
|
|
|
Extract: pipeline.ModuleBinding{Module: spells.Key, Mode: &test.mode, MaxAdditionalTurns: &turns,
|
2026-11-22 00:02:13 +00:00
|
|
|
Validators: pipeline.ValidatorOverride{Set: true, Validators: bindings},
|
|
|
|
|
References: map[string]pipeline.ReferenceSource{"party": pipeline.ExternalReference(partyFile)}},
|
|
|
|
|
Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key),
|
|
|
|
|
}},
|
|
|
|
|
}
|
|
|
|
|
catalog := pipeline.ModuleCatalog{
|
|
|
|
|
Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs,
|
|
|
|
|
Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers,
|
|
|
|
|
Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs,
|
|
|
|
|
}
|
|
|
|
|
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog)
|
|
|
|
|
check(err)
|
|
|
|
|
resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
|
|
|
check(err)
|
|
|
|
|
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine})
|
|
|
|
|
check(err)
|
|
|
|
|
model := "run-wide-spell-model"
|
|
|
|
|
var preparations []promptkit.PreparedRun
|
|
|
|
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
2026-11-22 01:45:31 +00:00
|
|
|
Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", Workers: 1,
|
2026-11-22 04:19:23 +00:00
|
|
|
RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`),
|
2026-11-22 00:02:13 +00:00
|
|
|
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model},
|
|
|
|
|
OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }},
|
|
|
|
|
})
|
|
|
|
|
if (err != nil) != test.wantError {
|
|
|
|
|
t.Fatalf("pipeline error = %v, want error %t", err, test.wantError)
|
|
|
|
|
}
|
|
|
|
|
wantCalls := len(test.responses)
|
|
|
|
|
if len(client.requests) != wantCalls || len(preparations) != wantCalls {
|
|
|
|
|
t.Fatalf("provider/preparation calls = %d/%d, want %d", len(client.requests), len(preparations), wantCalls)
|
|
|
|
|
}
|
|
|
|
|
var expected []string
|
|
|
|
|
for range wantCalls {
|
|
|
|
|
expected = append(expected, names...)
|
|
|
|
|
}
|
|
|
|
|
if !reflect.DeepEqual(calls, expected) {
|
|
|
|
|
t.Fatalf("ordered assessments = %v, want %v (no outer assessment)", calls, expected)
|
|
|
|
|
}
|
|
|
|
|
for i, request := range client.requests {
|
|
|
|
|
if request.Target.Model != model || request.Prompt.SessionID != "spell-session-override" || preparations[i].SelectedProfileID != "profile-spells" {
|
|
|
|
|
t.Fatal("native profile/session/execution override lost")
|
|
|
|
|
}
|
2026-11-22 04:33:28 +00:00
|
|
|
wantID := spells.PromptID
|
2026-11-22 00:02:13 +00:00
|
|
|
if i > 0 {
|
2026-11-22 04:33:28 +00:00
|
|
|
wantID = spells.CorrectionPromptID
|
2026-11-22 00:02:13 +00:00
|
|
|
}
|
|
|
|
|
if preparations[i].PromptID != wantID {
|
|
|
|
|
t.Fatalf("prompt = %q, want %q", preparations[i].PromptID, wantID)
|
|
|
|
|
}
|
|
|
|
|
if i == 0 {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
previous := client.requests[i-1].Prompt.Messages
|
|
|
|
|
current := request.Prompt.Messages
|
|
|
|
|
if len(current) != len(previous)+2 || !reflect.DeepEqual(current[:len(previous)], previous) {
|
|
|
|
|
t.Fatal("native continuation did not retain exact ordered history")
|
|
|
|
|
}
|
|
|
|
|
if current[len(previous)].Role != "assistant" || current[len(previous)].Content != test.responses[i-1] ||
|
|
|
|
|
!strings.Contains(current[len(previous)+1].Content, "recognized D&D spell") {
|
|
|
|
|
t.Fatal("correction lost the actual candidate or catalog guidance")
|
|
|
|
|
}
|
2026-11-22 04:33:28 +00:00
|
|
|
if strings.Count(nativePromptText(request), "party-grounding-sentinel") != 1 {
|
2026-11-22 00:02:13 +00:00
|
|
|
t.Fatal("reference bytes were regenerated in correction")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if test.wantError {
|
|
|
|
|
return
|
|
|
|
|
}
|
2026-11-22 04:19:23 +00:00
|
|
|
if output.FallbackCount != test.fallbacks || len(output.OutputFiles) != 1 {
|
2026-11-22 00:02:13 +00:00
|
|
|
t.Fatalf("outcome = %#v", output)
|
|
|
|
|
}
|
2026-11-22 04:19:23 +00:00
|
|
|
value, err := spellcodec.New().Decode(output.OutputFiles[0].Bytes)
|
2026-11-22 00:02:13 +00:00
|
|
|
check(err)
|
|
|
|
|
if test.fallbacks > 0 {
|
|
|
|
|
if value.SpellCasts == nil || len(value.SpellCasts) != 0 {
|
|
|
|
|
t.Fatalf("invalid empty fallback: %#v", value)
|
|
|
|
|
}
|
|
|
|
|
} else if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" || value.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
|
|
|
|
|
t.Fatalf("durable spell artifact = %#v", value)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|