package integration_test import ( "context" "fmt" "os" "path/filepath" "reflect" "strings" "sync" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells" spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" spellcatalogvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/catalog" spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape" spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs" spellsourcerelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" "go.maximumdirect.net/promptkit" ) // Count the real validator boundary, not a second implementation of its rules. type observedSpellValidator struct { contracts.TypedValidator[dnd.SpellList] record func(string) } func (v observedSpellValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) { v.record(v.Name()) return v.TypedValidator.Validate(ctx, req) } func TestNativeSpellPipelineAssessesEveryCandidateOnce(t *testing.T) { response := func(name string) string { return fmt.Sprintf(`{"spell_casts":[{"caster":"Aria","spell":%q,"source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, name) } for _, test := range []struct { name string mode contracts.ExecutionMode responses []string wantError bool fallbacks int }{ {"initial acceptance", contracts.ModeStrict, []string{response("Cure Wounds")}, false, 0}, {"final correction accepted", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Cure Wounds")}, false, 0}, {"strict exhaustion", contracts.ModeStrict, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, true, 0}, {"permissive empty fallback", contracts.ModePermissive, []string{response("Unknown Spell"), response("Unknown Spell"), response("Unknown Spell")}, false, 1}, } { t.Run(test.name, func(t *testing.T) { client := &nativeSequenceClient{responses: append([]string(nil), test.responses...)} engine := nativeTestEngine(t, client, "profile-spells") registries := pipeline.Registries{ Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry(), } check := func(err error) { t.Helper() if err != nil { t.Fatal(err) } } check(transcript.Register(registries.Inputs)) check(units.Register(registries.Chunkers)) check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, spellcodec.New())) check(spells.Register(registries.Extractors)) check(appendorder.RegisterTyped(registries.Mergers, dnd.SpellListKind, func(values []dnd.SpellList) (dnd.SpellList, error) { result := dnd.SpellList{SpellCasts: []dnd.SpellCast{}} for _, value := range values { result.SpellCasts = append(result.SpellCasts, value.SpellCasts...) } return result, nil })) check(noop.RegisterTyped[dnd.SpellList](registries.Normalizers, dnd.SpellListKind)) check(jsonoutput.Register(registries.Outputs)) catalogValidator, err := spellcatalogvalidator.New(spellcatalogvalidator.Options{}) check(err) validators := []contracts.TypedValidator[dnd.SpellList]{ spellshape.New(spellshape.Options{}), spellsourcerefs.New(spellsourcerefs.Options{}), catalogValidator, spellsourcerelatedness.New(spellsourcerelatedness.Options{}), } var mu sync.Mutex var calls []string record := func(name string) { mu.Lock(); defer mu.Unlock(); calls = append(calls, name) } var bindings []pipeline.ModuleBinding var names []string for _, validator := range validators { check(pipeline.RegisterTypedValidator(registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: validator.Name(), ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) { return observedSpellValidator{validator, record}, nil })) bindings = append(bindings, pipeline.Binding(validator.Name())) names = append(names, validator.Name()) } partyFile := filepath.Join(t.TempDir(), "party.txt") check(os.WriteFile(partyFile, []byte("party-grounding-sentinel: Aria"), 0600)) turns := 2 profile := pipeline.PipelineProfile{ ID: "native-spells", Input: pipeline.Binding(transcript.Key), Chunk: pipeline.Binding(units.Key), Output: pipeline.Binding(jsonoutput.Key), LLMProfile: "profile-spells", Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": { Extract: pipeline.ModuleBinding{Module: spells.Key, Mode: &test.mode, MaxAdditionalTurns: &turns, Validators: pipeline.ValidatorOverride{Set: true, Validators: bindings}, References: map[string]pipeline.ReferenceSource{"party": pipeline.ExternalReference(partyFile)}}, Merge: pipeline.Binding(appendorder.Key), Normalize: pipeline.Binding(noop.Key), }}, } catalog := pipeline.ModuleCatalog{ Inputs: registries.Inputs, Chunkers: registries.Chunkers, ArtifactCodecs: registries.ArtifactCodecs, Extractors: registries.Extractors, Mergers: registries.Mergers, Normalizers: registries.Normalizers, Validators: registries.Validators, ValidatorChains: registries.ValidatorChains, Outputs: registries.Outputs, } resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{}, catalog) check(err) resolved, _, err = pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{}) check(err) prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{Engine: engine}) check(err) model := "run-wide-spell-model" var preparations []promptkit.PreparedRun output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ Prepared: prepared, SourceID: "spell-session", SessionID: "spell-session-override", Workers: 1, RawInput: []byte(`{"metadata":{},"segments":[{"id":1,"speaker":"Alice","start":0,"end":1,"text":"I cast Cure Wounds."}]}`), Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: model}, OnPrepared: func(_ context.Context, value promptkit.PreparedRun) { preparations = append(preparations, value) }}, }) if (err != nil) != test.wantError { t.Fatalf("pipeline error = %v, want error %t", err, test.wantError) } wantCalls := len(test.responses) if len(client.requests) != wantCalls || len(preparations) != wantCalls { t.Fatalf("provider/preparation calls = %d/%d, want %d", len(client.requests), len(preparations), wantCalls) } var expected []string for range wantCalls { expected = append(expected, names...) } if !reflect.DeepEqual(calls, expected) { t.Fatalf("ordered assessments = %v, want %v (no outer assessment)", calls, expected) } for i, request := range client.requests { if request.Target.Model != model || request.Prompt.SessionID != "spell-session-override" || preparations[i].SelectedProfileID != "profile-spells" { t.Fatal("native profile/session/execution override lost") } wantID := spells.PromptID if i > 0 { wantID = spells.CorrectionPromptID } if preparations[i].PromptID != wantID { t.Fatalf("prompt = %q, want %q", preparations[i].PromptID, wantID) } if i == 0 { continue } previous := client.requests[i-1].Prompt.Messages current := request.Prompt.Messages if len(current) != len(previous)+2 || !reflect.DeepEqual(current[:len(previous)], previous) { t.Fatal("native continuation did not retain exact ordered history") } if current[len(previous)].Role != "assistant" || current[len(previous)].Content != test.responses[i-1] || !strings.Contains(current[len(previous)+1].Content, "recognized D&D spell") { t.Fatal("correction lost the actual candidate or catalog guidance") } if strings.Count(nativePromptText(request), "party-grounding-sentinel") != 1 { t.Fatal("reference bytes were regenerated in correction") } } if test.wantError { return } if output.FallbackCount != test.fallbacks || len(output.OutputFiles) != 1 { t.Fatalf("outcome = %#v", output) } value, err := spellcodec.New().Decode(output.OutputFiles[0].Bytes) check(err) if test.fallbacks > 0 { if value.SpellCasts == nil || len(value.SpellCasts) != 0 { t.Fatalf("invalid empty fallback: %#v", value) } } else if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" || value.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" { t.Fatalf("durable spell artifact = %#v", value) } }) } }