150 lines
7.0 KiB
Go
150 lines
7.0 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
|
)
|
|
|
|
func TestMaterializedSpellCatalogOrderingPreservesSemanticContents(t *testing.T) {
|
|
components := productionTestComponents(t)
|
|
configPath := writeProductionSpellCatalogContractConfig(t)
|
|
catalog := catalogFromRegistries(components.registries)
|
|
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resolved := effective.ResolvedPipeline
|
|
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
|
|
for _, target := range []*pipeline.ResolvedReferenceTarget{&resolved.Steps[0].ArtifactLanes[0].ExtractReferences, &resolved.Steps[0].ArtifactLanes[0].NormalizeReferences} {
|
|
matched := false
|
|
for index := range target.Bindings {
|
|
if target.Bindings[index].SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
|
target.Bindings[index].Source = overlayPath
|
|
matched = true
|
|
}
|
|
}
|
|
if !matched {
|
|
t.Fatal("missing configured spell catalog binding")
|
|
}
|
|
}
|
|
materialize := func(contents string) pipeline.ResolvedPipeline {
|
|
t.Helper()
|
|
if err := os.WriteFile(overlayPath, []byte(contents), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value, _, err := pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return value
|
|
}
|
|
first, second := materialize(reorderedOverlayA), materialize(reorderedOverlayB)
|
|
laneA, laneB := first.Steps[0].ArtifactLanes[0], second.Steps[0].ArtifactLanes[0]
|
|
for _, pair := range [][2]pipeline.ResolvedReferenceTarget{
|
|
{laneA.ExtractReferences, laneB.ExtractReferences},
|
|
{laneA.NormalizeReferences, laneB.NormalizeReferences},
|
|
} {
|
|
itemsA, itemsB := pair[0].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items, pair[1].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items
|
|
if len(itemsA) != 1 || len(itemsB) != 1 {
|
|
t.Fatal("materialized catalog missing")
|
|
}
|
|
if itemsA[0].Digest == itemsB[0].Digest || itemsA[0].Origin != itemsB[0].Origin {
|
|
t.Fatalf("raw catalog provenance=%#v/%#v", itemsA, itemsB)
|
|
}
|
|
catalogA, err := spellcatalog.ResolveEffectiveCatalog(pair[0].ReferenceSet)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
catalogB, err := spellcatalog.ResolveEffectiveCatalog(pair[1].ReferenceSet)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if catalogA.Digest() != catalogB.Digest() || !reflect.DeepEqual(catalogA.OverlayIDs(), []string{"campaign.a", "campaign.b"}) || !reflect.DeepEqual(catalogB.OverlayIDs(), catalogA.OverlayIDs()) || !reflect.DeepEqual(catalogA.PromptSpells(), catalogB.PromptSpells()) {
|
|
t.Fatal("catalog ordering changed semantic recognition")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
|
|
base := productionSpellCatalogContractConfig(t)
|
|
changed := strings.Replace(base, repositoryPath("examples", "dnd-spell-catalog.json"), filepath.Join(t.TempDir(), "alternate-spell-catalog.json"), 1)
|
|
if changed == base {
|
|
t.Fatal("production configuration did not contain the maintained catalog binding")
|
|
}
|
|
root := t.TempDir()
|
|
firstPath := filepath.Join(root, "first.yml")
|
|
secondPath := filepath.Join(root, "second.yml")
|
|
if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
components := productionTestComponents(t)
|
|
first, err := loadMaintainedExample(t, firstPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
|
if err != nil {
|
|
t.Fatalf("resolve first configuration: %v", err)
|
|
}
|
|
second, err := loadMaintainedExample(t, secondPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
|
if err != nil {
|
|
t.Fatalf("resolve changed configuration: %v", err)
|
|
}
|
|
if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
|
|
t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest)
|
|
}
|
|
}
|
|
|
|
func TestProductionOverlayRunAlignsGroundingAndValidation(t *testing.T) {
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
client := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
|
|
configPath := writeProductionContractConfig(t, productionSpellCatalogContractConfig(t)+productionProviderConfiguration)
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client))
|
|
if code != 0 {
|
|
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
var report runResult
|
|
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.ArtifactCount != 1 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) {
|
|
t.Fatalf("completion=%#v", report)
|
|
}
|
|
requests := client.requestsWithProperty("spell_casts")
|
|
if len(requests) != 1 {
|
|
t.Fatalf("native spell requests=%d", len(requests))
|
|
}
|
|
var prompt strings.Builder
|
|
for _, message := range requests[0].Prompt.Messages {
|
|
prompt.WriteString(message.Content)
|
|
}
|
|
for _, fragment := range []string{`"canonical_name":"Aegis of Emberfall"`, `"aliases":["Emberfall Aegis"]`} {
|
|
if !strings.Contains(prompt.String(), fragment) {
|
|
t.Fatalf("catalog prompt omitted overlay recognition %q", fragment)
|
|
}
|
|
}
|
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(report.OutputDirectory, "lanes", "spells.json"))
|
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
|
t.Fatalf("spell artifact=%#v", artifact)
|
|
}
|
|
}
|
|
|
|
const reorderedOverlayA = `{
|
|
"schema_version": "notarius.dnd.spell-catalog-overlay.v1",
|
|
"catalogs": [
|
|
{"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]},
|
|
{"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]}
|
|
]
|
|
}`
|
|
|
|
const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`
|