notarius/internal/cli/spell_catalog_identity_contract_test.go

package cli

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
	spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
	spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
)

func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
	components := productionTestComponents(t)
	configPath := writeProductionSpellCatalogContractConfig(t)
	effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
	if err != nil {
		t.Fatalf("resolve production configuration: %v", err)
	}
	overlayPath := filepath.Join(t.TempDir(), "catalog.json")
	resolved := effective.ResolvedPipeline
	bindings := resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings
	catalogBindingIndex := -1
	for index, binding := range bindings {
		if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
			catalogBindingIndex = index
			break
		}
	}
	if catalogBindingIndex < 0 {
		t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
	}
	resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
	normalizeBindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings
	normalizeCatalogBindingIndex := -1
	for index, binding := range normalizeBindings {
		if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
			normalizeCatalogBindingIndex = index
			break
		}
	}
	if normalizeCatalogBindingIndex < 0 {
		t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings)
	}
	resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath

	if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
		t.Fatal(err)
	}
	materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
	if err != nil {
		t.Fatalf("materialize first catalog: %v", err)
	}
	identityA := catalogCheckpointIdentity(t, materializedA)
	metadataA := catalogExtractorMetadata(t, materializedA)
	normalizerMetadataA := catalogNormalizerMetadata(t, materializedA)
	referenceA := catalogReference(t, materializedA)

	if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
		t.Fatal(err)
	}
	materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
	if err != nil {
		t.Fatalf("materialize reordered catalog: %v", err)
	}
	identityB := catalogCheckpointIdentity(t, materializedB)
	metadataB := catalogExtractorMetadata(t, materializedB)
	normalizerMetadataB := catalogNormalizerMetadata(t, materializedB)
	referenceB := catalogReference(t, materializedB)

	if identityA.Digest == identityB.Digest {
		t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
	}
	if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
		t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
	}
	digestA, ok := metadataA["catalog_digest"].(string)
	if !ok {
		t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
	}
	digestB, ok := metadataB["catalog_digest"].(string)
	if !ok || digestA != digestB {
		t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
	}
	if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
		t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
	}
	normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string)
	normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string)
	if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB {
		t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB)
	}
	if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) {
		t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want)
	}
}

func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing.T) {
	base := productionSpellCatalogContractConfig(t)
	changed := strings.Replace(base, repositoryPath("examples", "dnd-spell-catalog.json"), filepath.Join(t.TempDir(), "alternate-spell-catalog.json"), 1)
	if changed == base {
		t.Fatal("production configuration did not contain the maintained catalog binding")
	}
	root := t.TempDir()
	firstPath := filepath.Join(root, "first.yml")
	secondPath := filepath.Join(root, "second.yml")
	if err := os.WriteFile(firstPath, []byte(base), 0o600); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(secondPath, []byte(changed), 0o600); err != nil {
		t.Fatal(err)
	}
	components := productionTestComponents(t)
	first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
	if err != nil {
		t.Fatalf("resolve first configuration: %v", err)
	}
	second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
	if err != nil {
		t.Fatalf("resolve changed configuration: %v", err)
	}
	if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest {
		t.Fatalf("resolved pipeline digest = %q for different catalog bindings, want change", first.ResolvedPipeline.Digest)
	}
}

func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
	components := productionTestComponents(t)
	configPath := writeProductionSpellCatalogContractConfig(t)
	effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
	if err != nil {
		t.Fatal(err)
	}
	materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
	if err != nil {
		t.Fatal(err)
	}
	prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
	if err != nil {
		t.Fatal(err)
	}
	fingerprints := prepared.CheckpointFingerprints()
	wantNames := map[string]struct{}{
		"extract:spells:" + spells.Key + ":effective_catalog":                                                  {},
		"extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog":           {},
		"normalize:spells:" + spellnormalize.Key + ":effective_catalog":                                        {},
		"normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
	}
	seen := make(map[string]string, len(fingerprints))
	for _, fingerprint := range fingerprints {
		if _, ok := wantNames[fingerprint.Name]; ok {
			seen[fingerprint.Name] = fingerprint.Value
		}
	}
	if len(seen) != len(wantNames) {
		t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints)
	}
	var catalogDigest string
	for name, value := range seen {
		if catalogDigest == "" {
			catalogDigest = value
		} else if value != catalogDigest {
			t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest)
		}
	}

	identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
		identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
			Pipeline:               materialized,
			InputKey:               materialized.Input.Module,
			RawInputDigest:         "sha256:unchanged-input",
			References:             pipeline.ReferenceProvenance(materialized),
			ProvenanceFingerprints: checkpointIdentityFingerprints(values),
		})
		if identityErr != nil {
			t.Fatal(identityErr)
		}
		return identity
	}
	first := identityFor(fingerprints)
	changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
	assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
	second := identityFor(changed)
	if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
		t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
	}
}

func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
	components := productionTestComponents(t)
	configPath := writeProductionSpellCatalogContractConfig(t)
	effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
	if err != nil {
		t.Fatal(err)
	}
	materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
	if err != nil {
		t.Fatal(err)
	}
	prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
	if err != nil {
		t.Fatal(err)
	}
	fingerprints := prepared.CheckpointFingerprints()
	llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
	settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
	recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
	if err != nil {
		t.Fatal(err)
	}
	doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
	doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
	doc.Digest, err = source.DigestDocument(&doc)
	if err != nil {
		t.Fatal(err)
	}
	if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
		t.Fatal(err)
	}
	normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}}
	normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"}
	normalizeArtifact := pipeline.CheckpointArtifact{
		LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID,
		SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema),
		Artifact: contracts.SerializedArtifact{
			Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
		},
	}
	if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
		t.Fatal(err)
	}

	_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
	if err != nil {
		t.Fatal(err)
	}
	if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
		t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
	}
	if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` {
		t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision)
	}
	changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
	assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
	_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
	if err != nil {
		t.Fatal(err)
	}
	if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
		t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
	}
	if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
		t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
	}
	changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.changed")
	assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
	_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
	if err != nil {
		t.Fatal(err)
	}
	if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused {
		t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
	}

	changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
	_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
	if err != nil {
		t.Fatal(err)
	}
	if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
		t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
	}
	if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
		t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
	}
}

func normalizeSpellCatalogFingerprintName() string {
	return "normalize:spells:" + spellnormalize.Key + ":effective_catalog"
}

func extractSpellMappingFingerprintName() string {
	return "extract:spells:" + spells.Key + ":mapping_policy"
}

func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint {
	t.Helper()
	changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
	matches := 0
	for index := range changed {
		if changed[index].Name == name {
			changed[index].Value = value
			matches++
		}
	}
	if matches != 1 {
		t.Fatalf("checkpoint fingerprints = %#v, want exactly one fingerprint named %q", fingerprints, name)
	}
	return changed
}

func assertOnlyCheckpointFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) {
	t.Helper()
	if len(before) != len(after) {
		t.Fatalf("fingerprint lengths = %d and %d, want equal", len(before), len(after))
	}
	changes := 0
	for index := range before {
		if before[index].Name != after[index].Name {
			t.Fatalf("fingerprint[%d] name changed from %q to %q", index, before[index].Name, after[index].Name)
		}
		if before[index].Value == after[index].Value {
			continue
		}
		changes++
		if before[index].Name != changedName {
			t.Fatalf("fingerprint %q changed unexpectedly", before[index].Name)
		}
	}
	if changes != 1 {
		t.Fatalf("fingerprints changed %d values, want exactly %q", changes, changedName)
	}
}

func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
	outputRoot := filepath.Join(t.TempDir(), "output")
	fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
	options := productionRunOptions(t, fake)
	var stdout, stderr strings.Builder
	code := RunWithOptions([]string{
		"run", "dnd-session",
		"--config", writeProductionSpellCatalogContractConfig(t),
		"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
		"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
	}, &stdout, &stderr, options)
	if code != 0 {
		t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
	}

	runRoot := filepath.Join(outputRoot, productionRunID)
	manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
	if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
		t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
	}
	lane := manifest.ArtifactLanes[0]
	extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
	if !ok {
		t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
	}
	if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
		t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
	}
	if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
		t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
	}
	normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
	if !ok {
		t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata)
	}
	if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) {
		t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata)
	}
	if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) {
		t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata)
	}

	var catalogProvenances []artifacts.ReferenceProvenance
	for index := range manifest.References {
		reference := &manifest.References[index]
		if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
			catalogProvenances = append(catalogProvenances, *reference)
		}
	}
	if len(catalogProvenances) != 2 {
		t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References)
	}
	overlayBytes := readRepositoryFile(t, "examples", "dnd-spell-catalog.json")
	for _, catalogProvenance := range catalogProvenances {
		if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" {
			t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance)
		}
		if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spell-catalog.json") {
			t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance)
		}
	}
	manifestBytes, err := json.Marshal(manifest)
	if err != nil {
		t.Fatal(err)
	}
	for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
		if strings.Contains(string(manifestBytes), leaked) {
			t.Fatalf("manifest leaked overlay content %q", leaked)
		}
	}

	requests := fake.requestsFor(spells.PromptID)
	if len(requests) != 1 {
		t.Fatalf("spell requests = %d, want one", len(requests))
	}
	catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
	if !ok || !strings.Contains(string(catalogInput.Content), `"canonical_name":"Aegis of Emberfall"`) || !strings.Contains(string(catalogInput.Content), `"aliases":["Emberfall Aegis"]`) {
		t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name and recognition alias", catalogInput)
	}
	artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
	if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
		t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
	}
	rejected := readProductionJSON[struct {
		Rejected []json.RawMessage `json:"rejected"`
	}](t, filepath.Join(runRoot, "rejected.json"))
	if len(rejected.Rejected) != 0 {
		t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
	}
}

func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
	t.Helper()
	identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
		Pipeline:       resolved,
		InputKey:       resolved.Input.Module,
		RawInputDigest: "sha256:catalog-test-input",
		References:     pipeline.ReferenceProvenance(resolved),
	})
	if err != nil {
		t.Fatalf("create checkpoint identity: %v", err)
	}
	return identity
}

func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
	t.Helper()
	lane := resolved.Steps[0].ArtifactLanes[0]
	extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
	if err != nil {
		t.Fatalf("construct extractor: %v", err)
	}
	return extractor.ManifestMetadata()
}

func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
	t.Helper()
	lane := resolved.Steps[0].ArtifactLanes[0]
	normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet)
	if err != nil {
		t.Fatalf("construct normalizer: %v", err)
	}
	return normalizer.ManifestMetadata()
}

func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
	t.Helper()
	for _, reference := range pipeline.ReferenceProvenance(resolved) {
		if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
			return reference
		}
	}
	t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
	return artifacts.ReferenceProvenance{}
}

func stringValue(value any) string {
	result, _ := value.(string)
	return result
}

func stringValues(value any) []string {
	raw, err := json.Marshal(value)
	if err != nil {
		return nil
	}
	var values []string
	if err := json.Unmarshal(raw, &values); err != nil {
		return nil
	}
	return values
}

func digestBytes(value []byte) string {
	sum := sha256.Sum256(value)
	return "sha256:" + hex.EncodeToString(sum[:])
}

const reorderedOverlayA = `{
  "schema_version": "notarius.dnd.spell-catalog-overlay.v1",
  "catalogs": [
    {"id":"campaign.a","ruleset":"dnd-5e-2014","source":{"title":"Campaign A"},"spells":[{"name":"Aegis of Emberfall","aliases":["Emberfall Aegis"]}]},
    {"id":"campaign.b","ruleset":"dnd-5e-2014","source":{"title":"Campaign B"},"spells":[{"name":"Cinder Veil","aliases":["Veil of Cinder","Cinder Shroud"]}]}
  ]
}`

const reorderedOverlayB = `{"catalogs":[{"spells":[{"aliases":["Cinder Shroud","Veil of Cinder"],"name":"Cinder Veil"}],"source":{"title":"Campaign B"},"ruleset":"dnd-5e-2014","id":"campaign.b"},{"spells":[{"aliases":["Emberfall Aegis"],"name":"Aegis of Emberfall"}],"source":{"title":"Campaign A"},"ruleset":"dnd-5e-2014","id":"campaign.a"}],"schema_version":"notarius.dnd.spell-catalog-overlay.v1"}`