notarius/internal/modules/dnd/normalize/itemregistry/normalizer_test.go

package itemregistry

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
	identityvalidator "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/itemregistry/identity"
	"gitea.maximumdirect.net/eric/promptkit"
)

func TestModuleContractAndMetadata(t *testing.T) {
	want := pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: dnd.ItemRegistryKind}
	if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
		t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
	}
	registry := pipeline.NewNormalizerRegistry()
	if err := Register(registry); err != nil {
		t.Fatal(err)
	}
	if _, err := New(nil, Options{}); err == nil {
		t.Fatal("New() accepted nil client")
	}
	metadata := newNormalizer(t, &recordingNormalizerClient{}).ManifestMetadata()
	limits, ok := metadata["semantic_reconciliation_limits"].(map[string]any)
	if !ok || metadata["identity_policy"] != identity.Policy || metadata["normalization_policy"] != normalizationPolicy || metadata["prompt_id"] != PromptID || metadata["prompt_version"] != PromptVersion || metadata["response_schema_key"] != string(semanticreconcile.ResponseSchemaKey) || metadata["response_schema_id"] != semanticreconcile.ResponseSchemaID || metadata["response_schema_name"] != semanticreconcile.ResponseSchemaName || metadata["semantic_reconciliation_policy"] != semanticreconcile.Policy || len(limits) != 3 {
		t.Fatalf("metadata = %#v", metadata)
	}
	for _, name := range []string{"prompt", "response_schema", "semantic_reconciliation_policy", "semantic_reconciliation_limits", "identity_policy", "normalization_policy"} {
		if !hasFingerprint(newNormalizer(t, &recordingNormalizerClient{}).CheckpointFingerprints(), name) {
			t.Fatalf("fingerprints missing %q", name)
		}
	}
}

func TestNormalizeRejectsNilSourceDocument(t *testing.T) {
	_, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(
		context.Background(),
		contracts.TypedNormalizeRequest[dnd.ItemRegistry]{},
	)
	if err == nil || !strings.Contains(err.Error(), "source document must not be nil") {
		t.Fatalf("Normalize() error = %v, want nil source rejection", err)
	}
}

func TestNormalizeConsolidatesEqualNamesAcrossEvidenceWithoutMutation(t *testing.T) {
	doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{
		{ID: 1, Text: "The rope is secured."},
		{ID: 2, Text: "The party takes the rope."},
		{ID: 3, Text: "The rope is packed away."},
		{ID: 4, Text: "A lantern lights the path."},
	}}
	input := dnd.ItemRegistry{Items: []dnd.Item{
		{Name: "  Rope  ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
		{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
		{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
	}}
	before := dnd.ItemRegistry{Items: []dnd.Item{
		{Name: "  Rope  ", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 3, EndUnitID: 3}, {SourceID: "session", StartUnitID: 1, EndUnitID: 1}}},
		{Name: "rope", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}},
		{Name: "Lantern", SourceRefs: []source.SourceRef{{SourceID: "session", StartUnitID: 4, EndUnitID: 4}}},
	}}
	client := &recordingNormalizerClient{}
	result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	if err != nil || len(result.Value.Items) != 2 || !reflect.DeepEqual(input, before) || len(client.requests) != 1 {
		t.Fatalf("Normalize() = %#v, %v; want deterministic non-mutating consolidation", result, err)
	}
	rope := result.Value.Items[0]
	wantRefs := []source.SourceRef{{SourceID: "session", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session", StartUnitID: 2, EndUnitID: 2}, {SourceID: "session", StartUnitID: 3, EndUnitID: 3}}
	if rope.Name != "Rope" || rope.ID != identity.DeriveID("Rope") || !reflect.DeepEqual(rope.SourceRefs, wantRefs) || result.Value.Items[1].Name != "Lantern" || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
		t.Fatalf("items = %#v, diagnostics = %#v; want earliest display name, canonical evidence union, and stable placement", result.Value.Items, result.Diagnostics)
	}
	validation, validationErr := identityvalidator.New(identityvalidator.Options{}).Validate(context.Background(), contracts.TypedValidationRequest[dnd.ItemRegistry]{Value: result.Value})
	if validationErr != nil || !validation.Approved {
		t.Fatalf("default item-registry identity validation = %#v, %v; want normalized cross-chunk candidates accepted", validation, validationErr)
	}
	var candidates struct {
		Candidates []struct {
			Label string `json:"label"`
		} `json:"candidates"`
	}
	if err := json.Unmarshal(client.requests[0].Inputs["candidates"].Content, &candidates); err != nil || len(candidates.Candidates) != 2 || candidates.Candidates[0].Label != "Rope" || candidates.Candidates[1].Label != "Lantern" {
		t.Fatalf("semantic candidates = %#v, %v; want one candidate per comparison name", candidates, err)
	}
	repeated, repeatErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(result.Value, doc))
	if repeatErr != nil || !reflect.DeepEqual(repeated.Value, result.Value) {
		t.Fatalf("repeated normalization = %#v, %v; want stable output", repeated, repeatErr)
	}
}

func BenchmarkComparisonNameGroupsManyDistinct(b *testing.B) {
	records := make([]normalizedRecord, 1_000)
	for index := range records {
		records[index] = normalizedRecord{item: dnd.Item{Name: "Item " + strconv.Itoa(index)}}
	}
	b.ReportAllocs()
	b.ResetTimer()
	for iteration := 0; iteration < b.N; iteration++ {
		if groups := comparisonNameGroups(records); len(groups) != len(records) {
			b.Fatalf("group count = %d, want %d", len(groups), len(records))
		}
	}
}

func TestNormalizeAppliesSafeAliasProposal(t *testing.T) {
	doc := semanticDocument()
	input := dnd.ItemRegistry{Items: []dnd.Item{
		{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
		{Name: "Compass of the Stars", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
		{Name: "Gold Pieces", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
	}}
	client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`}
	result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	if err != nil || result.Retry != nil || len(result.Value.Items) != 2 {
		t.Fatalf("Normalize() = %#v, %v", result, err)
	}
	merged := result.Value.Items[0]
	if merged.Name != "Compass of the Stars" || merged.ID != identity.DeriveID(merged.Name) || len(merged.SourceRefs) != 2 || !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) {
		t.Fatalf("merged item = %#v, diagnostics = %#v", merged, result.Diagnostics)
	}
	encoded := string(client.requests[0].Inputs["candidates"].Content) + string(client.requests[0].Inputs["transcript"].Content)
	if strings.Contains(encoded, doc.ID) || strings.Contains(encoded, "candidate-") || strings.Contains(encoded, merged.ID) || !strings.Contains(encoded, `"source_refs"`) {
		t.Fatalf("private inputs = %s", encoded)
	}
}

func TestNormalizeAppliesCurrencyReconciliationSafely(t *testing.T) {
	doc := semanticDocument()
	ref := func(unitID int) []source.SourceRef {
		return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}}
	}
	for _, test := range []struct {
		name          string
		items         []dnd.Item
		response      string
		wantNames     []string
		wantRefCounts []int
		wantRetry     bool
		reasonCode    string
		disposition   contracts.DiagnosticDisposition
	}{
		{
			name: "same denomination aliases",
			items: []dnd.Item{
				{Name: "GP", SourceRefs: ref(10)},
				{Name: "Gold Piece", SourceRefs: ref(20)},
				{Name: "Gold Pieces", SourceRefs: ref(30)},
			},
			response:      `{"duplicate_groups":[{"candidate_numbers":[1,2,3],"canonical_candidate_number":2}]}`,
			wantNames:     []string{"Gold Piece"},
			wantRefCounts: []int{3},
			reasonCode:    ReasonCodeDuplicateItemCollapsed,
			disposition:   contracts.DiagnosticDispositionObservation,
		},
		{
			name: "different denominations",
			items: []dnd.Item{
				{Name: "Gold Pieces", SourceRefs: ref(10)},
				{Name: "Silver Pieces", SourceRefs: ref(20)},
			},
			response:      `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
			wantNames:     []string{"Gold Pieces", "Silver Pieces"},
			wantRefCounts: []int{1, 1},
			wantRetry:     true,
			reasonCode:    ReasonCodeItemSemanticProposalInvalid,
			disposition:   contracts.DiagnosticDispositionAdvisory,
		},
		{
			name: "currency plus ordinary item",
			items: []dnd.Item{
				{Name: "Gold Pieces", SourceRefs: ref(10)},
				{Name: "Longsword", SourceRefs: ref(20)},
			},
			response:      `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1}]}`,
			wantNames:     []string{"Gold Pieces", "Longsword"},
			wantRefCounts: []int{1, 1},
			wantRetry:     true,
			reasonCode:    ReasonCodeItemSemanticProposalInvalid,
			disposition:   contracts.DiagnosticDispositionAdvisory,
		},
		{
			name: "ordinary items",
			items: []dnd.Item{
				{Name: "Star Compass", SourceRefs: ref(10)},
				{Name: "Compass of the Stars", SourceRefs: ref(20)},
			},
			response:      `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`,
			wantNames:     []string{"Compass of the Stars"},
			wantRefCounts: []int{2},
			reasonCode:    ReasonCodeDuplicateItemCollapsed,
			disposition:   contracts.DiagnosticDispositionObservation,
		},
		{
			name: "currency plus ordinary canonical item",
			items: []dnd.Item{
				{Name: "Gold Pieces", SourceRefs: ref(10)},
				{Name: "Longsword", SourceRefs: ref(20)},
			},
			response:      `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2}]}`,
			wantNames:     []string{"Gold Pieces", "Longsword"},
			wantRefCounts: []int{1, 1},
			wantRetry:     true,
			reasonCode:    ReasonCodeItemSemanticProposalInvalid,
			disposition:   contracts.DiagnosticDispositionAdvisory,
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			input := dnd.ItemRegistry{Items: make([]dnd.Item, len(test.items))}
			for index, item := range test.items {
				input.Items[index] = cloneItem(item)
			}
			before := dnd.ItemRegistry{Items: make([]dnd.Item, len(input.Items))}
			for index, item := range input.Items {
				before.Items[index] = cloneItem(item)
			}

			result, err := newNormalizer(t, &recordingNormalizerClient{response: test.response}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
			if err != nil || (result.Retry != nil) != test.wantRetry || !reflect.DeepEqual(input, before) || !hasDiagnostic(result.Diagnostics, test.reasonCode, test.disposition) {
				t.Fatalf("Normalize() = %#v, %v", result, err)
			}
			if len(result.Value.Items) != len(test.wantNames) {
				t.Fatalf("items = %#v, want names %#v", result.Value.Items, test.wantNames)
			}
			for index, item := range result.Value.Items {
				if item.Name != test.wantNames[index] || len(item.SourceRefs) != test.wantRefCounts[index] {
					t.Fatalf("item %d = %#v, want name %q with %d source refs", index, item, test.wantNames[index], test.wantRefCounts[index])
				}
			}
			if test.wantRetry && (len(result.Retry.FallbackDiagnostics) != 1 || result.Retry.FallbackDiagnostics[0].ReasonCode != ReasonCodeItemSemanticReconciliationExhausted || result.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid) {
				t.Fatalf("retry = %#v, want preserved-group fallback", result.Retry)
			}
		})
	}
}

func TestNormalizePreservesCandidatesForUnsafeProposalGroups(t *testing.T) {
	doc := semanticDocument()
	input := dnd.ItemRegistry{Items: []dnd.Item{
		{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}},
		{Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}},
		{Name: "Rope", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 30, EndUnitID: 30}}},
	}}
	client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":1},{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,3],"canonical_candidate_number":99}]}`}
	result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	if err != nil || result.Retry == nil || len(result.Value.Items) != 3 || !strings.Contains(result.Retry.Message, "overlapping_member") || !strings.Contains(result.Retry.Message, "canonical_unknown") || strings.Contains(result.Retry.Message, "Star Compass") || len(result.Retry.Message) > 4096 {
		t.Fatalf("Normalize() = %#v, %v; want deterministic retry fallback", result, err)
	}
}

func TestNormalizeAppliesIndependentGroupAndCountsAllOmissions(t *testing.T) {
	doc := semanticDocument()
	ref := func(unitID int) []source.SourceRef {
		return []source.SourceRef{{SourceID: doc.ID, StartUnitID: unitID, EndUnitID: unitID}}
	}
	input := dnd.ItemRegistry{Items: []dnd.Item{
		{Name: "Star Compass", SourceRefs: ref(10)},
		{Name: "Compass of the Stars", SourceRefs: ref(20)},
		{Name: "Gold Pieces", SourceRefs: ref(30)},
		{Name: "Silver Pieces", SourceRefs: ref(30)},
		{Name: "Rope", SourceRefs: ref(10)},
	}}
	client := &recordingNormalizerClient{response: `{"duplicate_groups":[{"candidate_numbers":[1,2],"canonical_candidate_number":2},{"candidate_numbers":[3,4],"canonical_candidate_number":3},{"candidate_numbers":[5,99],"canonical_candidate_number":5}]}`}
	result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	if err != nil || result.Retry == nil {
		t.Fatalf("Normalize() = %#v, %v; want retry with independently accepted output", result, err)
	}
	wantNames := []string{"Compass of the Stars", "Gold Pieces", "Silver Pieces", "Rope"}
	if len(result.Value.Items) != len(wantNames) {
		t.Fatalf("items = %#v, want %v", result.Value.Items, wantNames)
	}
	for index, name := range wantNames {
		if result.Value.Items[index].Name != name {
			t.Fatalf("item %d = %#v, want %q", index, result.Value.Items[index], name)
		}
	}
	if !hasDiagnostic(result.Diagnostics, ReasonCodeDuplicateItemCollapsed, contracts.DiagnosticDispositionObservation) || !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticProposalInvalid, contracts.DiagnosticDispositionAdvisory) {
		t.Fatalf("diagnostics = %#v, want accepted and guarded-group diagnostics", result.Diagnostics)
	}
	if !strings.Contains(result.Retry.CorrectionGuidance, "aliases of the same denomination") || !strings.Contains(result.Retry.CorrectionGuidance, "do not combine currency with non-currency items") || !strings.Contains(result.Retry.CorrectionGuidance, "candidate-number range 1 through 5") || !strings.Contains(result.Retry.CorrectionGuidance, "source positions") || strings.Contains(result.Retry.CorrectionGuidance, "member_unknown") || strings.Contains(result.Retry.CorrectionGuidance, ReasonCodeItemSemanticRetryProposalInvalid) {
		t.Fatalf("correction guidance = %q, want shared and item-specific prose only", result.Retry.CorrectionGuidance)
	}
	if len(result.Retry.FallbackDiagnostics) != 1 || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "2 proposal group(s)") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "preserved separately") || !strings.Contains(result.Retry.FallbackDiagnostics[0].Samples[0].Message, "same denomination") {
		t.Fatalf("retry = %#v, want one guarded and one malformed group counted", result.Retry)
	}
}

func TestNormalizeLimitSkipDoesNotCallLLMAndAddsBoundedFallbackWarning(t *testing.T) {
	client := &recordingNormalizerClient{}
	doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Kind: "narration", Text: "A crowded storeroom"}}}
	limit := semanticreconcile.DefaultLimits().MaximumCandidates
	input := dnd.ItemRegistry{Items: make([]dnd.Item, limit+1)}
	for index := range input.Items {
		input.Items[index] = dnd.Item{Name: fmt.Sprintf("Item %d", index), SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
	}
	result, err := newNormalizer(t, client).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	if err != nil || result.Retry != nil {
		t.Fatalf("Normalize() = %#v, %v; want deterministic limit fallback", result, err)
	}
	if len(client.requests) != 0 || len(result.Value.Items) != limit+1 {
		t.Fatalf("completion calls = %d, items = %d; want no call and all records", len(client.requests), len(result.Value.Items))
	}
	if !hasDiagnostic(result.Diagnostics, ReasonCodeItemSemanticReconciliationExhausted, contracts.DiagnosticDispositionWarning) {
		t.Fatalf("diagnostics = %#v, want reconciliation fallback warning", result.Diagnostics)
	}
}

func TestNormalizeRetryFallbackErrorsAndIdempotence(t *testing.T) {
	doc := semanticDocument()
	input := dnd.ItemRegistry{Items: []dnd.Item{{Name: "Star Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 10, EndUnitID: 10}}}, {Name: "Compass", SourceRefs: []source.SourceRef{{SourceID: doc.ID, StartUnitID: 20, EndUnitID: 20}}}}}
	invalid, err := newNormalizer(t, &recordingNormalizerClient{err: contracts.ErrInvalidStructuredOutput}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	if err != nil || invalid.Retry == nil || invalid.Retry.ReasonCode != ReasonCodeItemSemanticRetryProposalInvalid {
		t.Fatalf("invalid result = %#v, %v", invalid, err)
	}
	if invalid.Retry.CorrectionGuidance != "" || !strings.Contains(invalid.Retry.FallbackDiagnostics[0].Samples[0].Message, "structured-output contract") {
		t.Fatalf("invalid structured retry = %#v, want feedback-free retry with final-cause fallback", invalid.Retry)
	}
	_, err = newNormalizer(t, &recordingNormalizerClient{err: errors.New("provider unavailable")}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	if err == nil || !strings.Contains(err.Error(), "provider unavailable") {
		t.Fatalf("provider error = %v", err)
	}
	first, err := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(input, doc))
	second, secondErr := newNormalizer(t, &recordingNormalizerClient{}).Normalize(context.Background(), normalizeRequestWithSource(first.Value, doc))
	if err != nil || secondErr != nil || !reflect.DeepEqual(first.Value, second.Value) {
		t.Fatalf("normalization must be idempotent: first=%#v err=%v second=%#v err=%v", first, err, second, secondErr)
	}
}

func TestRegisterPromptAssetsPreparesItemNormalizationPrompt(t *testing.T) {
	registry := llm.NewAssetRegistry()
	if err := semanticreconcile.RegisterAssets(registry); err != nil {
		t.Fatal(err)
	}
	if err := RegisterPromptAssets(registry); err != nil {
		t.Fatal(err)
	}
	options, err := registry.PromptKitOptions()
	if err != nil {
		t.Fatal(err)
	}
	options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ID: "item-normalize-test", Endpoint: "http://127.0.0.1:1/v1", Model: "test"})))
	engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...)
	if err != nil {
		t.Fatal(err)
	}
	prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: PromptID, PromptVersion: PromptVersion, ProfileID: "item-normalize-test", Inputs: map[string]promptkit.ArtifactRef{"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":1},"candidates":[{"candidate_number":1,"label":"Rope","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`), "transcript": promptkit.Inline(`{"windows":[{"units":[]}]}`)}})
	if err != nil {
		t.Fatal(err)
	}
	if prepared.OutputContract.SchemaPath != "semantic_reconciliation_llm.v1.json" || !strings.Contains(prepared.Messages[1].Content, "candidate_number_range") || !strings.Contains(prepared.Messages[1].Content, "start_unit_id") || !strings.Contains(prepared.Messages[2].Content, "currency denominations") || !strings.Contains(prepared.Messages[2].Content, "materially different item") || !strings.Contains(prepared.Messages[2].Content, "canonical_candidate_number") || prepared.Messages[2].CacheControl == nil || prepared.Messages[2].CacheControl.Type != promptkit.CacheControlEphemeral {
		t.Fatalf("prepared prompt = %#v", prepared)
	}
	if prepared.Messages[4].CacheControl == nil || prepared.Messages[4].CacheControl.Type != promptkit.CacheControlEphemeral || !strings.Contains(prepared.Messages[3].Content, `"Rope"`) || strings.Contains(prepared.Messages[3].Content, `"windows"`) || !strings.Contains(prepared.Messages[4].Content, `"windows"`) || strings.Contains(prepared.Messages[4].Content, `"Rope"`) {
		t.Fatalf("prepared prompt = %#v, want isolated candidate and transcript presentation", prepared)
	}
}

type recordingNormalizerClient struct {
	response string
	err      error
	requests []contracts.StructuredCompletionRequest
}

func (c *recordingNormalizerClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, output any) (contracts.StructuredCompletionResponse, error) {
	snapshot, err := contracts.CloneStructuredCompletionRequest(request)
	if err != nil {
		return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone recording request: %w", err)
	}
	c.requests = append(c.requests, snapshot)
	if c.err != nil {
		return contracts.StructuredCompletionResponse{}, c.err
	}
	response := c.response
	if response == "" {
		response = `{"duplicate_groups":[]}`
	}
	content := []byte(response)
	if err := json.Unmarshal(content, output); err != nil {
		return contracts.StructuredCompletionResponse{}, err
	}
	return contracts.StructuredCompletionResponse{Content: content}, nil
}

func newNormalizer(t *testing.T, client contracts.StructuredLLMClient) *Normalizer {
	t.Helper()
	normalizer, err := New(client, Options{})
	if err != nil {
		t.Fatalf("New() error = %v", err)
	}
	return normalizer
}
func normalizeRequest(value dnd.ItemRegistry) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
	return contracts.TypedNormalizeRequest[dnd.ItemRegistry]{
		Source:      &source.SourceDocument{},
		MergeOutput: contracts.MergeArtifact[dnd.ItemRegistry]{Value: value},
	}
}
func normalizeRequestWithSource(value dnd.ItemRegistry, doc *source.SourceDocument) contracts.TypedNormalizeRequest[dnd.ItemRegistry] {
	request := normalizeRequest(value)
	request.Source = doc
	return request
}
func semanticDocument() *source.SourceDocument {
	return &source.SourceDocument{ID: "item-session", Units: []source.SourceUnit{{ID: 10, Text: "The Star Compass points north."}, {ID: 20, Text: "The compass of the stars glows."}, {ID: 30, Text: "The chest holds gold pieces."}}}
}
func hasDiagnostic(diagnostics []contracts.ProducerDiagnostic, reason string, disposition contracts.DiagnosticDisposition) bool {
	for _, diagnostic := range diagnostics {
		if diagnostic.ReasonCode == reason && diagnostic.Disposition == disposition {
			return true
		}
	}
	return false
}

func hasFingerprint(fingerprints []pipeline.CheckpointFingerprint, name string) bool {
	for _, fingerprint := range fingerprints {
		if fingerprint.Name == name && fingerprint.Value != "" {
			return true
		}
	}
	return false
}