notarius/internal/modules/dnd/normalize/itemregistry/normalizer.go

// Package itemregistry normalizes merged D&D item-registry candidates conservatively.
package itemregistry

import (
	"context"
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
)

const (
	Key                 = "dnd/item-registry"
	PromptID            = "dnd.item_registry.normalize"
	PromptVersion       = "v1"
	normalizationPolicy = "dnd.item_registry.normalize.v4"
	NormalizationPolicy = normalizationPolicy

	ReasonCodeItemFieldsNormalized                = "item_fields_normalized"
	ReasonCodeItemIDRecomputed                    = "item_id_recomputed"
	ReasonCodeSourceReferencesNormalized          = "source_references_normalized"
	ReasonCodeDuplicateItemCollapsed              = "duplicate_item_collapsed"
	ReasonCodeItemSemanticProposalInvalid         = "item_semantic_proposal_invalid"
	ReasonCodeItemSemanticRetryProposalInvalid    = "item_semantic_retry_proposal_invalid"
	ReasonCodeItemSemanticReconciliationExhausted = "item_semantic_reconciliation_exhausted"
)

var requiredCapabilities = []string{"merged"}
var providedCapabilities = []string{"normalized"}

var _ contracts.Normalizer[dnd.ItemRegistry] = (*Normalizer)(nil)
var _ contracts.ManifestMetadataProvider = (*Normalizer)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Normalizer)(nil)

type Options struct{}

type Normalizer struct {
	engine *semanticreconcile.Engine
}

func New(llmClient contracts.StructuredLLMClient, _ Options) (*Normalizer, error) {
	if llmClient == nil {
		return nil, normalizerErrorf("LLM client must not be nil")
	}
	promptSHA, err := promptAssetMetadata()
	if err != nil {
		return nil, normalizerErrorf("load prompt metadata: %w", err)
	}
	engine, err := semanticreconcile.NewEngine(llmClient, semanticreconcile.PromptSpec{
		ID: PromptID, Version: PromptVersion, SHA256: promptSHA,
	}, semanticreconcile.DefaultLimits())
	if err != nil {
		return nil, normalizerErrorf("construct semantic reconciliation engine: %w", err)
	}
	return &Normalizer{engine: engine}, nil
}

func (n *Normalizer) Key() string                               { return Key }
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }

func (n *Normalizer) ManifestMetadata() map[string]any {
	if n == nil || n.engine == nil {
		return nil
	}
	metadata := n.engine.ManifestMetadata()
	metadata["identity_policy"] = identity.Policy
	metadata["normalization_policy"] = normalizationPolicy
	return metadata
}

func (n *Normalizer) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
	if n == nil || n.engine == nil {
		return nil
	}
	fingerprints := n.engine.CheckpointFingerprints()
	return append(fingerprints,
		pipeline.CheckpointFingerprint{Name: "identity_policy", Value: identity.Policy},
		pipeline.CheckpointFingerprint{Name: "normalization_policy", Value: normalizationPolicy},
	)
}

func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.ItemRegistry]) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
	if n == nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("normalizer must not be nil")
	}
	if n.engine == nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("semantic reconciliation engine must not be nil")
	}
	if ctx == nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context must not be nil")
	}
	if err := ctx.Err(); err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context error before normalize: %w", err)
	}
	if req.Source == nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("source document must not be nil")
	}

	order := shared.NewSourceRefOrder(req.Source)
	records, findings := preprocessRecords(req.MergeOutput.Value, order)
	deterministic := recordList(records)
	if len(records) < 2 {
		return normalizationResult(deterministic, findings, nil, nil)
	}

	candidates, envelopes, err := reconciliationInputs(records)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err)
	}
	reconciliation, err := n.engine.Reconcile(ctx, semanticreconcile.Request{
		StageName: Key, Source: req.Source, Candidates: candidates,
		ProfileID: req.LLMProfile, SessionID: req.SessionID, StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts, Correction: req.Correction,
	})
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("reconcile semantic duplicates: %w", err)
	}

	switch reconciliation.Disposition() {
	case semanticreconcile.SkippedInsufficientCandidates:
		return normalizationResult(deterministic, findings, nil, nil)
	case semanticreconcile.SkippedLimitExceeded:
		return fallbackResult(deterministic, findings, nil, semanticFallbackFinding(-1, "the candidate set exceeded the configured semantic-reconciliation limits"))
	case semanticreconcile.RetryableInvalidStructuredOutput:
		return n.invalidStructuredResult(deterministic, findings)
	case semanticreconcile.Complete, semanticreconcile.RetryableDiscardedProposalGroups:
	default:
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("unknown semantic reconciliation disposition %d", reconciliation.Disposition())
	}

	applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(reconciliation.Plan(), records, envelopes, order)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("apply semantic reconciliation plan: %w", err)
	}
	findings = append(findings, semanticFindings...)
	discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups
	if discardedGroups == 0 {
		return normalizationResult(recordList(applied), findings, advisoryFindings, reconciliation.ModelCandidate())
	}
	return retryResult(recordList(applied), findings, advisoryFindings, reconciliation, rejectedGroups)
}

func (n *Normalizer) invalidStructuredResult(value dnd.ItemRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
	return retryResultWithFallback(value, findings, nil, nil, ReasonCodeItemSemanticRetryProposalInvalid, "", "semantic proposal requires retry: invalid structured output", semanticFallbackFinding(-1, "the final model response did not satisfy the structured-output contract"))
}

func retryResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, reconciliation semanticreconcile.Result, rejectedGroups int) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
	issues := reconciliation.Issues()
	operatorDetails := semanticreconcile.IssueDetails(issues)
	correctionDetails, err := reconciliation.CorrectionDetails()
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction details: %w", err)
	}
	var additionalGuidance []string
	if rejectedGroups > 0 {
		const currencyGuidance = "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items."
		operatorDetails = append(operatorDetails, "currency may only be consolidated with aliases of one denomination")
		correctionDetails = append(correctionDetails, currencyGuidance)
		additionalGuidance = append(additionalGuidance, currencyGuidance)
	}
	correctionGuidance, err := reconciliation.CorrectionGuidance(additionalGuidance...)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build semantic correction guidance: %w", err)
	}
	discardedGroups := reconciliation.DiscardedGroupCount() + rejectedGroups
	return retryResultWithFallback(value, findings, advisoryFindings, reconciliation.ModelCandidate(), ReasonCodeItemSemanticRetryProposalInvalid, correctionGuidance, diagnostics.Aggregate("semantic proposal requires retry", operatorDetails), semanticFallbackFinding(discardedGroups, correctionDetails...))
}

func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding {
	message := "semantic reconciliation could not be completed; all affected candidates were preserved separately"
	if discarded >= 0 {
		message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded)
	}
	if len(details) > 0 {
		message = diagnostics.Aggregate(message, details)
	}
	return diagnostics.Finding{Scope: "items", ReasonCode: ReasonCodeItemSemanticReconciliationExhausted, Message: message}
}

func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
	diagnosticGroups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err)
	}
	advisoryGroups, err := diagnostics.Collect(advisoryFindings, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect data-quality diagnostics: %w", err)
	}
	diagnosticGroups = append(diagnosticGroups, advisoryGroups...)
	return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Diagnostics: diagnosticGroups, ModelCandidate: candidate}, nil
}

func fallbackResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
	result, err := normalizationResult(value, findings, advisoryFindings, nil)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
	}
	fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
	}
	result.Diagnostics = append(result.Diagnostics, fallbackGroups...)
	return result, nil
}

func retryResultWithFallback(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding, candidate *contracts.ModelCandidate, reasonCode, correctionGuidance, message string, fallback diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) {
	result, err := normalizationResult(value, findings, advisoryFindings, candidate)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
	}
	fallbackGroups, err := diagnostics.Collect([]diagnostics.Finding{fallback}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
	if err != nil {
		return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
	}
	result.Retry = &contracts.NormalizeRetry{ReasonCode: reasonCode, Message: message, CorrectionGuidance: correctionGuidance, FallbackDiagnostics: fallbackGroups}
	return result, nil
}

type normalizedRecord struct {
	item         dnd.Item
	inputIndexes []int
	earliest     int
}

func preprocessRecords(input dnd.ItemRegistry, order shared.SourceRefOrder) ([]normalizedRecord, []diagnostics.Finding) {
	if input.Items == nil {
		return nil, nil
	}
	records := make([]normalizedRecord, len(input.Items))
	findings := make([]diagnostics.Finding, 0)
	for index, inputItem := range input.Items {
		item, fieldsChanged, refsChanged := normalizeRecord(inputItem, order)
		records[index] = normalizedRecord{item: item, inputIndexes: []int{index}, earliest: index}
		if fieldsChanged {
			findings = append(findings, diagnostics.Finding{Scope: itemScope(index), ReasonCode: ReasonCodeItemFieldsNormalized, Message: fmt.Sprintf("input index %d: item name normalized for %s", index, diagnostics.Quote(inputItem.Name))})
		}
		if refsChanged {
			findings = append(findings, diagnostics.Finding{Scope: itemScope(index), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputItem.SourceRefs), len(item.SourceRefs))})
		}
		if inputItem.ID != item.ID {
			findings = append(findings, diagnostics.Finding{Scope: itemScope(index), ReasonCode: ReasonCodeItemIDRecomputed, Message: fmt.Sprintf("input index %d: item ID recomputed from %s", index, diagnostics.Quote(item.Name))})
		}
	}
	groups := comparisonNameGroups(records)
	output := make([]normalizedRecord, 0, len(groups))
	for _, members := range groups {
		retained := cloneRecord(records[members[0]])
		for _, member := range members[1:] {
			retained.item.SourceRefs = append(retained.item.SourceRefs, records[member].item.SourceRefs...)
			retained.inputIndexes = append(retained.inputIndexes, records[member].inputIndexes...)
		}
		retained.item.SourceRefs = order.Canonicalize(retained.item.SourceRefs)
		retained.item.ID = identity.DeriveID(retained.item.Name)
		retained.inputIndexes = sortedUniqueIndexes(retained.inputIndexes)
		output = append(output, retained)
		if len(members) > 1 {
			findings = append(findings, duplicateFinding(retained.earliest, memberInputIndexes(records, members[1:])))
		}
	}
	return output, findings
}

func normalizeRecord(input dnd.Item, order shared.SourceRefOrder) (dnd.Item, bool, bool) {
	output := cloneItem(input)
	output.Name = identity.NormalizeDisplay(input.Name)
	output.SourceRefs = order.Canonicalize(input.SourceRefs)
	output.ID = identity.DeriveID(output.Name)
	return output, input.Name != output.Name, !reflect.DeepEqual(input.SourceRefs, output.SourceRefs)
}

func comparisonNameGroups(records []normalizedRecord) [][]int {
	groups := make([][]int, 0, len(records))
	groupPositions := make(map[string]int, len(records))
	for index, record := range records {
		key := identity.ComparisonKey(record.item.Name)
		if groupIndex, found := groupPositions[key]; found {
			groups[groupIndex] = append(groups[groupIndex], index)
			continue
		}
		groupPositions[key] = len(groups)
		groups = append(groups, []int{index})
	}
	return groups
}

func cloneItem(input dnd.Item) dnd.Item {
	input.SourceRefs = cloneSourceRefs(input.SourceRefs)
	return input
}
func cloneRecord(input normalizedRecord) normalizedRecord {
	input.item = cloneItem(input.item)
	input.inputIndexes = append([]int(nil), input.inputIndexes...)
	return input
}
func cloneSourceRefs(input []source.SourceRef) []source.SourceRef {
	return append([]source.SourceRef(nil), input...)
}

func sortedUniqueIndexes(indexes []int) []int {
	if len(indexes) == 0 {
		return nil
	}
	out := append([]int(nil), indexes...)
	sort.Ints(out)
	write := 1
	for _, index := range out[1:] {
		if index != out[write-1] {
			out[write] = index
			write++
		}
	}
	return out[:write]
}
func memberInputIndexes(records []normalizedRecord, members []int) []int {
	indexes := make([]int, 0, len(members))
	for _, member := range members {
		indexes = append(indexes, records[member].inputIndexes...)
	}
	return sortedUniqueIndexes(indexes)
}
func recordValues(records []normalizedRecord) []dnd.Item {
	if records == nil {
		return nil
	}
	values := make([]dnd.Item, len(records))
	for index, record := range records {
		values[index] = cloneItem(record.item)
	}
	return values
}
func recordList(records []normalizedRecord) dnd.ItemRegistry {
	if records == nil {
		return dnd.ItemRegistry{}
	}
	return dnd.ItemRegistry{Items: recordValues(records)}
}

func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding {
	const maxDisplayedIndices = 20
	displayed := removed
	if len(displayed) > maxDisplayedIndices {
		displayed = displayed[:maxDisplayedIndices]
	}
	indices := make([]string, len(displayed))
	for index, removedIndex := range displayed {
		indices[index] = strconv.Itoa(removedIndex)
	}
	message := fmt.Sprintf("retained input index %d; removed input indices [%s]", retainedIndex, strings.Join(indices, ", "))
	if omitted := len(removed) - len(displayed); omitted > 0 {
		message += fmt.Sprintf("; %d additional removed input indices omitted", omitted)
	}
	return diagnostics.Finding{Scope: itemScope(retainedIndex), ReasonCode: ReasonCodeDuplicateItemCollapsed, Message: message}
}
func itemScope(index int) string { return fmt.Sprintf("items[%d]", index) }

func ModuleSpec() pipeline.ModuleSpec {
	return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemRegistryKind}
}
func Register(registry *pipeline.NormalizerRegistry) error {
	return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.ItemRegistry], error) {
		options, err := DecodeOptions(request.Options)
		if err != nil {
			return nil, err
		}
		return New(request.Dependencies.LLM, options)
	})
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func DecodeOptions(options map[string]any) (Options, error) {
	if err := pipeline.RejectUnknownOptions(options); err != nil {
		return Options{}, normalizerErrorf("%w", err)
	}
	return Options{}, nil
}
func normalizerErrorf(format string, args ...any) error {
	return fmt.Errorf("dnd item registry normalizer: "+format, args...)
}