// Package itemregistry normalizes merged D&D item-registry candidates conservatively. package itemregistry import ( "context" "fmt" "reflect" "sort" "strconv" "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics" "go.maximumdirect.net/promptkit" ) const ( Key = "dnd/item-registry" PromptID = "notarius/dnd.item_registry.normalize" CorrectionPromptID = "notarius/dnd.item_registry.normalize.repair" ReasonCodeItemFieldsNormalized = "item_fields_normalized" ReasonCodeItemIDRecomputed = "item_id_recomputed" ReasonCodeSourceReferencesNormalized = "source_references_normalized" ReasonCodeDuplicateItemCollapsed = "duplicate_item_collapsed" ReasonCodeItemSemanticProposalInvalid = "item_semantic_proposal_invalid" ReasonCodeItemSemanticReconciliationExhausted = "item_semantic_reconciliation_exhausted" ) var requiredCapabilities = []string{"merged"} var providedCapabilities = []string{"normalized"} var _ contracts.Normalizer[dnd.ItemRegistry] = (*Normalizer)(nil) type Options struct{} type Normalizer struct { engine *promptkit.Engine } func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) { if engine == nil { return nil, normalizerErrorf("native engine must not be nil") } return &Normalizer{engine: engine}, nil } func (n *Normalizer) Key() string { return Key } func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.ItemRegistry]) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { if n == nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("normalizer must not be nil") } if n.engine == nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("native engine must not be nil") } if ctx == nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context must not be nil") } if err := ctx.Err(); err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("context error before normalize: %w", err) } if req.Source == nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("source document must not be nil") } // Keep the already available merged artifact separate from preprocessed // originals. A failed native response must not recover an earlier proposal. merged := cloneRegistry(req.MergeOutput.Value) order := shared.NewSourceRefOrder(req.Source) records, baseFindings := preprocessRecords(merged, order) deterministic := recordList(records) findings := append([]diagnostics.Finding(nil), baseFindings...) var advisories []diagnostics.Finding discardedGroups := -1 var omissionDetails []string fallback := func(_ context.Context, final *dnd.ItemRegistry) (dnd.ItemRegistry, error) { value := cloneRegistry(merged) if final != nil { // Each interpreted candidate was built afresh from fixed original records, // using only the final proposal's compatible, non-overlapping safe groups. value = cloneRegistry(*final) } else { findings, advisories = nil, nil discardedGroups = -1 omissionDetails = nil } if err := validateFallback(value, req.Source); err != nil { return dnd.ItemRegistry{}, err } return value, nil } finish := func(value dnd.ItemRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("normalize items: %w", err) } result, err := normalizationResult(value, findings, advisories) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err } result.Outcome = outcome if outcome.Disposition == contracts.ProducerFallback { details := append([]string(nil), omissionDetails...) details = append(details, outcome.FallbackReason) groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err) } result.Diagnostics = append(result.Diagnostics, groups...) } return result, nil } noModel := func() (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.ItemRegistry, error) { return fallback(ctx, nil) }) return finish(value, outcome, err) } if len(records) < 2 { return noModel() } candidates, envelopes, err := reconciliationInputs(records) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err) } preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits()) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err) } switch preparation.Disposition() { case semanticreconcile.InsufficientCandidates: return noModel() case semanticreconcile.LimitExceeded: value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.ItemRegistry, error) { return fallback(ctx, nil) }) return finish(value, outcome, err) } initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("build native request: %w", err) } value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial, func(_ context.Context, result *promptkit.RunResult) (*dnd.ItemRegistry, []string, error) { response, err := semanticreconcile.DecodeProposal(result.RawOutput) if err != nil { return nil, nil, err } assessment := preparation.Assess(response) applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order) if err != nil { return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err) } findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...) advisories = advisoryFindings discardedGroups = assessment.DiscardedGroupCount() + rejectedGroups omissionDetails = semanticreconcile.IssueDetails(assessment.Issues()) guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange()) if err != nil { return nil, nil, err } if rejectedGroups > 0 { guidance = append(guidance, "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items.") omissionDetails = append(omissionDetails, "currency may only be consolidated with aliases of one denomination") } candidate := recordList(applied) return &candidate, guidance, nil }, func(_ context.Context, guidance []string) (promptkit.RunRequest, error) { return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil }, fallback) return finish(value, outcome, err) } // validateFallback checks the durable artifact and the identity/source invariants // used by registry consumers, not the configured ordinary validator chain. func validateFallback(value dnd.ItemRegistry, document *source.SourceDocument) error { if _, err := itemcodec.New().Encode(value); err != nil { return normalizerErrorf("construct item fallback: %w", err) } if issues := identity.ValidateList(value); len(issues) > 0 { return normalizerErrorf("construct item fallback: registry identity is not usable downstream") } index := source.NewDocumentIndex(document) for _, item := range value.Items { for _, reference := range item.SourceRefs { if err := index.ValidateRef(reference); err != nil { return normalizerErrorf("construct item fallback: %w", err) } } } return nil } func cloneRegistry(value dnd.ItemRegistry) dnd.ItemRegistry { if value.Items == nil { return dnd.ItemRegistry{} } result := dnd.ItemRegistry{Items: make([]dnd.Item, len(value.Items))} for i, item := range value.Items { result.Items[i] = cloneItem(item) } return result } func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding { message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output" if discarded > 0 { message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded) } if len(details) > 0 { message = diagnostics.Aggregate(message, details) } return diagnostics.Finding{Scope: "items", ReasonCode: ReasonCodeItemSemanticReconciliationExhausted, Message: message} } func normalizationResult(value dnd.ItemRegistry, findings, advisoryFindings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.ItemRegistry], error) { groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err) } advisoryGroups, err := diagnostics.Collect(advisoryFindings, contracts.DiagnosticDispositionAdvisory, contracts.DiagnosticCategoryDataQuality) if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("collect data-quality diagnostics: %w", err) } return contracts.TypedNormalizeResult[dnd.ItemRegistry]{Value: value, Diagnostics: append(groups, advisoryGroups...)}, nil } type normalizedRecord struct { item dnd.Item inputIndexes []int earliest int } func preprocessRecords(input dnd.ItemRegistry, order shared.SourceRefOrder) ([]normalizedRecord, []diagnostics.Finding) { if input.Items == nil { return nil, nil } records := make([]normalizedRecord, len(input.Items)) findings := make([]diagnostics.Finding, 0) for index, inputItem := range input.Items { item, fieldsChanged, refsChanged := normalizeRecord(inputItem, order) records[index] = normalizedRecord{item: item, inputIndexes: []int{index}, earliest: index} if fieldsChanged { findings = append(findings, diagnostics.Finding{Scope: itemScope(index), ReasonCode: ReasonCodeItemFieldsNormalized, Message: fmt.Sprintf("input index %d: item name normalized for %s", index, diagnostics.Quote(inputItem.Name))}) } if refsChanged { findings = append(findings, diagnostics.Finding{Scope: itemScope(index), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputItem.SourceRefs), len(item.SourceRefs))}) } if inputItem.ID != item.ID { findings = append(findings, diagnostics.Finding{Scope: itemScope(index), ReasonCode: ReasonCodeItemIDRecomputed, Message: fmt.Sprintf("input index %d: item ID recomputed from %s", index, diagnostics.Quote(item.Name))}) } } groups := comparisonNameGroups(records) output := make([]normalizedRecord, 0, len(groups)) for _, members := range groups { retained := cloneRecord(records[members[0]]) for _, member := range members[1:] { retained.item.SourceRefs = append(retained.item.SourceRefs, records[member].item.SourceRefs...) retained.inputIndexes = append(retained.inputIndexes, records[member].inputIndexes...) } retained.item.SourceRefs = order.Canonicalize(retained.item.SourceRefs) retained.item.ID = identity.DeriveID(retained.item.Name) retained.inputIndexes = sortedUniqueIndexes(retained.inputIndexes) output = append(output, retained) if len(members) > 1 { findings = append(findings, duplicateFinding(retained.earliest, memberInputIndexes(records, members[1:]))) } } return output, findings } func normalizeRecord(input dnd.Item, order shared.SourceRefOrder) (dnd.Item, bool, bool) { output := cloneItem(input) output.Name = identity.NormalizeDisplay(input.Name) output.SourceRefs = order.Canonicalize(input.SourceRefs) output.ID = identity.DeriveID(output.Name) return output, input.Name != output.Name, !reflect.DeepEqual(input.SourceRefs, output.SourceRefs) } func comparisonNameGroups(records []normalizedRecord) [][]int { groups := make([][]int, 0, len(records)) groupPositions := make(map[string]int, len(records)) for index, record := range records { key := identity.ComparisonKey(record.item.Name) if groupIndex, found := groupPositions[key]; found { groups[groupIndex] = append(groups[groupIndex], index) continue } groupPositions[key] = len(groups) groups = append(groups, []int{index}) } return groups } func cloneItem(input dnd.Item) dnd.Item { input.SourceRefs = cloneSourceRefs(input.SourceRefs) return input } func cloneRecord(input normalizedRecord) normalizedRecord { input.item = cloneItem(input.item) input.inputIndexes = append([]int(nil), input.inputIndexes...) return input } func cloneSourceRefs(input []source.SourceRef) []source.SourceRef { return append([]source.SourceRef(nil), input...) } func sortedUniqueIndexes(indexes []int) []int { if len(indexes) == 0 { return nil } out := append([]int(nil), indexes...) sort.Ints(out) write := 1 for _, index := range out[1:] { if index != out[write-1] { out[write] = index write++ } } return out[:write] } func memberInputIndexes(records []normalizedRecord, members []int) []int { indexes := make([]int, 0, len(members)) for _, member := range members { indexes = append(indexes, records[member].inputIndexes...) } return sortedUniqueIndexes(indexes) } func recordValues(records []normalizedRecord) []dnd.Item { if records == nil { return nil } values := make([]dnd.Item, len(records)) for index, record := range records { values[index] = cloneItem(record.item) } return values } func recordList(records []normalizedRecord) dnd.ItemRegistry { if records == nil { return dnd.ItemRegistry{} } return dnd.ItemRegistry{Items: recordValues(records)} } func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding { const maxDisplayedIndices = 20 displayed := removed if len(displayed) > maxDisplayedIndices { displayed = displayed[:maxDisplayedIndices] } indices := make([]string, len(displayed)) for index, removedIndex := range displayed { indices[index] = strconv.Itoa(removedIndex) } message := fmt.Sprintf("retained input index %d; removed input indices [%s]", retainedIndex, strings.Join(indices, ", ")) if omitted := len(removed) - len(displayed); omitted > 0 { message += fmt.Sprintf("; %d additional removed input indices omitted", omitted) } return diagnostics.Finding{Scope: itemScope(retainedIndex), ReasonCode: ReasonCodeDuplicateItemCollapsed, Message: message} } func itemScope(index int) string { return fmt.Sprintf("items[%d]", index) } func ModuleSpec() pipeline.ModuleSpec { return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.ItemRegistryKind} } func Register(registry *pipeline.NormalizerRegistry) error { return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.ItemRegistry], error) { options, err := DecodeOptions(request.Options) if err != nil { return nil, err } return New(request.Dependencies.Engine, options) }) } func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err } func DecodeOptions(options map[string]any) (Options, error) { if err := pipeline.RejectUnknownOptions(options); err != nil { return Options{}, normalizerErrorf("%w", err) } return Options{}, nil } func normalizerErrorf(format string, args ...any) error { return fmt.Errorf("dnd item registry normalizer: "+format, args...) }