2026-08-05 19:39:06 +00:00
// Package itemregistry normalizes merged D&D item-registry candidates conservatively.
package itemregistry
import (
"context"
"fmt"
"reflect"
"sort"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
2026-08-09 16:46:43 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
2026-08-05 19:39:06 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
2026-11-22 00:17:58 +00:00
itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
2026-08-05 19:39:06 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
2026-11-22 00:17:58 +00:00
"go.maximumdirect.net/promptkit"
2026-08-05 19:39:06 +00:00
)
const (
2026-11-22 00:17:58 +00:00
Key = "dnd/item-registry"
PromptID = "notarius/dnd.item_registry.normalize"
CorrectionPromptID = "notarius/dnd.item_registry.normalize.repair"
2026-08-05 19:39:06 +00:00
ReasonCodeItemFieldsNormalized = "item_fields_normalized"
ReasonCodeItemIDRecomputed = "item_id_recomputed"
ReasonCodeSourceReferencesNormalized = "source_references_normalized"
ReasonCodeDuplicateItemCollapsed = "duplicate_item_collapsed"
ReasonCodeItemSemanticProposalInvalid = "item_semantic_proposal_invalid"
ReasonCodeItemSemanticReconciliationExhausted = "item_semantic_reconciliation_exhausted"
)
var requiredCapabilities = [ ] string { "merged" }
var providedCapabilities = [ ] string { "normalized" }
var _ contracts . Normalizer [ dnd . ItemRegistry ] = ( * Normalizer ) ( nil )
type Options struct { }
type Normalizer struct {
2026-11-22 00:17:58 +00:00
engine * promptkit . Engine
2026-08-05 19:39:06 +00:00
}
2026-11-22 00:17:58 +00:00
func New ( engine * promptkit . Engine , _ Options ) ( * Normalizer , error ) {
if engine == nil {
return nil , normalizerErrorf ( "native engine must not be nil" )
2026-08-05 19:39:06 +00:00
}
2026-08-09 16:46:43 +00:00
return & Normalizer { engine : engine } , nil
2026-08-05 19:39:06 +00:00
}
func ( n * Normalizer ) Key ( ) string { return Key }
func ( n * Normalizer ) ReferenceSlots ( ) [ ] contracts . ReferenceSlot { return nil }
func ( n * Normalizer ) Normalize ( ctx context . Context , req contracts . TypedNormalizeRequest [ dnd . ItemRegistry ] ) ( contracts . TypedNormalizeResult [ dnd . ItemRegistry ] , error ) {
if n == nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "normalizer must not be nil" )
}
2026-08-09 16:46:43 +00:00
if n . engine == nil {
2026-11-22 00:17:58 +00:00
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "native engine must not be nil" )
2026-08-05 19:39:06 +00:00
}
if ctx == nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "context must not be nil" )
}
if err := ctx . Err ( ) ; err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "context error before normalize: %w" , err )
}
2026-08-09 18:30:32 +00:00
if req . Source == nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "source document must not be nil" )
}
2026-08-05 19:39:06 +00:00
2026-11-22 00:17:58 +00:00
// Keep the already available merged artifact separate from preprocessed
// originals. A failed native response must not recover an earlier proposal.
merged := cloneRegistry ( req . MergeOutput . Value )
2026-08-05 19:39:06 +00:00
order := shared . NewSourceRefOrder ( req . Source )
2026-11-22 00:17:58 +00:00
records , baseFindings := preprocessRecords ( merged , order )
2026-08-05 19:39:06 +00:00
deterministic := recordList ( records )
2026-11-22 03:19:43 +00:00
if err := req . Operation . Execution . RecordEvidence ( "normalization/originals.json" , deterministic ) ; err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , err
}
2026-11-22 00:17:58 +00:00
findings := append ( [ ] diagnostics . Finding ( nil ) , baseFindings ... )
var advisories [ ] diagnostics . Finding
discardedGroups := - 1
var omissionDetails [ ] string
fallback := func ( _ context . Context , final * dnd . ItemRegistry ) ( dnd . ItemRegistry , error ) {
value := cloneRegistry ( merged )
if final != nil {
// Each interpreted candidate was built afresh from fixed original records,
// using only the final proposal's compatible, non-overlapping safe groups.
value = cloneRegistry ( * final )
} else {
findings , advisories = nil , nil
discardedGroups = - 1
omissionDetails = nil
}
if err := validateFallback ( value , req . Source ) ; err != nil {
return dnd . ItemRegistry { } , err
}
return value , nil
}
finish := func ( value dnd . ItemRegistry , outcome contracts . ProducerOutcome , err error ) ( contracts . TypedNormalizeResult [ dnd . ItemRegistry ] , error ) {
if err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "normalize items: %w" , err )
}
result , err := normalizationResult ( value , findings , advisories )
if err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , err
}
result . Outcome = outcome
if outcome . Disposition == contracts . ProducerFallback {
details := append ( [ ] string ( nil ) , omissionDetails ... )
details = append ( details , outcome . FallbackReason )
groups , err := diagnostics . Collect ( [ ] diagnostics . Finding { semanticFallbackFinding ( discardedGroups , details ... ) } , contracts . DiagnosticDispositionWarning , contracts . DiagnosticCategoryFallback )
if err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "collect fallback diagnostic: %w" , err )
}
result . Diagnostics = append ( result . Diagnostics , groups ... )
}
return result , nil
}
noModel := func ( ) ( contracts . TypedNormalizeResult [ dnd . ItemRegistry ] , error ) {
value , outcome , err := pipeline . AssessProducer ( ctx , req . Operation , deterministic , func ( ctx context . Context ) ( dnd . ItemRegistry , error ) { return fallback ( ctx , nil ) } )
return finish ( value , outcome , err )
}
2026-08-09 18:30:32 +00:00
if len ( records ) < 2 {
2026-11-22 00:17:58 +00:00
return noModel ( )
2026-08-09 16:46:43 +00:00
}
candidates , envelopes , err := reconciliationInputs ( records )
if err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "prepare semantic reconciliation inputs: %w" , err )
}
2026-11-22 00:17:58 +00:00
preparation , err := semanticreconcile . Prepare ( req . Source , candidates , semanticreconcile . DefaultLimits ( ) )
2026-08-05 19:39:06 +00:00
if err != nil {
2026-11-22 00:17:58 +00:00
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "prepare semantic reconciliation: %w" , err )
2026-08-05 19:39:06 +00:00
}
2026-11-22 03:19:43 +00:00
if err := req . Operation . Execution . RecordEvidence ( "normalization/preparation.json" , map [ string ] any {
"disposition" : preparation . Disposition ( ) , "candidate_number_range" : preparation . CandidateNumberRange ( ) ,
"candidate_mappings" : preparation . CandidateMappings ( ) , "candidates" : candidates ,
} ) ; err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , err
}
2026-11-22 00:17:58 +00:00
switch preparation . Disposition ( ) {
case semanticreconcile . InsufficientCandidates :
return noModel ( )
case semanticreconcile . LimitExceeded :
value , outcome , err := pipeline . FinishProducerFailure ( ctx , req . Operation . Controls . Mode , normalizerErrorf ( "candidate set exceeded semantic reconciliation preparation limits" ) , func ( ctx context . Context ) ( dnd . ItemRegistry , error ) { return fallback ( ctx , nil ) } )
return finish ( value , outcome , err )
2026-08-05 19:39:06 +00:00
}
2026-11-22 00:17:58 +00:00
initial , err := semanticreconcile . InitialRequest ( preparation , PromptID , req . LLMProfile , req . SessionID )
2026-08-09 16:46:43 +00:00
if err != nil {
2026-11-22 00:17:58 +00:00
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "build native request: %w" , err )
}
value , outcome , err := pipeline . RunProducer ( ctx , n . engine , req . Operation , initial ,
func ( _ context . Context , result * promptkit . RunResult ) ( * dnd . ItemRegistry , [ ] string , error ) {
response , err := semanticreconcile . DecodeProposal ( result . RawOutput )
if err != nil {
return nil , nil , err
}
assessment := preparation . Assess ( response )
2026-11-22 03:19:43 +00:00
if err := req . Operation . Execution . RecordEvidence ( "normalization/proposals/" + result . RunID + ".json" , map [ string ] any { "proposal" : response , "issues" : assessment . Issues ( ) , "discarded_group_count" : assessment . DiscardedGroupCount ( ) } ) ; err != nil {
return nil , nil , err
}
2026-11-22 00:17:58 +00:00
applied , semanticFindings , advisoryFindings , rejectedGroups , err := applyReconciliationPlan ( assessment . Plan ( ) , records , envelopes , order )
if err != nil {
return nil , nil , normalizerErrorf ( "apply semantic reconciliation plan: %w" , err )
}
findings = append ( append ( [ ] diagnostics . Finding ( nil ) , baseFindings ... ) , semanticFindings ... )
advisories = advisoryFindings
discardedGroups = assessment . DiscardedGroupCount ( ) + rejectedGroups
omissionDetails = semanticreconcile . IssueDetails ( assessment . Issues ( ) )
guidance , err := semanticreconcile . CorrectionDetails ( assessment . Issues ( ) , preparation . CandidateNumberRange ( ) )
if err != nil {
return nil , nil , err
}
if rejectedGroups > 0 {
guidance = append ( guidance , "Consolidate currency only with aliases of the same denomination; keep different denominations separate and do not combine currency with non-currency items." )
omissionDetails = append ( omissionDetails , "currency may only be consolidated with aliases of one denomination" )
}
candidate := recordList ( applied )
return & candidate , guidance , nil
} ,
func ( _ context . Context , guidance [ ] string ) ( promptkit . RunRequest , error ) {
return semanticreconcile . CorrectionRequest ( CorrectionPromptID , req . LLMProfile , req . SessionID , guidance ) , nil
} , fallback )
return finish ( value , outcome , err )
2026-08-05 19:39:06 +00:00
}
2026-11-22 00:17:58 +00:00
// validateFallback checks the durable artifact and the identity/source invariants
// used by registry consumers, not the configured ordinary validator chain.
func validateFallback ( value dnd . ItemRegistry , document * source . SourceDocument ) error {
if _ , err := itemcodec . New ( ) . Encode ( value ) ; err != nil {
return normalizerErrorf ( "construct item fallback: %w" , err )
}
if issues := identity . ValidateList ( value ) ; len ( issues ) > 0 {
return normalizerErrorf ( "construct item fallback: registry identity is not usable downstream" )
}
index := source . NewDocumentIndex ( document )
for _ , item := range value . Items {
for _ , reference := range item . SourceRefs {
if err := index . ValidateRef ( reference ) ; err != nil {
return normalizerErrorf ( "construct item fallback: %w" , err )
}
}
}
return nil
2026-08-05 19:39:06 +00:00
}
2026-11-22 00:17:58 +00:00
func cloneRegistry ( value dnd . ItemRegistry ) dnd . ItemRegistry {
if value . Items == nil {
return dnd . ItemRegistry { }
2026-08-28 18:25:00 +00:00
}
2026-11-22 00:17:58 +00:00
result := dnd . ItemRegistry { Items : make ( [ ] dnd . Item , len ( value . Items ) ) }
for i , item := range value . Items {
result . Items [ i ] = cloneItem ( item )
2026-08-09 16:46:43 +00:00
}
2026-11-22 00:17:58 +00:00
return result
2026-08-05 19:39:06 +00:00
}
2026-08-28 18:25:00 +00:00
func semanticFallbackFinding ( discarded int , details ... string ) diagnostics . Finding {
2026-11-22 00:17:58 +00:00
message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output"
if discarded > 0 {
2026-08-28 18:25:00 +00:00
message = fmt . Sprintf ( "%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately" , discarded )
}
if len ( details ) > 0 {
message = diagnostics . Aggregate ( message , details )
2026-08-05 19:39:06 +00:00
}
2026-08-27 16:40:17 +00:00
return diagnostics . Finding { Scope : "items" , ReasonCode : ReasonCodeItemSemanticReconciliationExhausted , Message : message }
2026-08-05 19:39:06 +00:00
}
2026-11-22 00:17:58 +00:00
func normalizationResult ( value dnd . ItemRegistry , findings , advisoryFindings [ ] diagnostics . Finding ) ( contracts . TypedNormalizeResult [ dnd . ItemRegistry ] , error ) {
groups , err := diagnostics . Collect ( findings , contracts . DiagnosticDispositionObservation , contracts . DiagnosticCategoryNormalization )
2026-08-27 15:57:29 +00:00
if err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "collect normalization diagnostics: %w" , err )
}
advisoryGroups , err := diagnostics . Collect ( advisoryFindings , contracts . DiagnosticDispositionAdvisory , contracts . DiagnosticCategoryDataQuality )
if err != nil {
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { } , normalizerErrorf ( "collect data-quality diagnostics: %w" , err )
}
2026-11-22 00:17:58 +00:00
return contracts . TypedNormalizeResult [ dnd . ItemRegistry ] { Value : value , Diagnostics : append ( groups , advisoryGroups ... ) } , nil
2026-08-09 16:46:43 +00:00
}
2026-08-05 19:39:06 +00:00
type normalizedRecord struct {
item dnd . Item
inputIndexes [ ] int
earliest int
}
2026-08-27 16:40:17 +00:00
func preprocessRecords ( input dnd . ItemRegistry , order shared . SourceRefOrder ) ( [ ] normalizedRecord , [ ] diagnostics . Finding ) {
2026-08-05 19:39:06 +00:00
if input . Items == nil {
return nil , nil
}
records := make ( [ ] normalizedRecord , len ( input . Items ) )
2026-08-27 16:40:17 +00:00
findings := make ( [ ] diagnostics . Finding , 0 )
2026-08-05 19:39:06 +00:00
for index , inputItem := range input . Items {
item , fieldsChanged , refsChanged := normalizeRecord ( inputItem , order )
records [ index ] = normalizedRecord { item : item , inputIndexes : [ ] int { index } , earliest : index }
if fieldsChanged {
2026-08-27 16:40:17 +00:00
findings = append ( findings , diagnostics . Finding { Scope : itemScope ( index ) , ReasonCode : ReasonCodeItemFieldsNormalized , Message : fmt . Sprintf ( "input index %d: item name normalized for %s" , index , diagnostics . Quote ( inputItem . Name ) ) } )
2026-08-05 19:39:06 +00:00
}
if refsChanged {
2026-08-27 16:40:17 +00:00
findings = append ( findings , diagnostics . Finding { Scope : itemScope ( index ) , ReasonCode : ReasonCodeSourceReferencesNormalized , Message : fmt . Sprintf ( "input index %d: source references normalized (original count %d, final count %d)" , index , len ( inputItem . SourceRefs ) , len ( item . SourceRefs ) ) } )
2026-08-05 19:39:06 +00:00
}
if inputItem . ID != item . ID {
2026-08-27 16:40:17 +00:00
findings = append ( findings , diagnostics . Finding { Scope : itemScope ( index ) , ReasonCode : ReasonCodeItemIDRecomputed , Message : fmt . Sprintf ( "input index %d: item ID recomputed from %s" , index , diagnostics . Quote ( item . Name ) ) } )
2026-08-05 19:39:06 +00:00
}
}
2026-08-06 13:28:12 +00:00
groups := comparisonNameGroups ( records )
2026-08-05 19:39:06 +00:00
output := make ( [ ] normalizedRecord , 0 , len ( groups ) )
for _ , members := range groups {
retained := cloneRecord ( records [ members [ 0 ] ] )
for _ , member := range members [ 1 : ] {
2026-08-06 13:28:12 +00:00
retained . item . SourceRefs = append ( retained . item . SourceRefs , records [ member ] . item . SourceRefs ... )
2026-08-05 19:39:06 +00:00
retained . inputIndexes = append ( retained . inputIndexes , records [ member ] . inputIndexes ... )
}
2026-08-06 13:28:12 +00:00
retained . item . SourceRefs = order . Canonicalize ( retained . item . SourceRefs )
retained . item . ID = identity . DeriveID ( retained . item . Name )
2026-08-05 19:39:06 +00:00
retained . inputIndexes = sortedUniqueIndexes ( retained . inputIndexes )
output = append ( output , retained )
if len ( members ) > 1 {
2026-08-27 18:41:59 +00:00
findings = append ( findings , duplicateFinding ( retained . earliest , memberInputIndexes ( records , members [ 1 : ] ) ) )
2026-08-05 19:39:06 +00:00
}
}
2026-08-27 15:57:29 +00:00
return output , findings
2026-08-05 19:39:06 +00:00
}
func normalizeRecord ( input dnd . Item , order shared . SourceRefOrder ) ( dnd . Item , bool , bool ) {
output := cloneItem ( input )
output . Name = identity . NormalizeDisplay ( input . Name )
output . SourceRefs = order . Canonicalize ( input . SourceRefs )
output . ID = identity . DeriveID ( output . Name )
return output , input . Name != output . Name , ! reflect . DeepEqual ( input . SourceRefs , output . SourceRefs )
}
2026-08-06 13:28:12 +00:00
func comparisonNameGroups ( records [ ] normalizedRecord ) [ ] [ ] int {
2026-08-05 19:39:06 +00:00
groups := make ( [ ] [ ] int , 0 , len ( records ) )
2026-08-09 02:26:49 +00:00
groupPositions := make ( map [ string ] int , len ( records ) )
2026-08-05 19:39:06 +00:00
for index , record := range records {
key := identity . ComparisonKey ( record . item . Name )
2026-08-09 02:26:49 +00:00
if groupIndex , found := groupPositions [ key ] ; found {
groups [ groupIndex ] = append ( groups [ groupIndex ] , index )
continue
2026-08-05 19:39:06 +00:00
}
2026-08-09 02:26:49 +00:00
groupPositions [ key ] = len ( groups )
groups = append ( groups , [ ] int { index } )
2026-08-05 19:39:06 +00:00
}
return groups
}
func cloneItem ( input dnd . Item ) dnd . Item {
input . SourceRefs = cloneSourceRefs ( input . SourceRefs )
return input
}
func cloneRecord ( input normalizedRecord ) normalizedRecord {
input . item = cloneItem ( input . item )
input . inputIndexes = append ( [ ] int ( nil ) , input . inputIndexes ... )
return input
}
func cloneSourceRefs ( input [ ] source . SourceRef ) [ ] source . SourceRef {
return append ( [ ] source . SourceRef ( nil ) , input ... )
}
func sortedUniqueIndexes ( indexes [ ] int ) [ ] int {
if len ( indexes ) == 0 {
return nil
}
out := append ( [ ] int ( nil ) , indexes ... )
sort . Ints ( out )
write := 1
for _ , index := range out [ 1 : ] {
if index != out [ write - 1 ] {
out [ write ] = index
write ++
}
}
return out [ : write ]
}
func memberInputIndexes ( records [ ] normalizedRecord , members [ ] int ) [ ] int {
indexes := make ( [ ] int , 0 , len ( members ) )
for _ , member := range members {
indexes = append ( indexes , records [ member ] . inputIndexes ... )
}
return sortedUniqueIndexes ( indexes )
}
func recordValues ( records [ ] normalizedRecord ) [ ] dnd . Item {
if records == nil {
return nil
}
values := make ( [ ] dnd . Item , len ( records ) )
for index , record := range records {
values [ index ] = cloneItem ( record . item )
}
return values
}
func recordList ( records [ ] normalizedRecord ) dnd . ItemRegistry {
if records == nil {
return dnd . ItemRegistry { }
}
return dnd . ItemRegistry { Items : recordValues ( records ) }
}
2026-08-27 18:41:59 +00:00
func duplicateFinding ( retainedIndex int , removed [ ] int ) diagnostics . Finding {
2026-08-05 19:39:06 +00:00
const maxDisplayedIndices = 20
displayed := removed
if len ( displayed ) > maxDisplayedIndices {
displayed = displayed [ : maxDisplayedIndices ]
}
indices := make ( [ ] string , len ( displayed ) )
for index , removedIndex := range displayed {
indices [ index ] = strconv . Itoa ( removedIndex )
}
message := fmt . Sprintf ( "retained input index %d; removed input indices [%s]" , retainedIndex , strings . Join ( indices , ", " ) )
if omitted := len ( removed ) - len ( displayed ) ; omitted > 0 {
message += fmt . Sprintf ( "; %d additional removed input indices omitted" , omitted )
}
2026-08-27 16:40:17 +00:00
return diagnostics . Finding { Scope : itemScope ( retainedIndex ) , ReasonCode : ReasonCodeDuplicateItemCollapsed , Message : message }
2026-08-05 19:39:06 +00:00
}
func itemScope ( index int ) string { return fmt . Sprintf ( "items[%d]" , index ) }
func ModuleSpec ( ) pipeline . ModuleSpec {
2026-11-22 00:17:58 +00:00
return pipeline . ModuleSpec { Key : Key , Stage : pipeline . StageNormalize , ExecutionClass : contracts . ExecutionClassLLMBacked , Prompts : [ ] string { PromptID , CorrectionPromptID } , Requires : append ( [ ] string ( nil ) , requiredCapabilities ... ) , Provides : append ( [ ] string ( nil ) , providedCapabilities ... ) , ArtifactKind : dnd . ItemRegistryKind }
2026-08-05 19:39:06 +00:00
}
func Register ( registry * pipeline . NormalizerRegistry ) error {
return pipeline . RegisterNormalizerBuilder ( registry , ModuleSpec ( ) , validateOptions , func ( request pipeline . BuildRequest ) ( contracts . Normalizer [ dnd . ItemRegistry ] , error ) {
options , err := DecodeOptions ( request . Options )
if err != nil {
return nil , err
}
2026-11-22 00:17:58 +00:00
return New ( request . Dependencies . Engine , options )
2026-08-05 19:39:06 +00:00
} )
}
func validateOptions ( options map [ string ] any ) error { _ , err := DecodeOptions ( options ) ; return err }
func DecodeOptions ( options map [ string ] any ) ( Options , error ) {
if err := pipeline . RejectUnknownOptions ( options ) ; err != nil {
return Options { } , normalizerErrorf ( "%w" , err )
}
return Options { } , nil
}
func normalizerErrorf ( format string , args ... any ) error {
return fmt . Errorf ( "dnd item registry normalizer: " + format , args ... )
}