420 lines
18 KiB
Go
420 lines
18 KiB
Go
// Package locationregistry normalizes merged D&D location-registry candidates conservatively.
|
|
package locationregistry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
registrycodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/locationregistry"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/locations/identity"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared/diagnostics"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
const (
|
|
Key = "dnd/location-registry"
|
|
PromptID = "notarius/dnd.location_registry.normalize"
|
|
CorrectionPromptID = "notarius/dnd.location_registry.normalize.repair"
|
|
|
|
ReasonCodeLocationFieldsNormalized = "location_fields_normalized"
|
|
ReasonCodeLocationIDRecomputed = "location_id_recomputed"
|
|
ReasonCodeSourceReferencesNormalized = "source_references_normalized"
|
|
ReasonCodeDuplicateLocationCollapsed = "duplicate_location_collapsed"
|
|
ReasonCodeLocationSemanticReconciliationExhausted = "location_semantic_reconciliation_exhausted"
|
|
)
|
|
|
|
var requiredCapabilities = []string{"merged"}
|
|
var providedCapabilities = []string{"normalized"}
|
|
|
|
var _ contracts.Normalizer[dnd.LocationRegistry] = (*Normalizer)(nil)
|
|
|
|
type Options struct{}
|
|
|
|
type Normalizer struct {
|
|
engine *promptkit.Engine
|
|
}
|
|
|
|
func New(engine *promptkit.Engine, _ Options) (*Normalizer, error) {
|
|
if engine == nil {
|
|
return nil, normalizerErrorf("native engine must not be nil")
|
|
}
|
|
return &Normalizer{engine: engine}, nil
|
|
}
|
|
|
|
func (n *Normalizer) Key() string { return Key }
|
|
func (n *Normalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
|
|
func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[dnd.LocationRegistry]) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
|
if n == nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("normalizer must not be nil")
|
|
}
|
|
if n.engine == nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("native engine must not be nil")
|
|
}
|
|
if ctx == nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("context must not be nil")
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("context error before normalize: %w", err)
|
|
}
|
|
if req.Source == nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("source document must not be nil")
|
|
}
|
|
|
|
// Keep the already available merged artifact separate from preprocessed
|
|
// originals. A failed native response must not recover an earlier proposal.
|
|
merged := cloneRegistry(req.MergeOutput.Value)
|
|
order := shared.NewSourceRefOrder(req.Source)
|
|
records, baseFindings := preprocessRecords(merged, order)
|
|
deterministic := recordList(records)
|
|
if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
|
}
|
|
findings := append([]diagnostics.Finding(nil), baseFindings...)
|
|
discardedGroups := -1
|
|
var omissionDetails []string
|
|
|
|
fallback := func(_ context.Context, final *dnd.LocationRegistry) (dnd.LocationRegistry, error) {
|
|
value := cloneRegistry(merged)
|
|
if final != nil {
|
|
// Each interpreted candidate was built afresh from fixed original records,
|
|
// using only the final proposal's compatible, non-overlapping safe groups.
|
|
value = cloneRegistry(*final)
|
|
} else {
|
|
findings = nil
|
|
discardedGroups = -1
|
|
omissionDetails = nil
|
|
}
|
|
if err := validateFallback(value, req.Source); err != nil {
|
|
return dnd.LocationRegistry{}, err
|
|
}
|
|
return value, nil
|
|
}
|
|
finish := func(value dnd.LocationRegistry, outcome contracts.ProducerOutcome, err error) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
|
if err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("normalize locations: %w", err)
|
|
}
|
|
result, err := normalizationResult(value, findings)
|
|
if err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
|
}
|
|
result.Outcome = outcome
|
|
if outcome.Disposition == contracts.ProducerFallback {
|
|
details := append([]string(nil), omissionDetails...)
|
|
details = append(details, outcome.FallbackReason)
|
|
groups, err := diagnostics.Collect([]diagnostics.Finding{semanticFallbackFinding(discardedGroups, details...)}, contracts.DiagnosticDispositionWarning, contracts.DiagnosticCategoryFallback)
|
|
if err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect fallback diagnostic: %w", err)
|
|
}
|
|
result.Diagnostics = append(result.Diagnostics, groups...)
|
|
}
|
|
return result, nil
|
|
}
|
|
noModel := func() (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
|
value, outcome, err := pipeline.AssessProducer(ctx, req.Operation, deterministic, func(ctx context.Context) (dnd.LocationRegistry, error) { return fallback(ctx, nil) })
|
|
return finish(value, outcome, err)
|
|
}
|
|
if len(records) < 2 {
|
|
return noModel()
|
|
}
|
|
candidates, envelopes, err := reconciliationInputs(records)
|
|
if err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation inputs: %w", err)
|
|
}
|
|
preparation, err := semanticreconcile.Prepare(req.Source, candidates, semanticreconcile.DefaultLimits())
|
|
if err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
|
|
}
|
|
if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{
|
|
"disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(),
|
|
"candidate_mappings": preparation.CandidateMappings(), "candidates": candidates,
|
|
}); err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
|
|
}
|
|
switch preparation.Disposition() {
|
|
case semanticreconcile.InsufficientCandidates:
|
|
return noModel()
|
|
case semanticreconcile.LimitExceeded:
|
|
value, outcome, err := pipeline.FinishProducerFailure(ctx, req.Operation.Controls.Mode, normalizerErrorf("candidate set exceeded semantic reconciliation preparation limits"), func(ctx context.Context) (dnd.LocationRegistry, error) { return fallback(ctx, nil) })
|
|
return finish(value, outcome, err)
|
|
}
|
|
initial, err := semanticreconcile.InitialRequest(preparation, PromptID, req.LLMProfile, req.SessionID)
|
|
if err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("build native request: %w", err)
|
|
}
|
|
value, outcome, err := pipeline.RunProducer(ctx, n.engine, req.Operation, initial,
|
|
func(_ context.Context, result *promptkit.RunResult) (*dnd.LocationRegistry, []string, error) {
|
|
response, err := semanticreconcile.DecodeProposal(result.RawOutput)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
assessment := preparation.Assess(response)
|
|
if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
|
|
if err != nil {
|
|
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)
|
|
}
|
|
findings = append(append([]diagnostics.Finding(nil), baseFindings...), semanticFindings...)
|
|
discardedGroups = assessment.DiscardedGroupCount()
|
|
omissionDetails = semanticreconcile.IssueDetails(assessment.Issues())
|
|
guidance, err := semanticreconcile.CorrectionDetails(assessment.Issues(), preparation.CandidateNumberRange())
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
candidate := recordList(applied)
|
|
return &candidate, guidance, nil
|
|
},
|
|
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
|
return semanticreconcile.CorrectionRequest(CorrectionPromptID, req.LLMProfile, req.SessionID, guidance), nil
|
|
}, fallback)
|
|
return finish(value, outcome, err)
|
|
}
|
|
|
|
// validateFallback checks the durable artifact and the identity/source invariants
|
|
// used by registry consumers, not the configured ordinary validator chain.
|
|
func validateFallback(value dnd.LocationRegistry, document *source.SourceDocument) error {
|
|
if _, err := registrycodec.New().Encode(value); err != nil {
|
|
return normalizerErrorf("construct locations fallback: %w", err)
|
|
}
|
|
if issues := identity.ValidateList(value); len(issues) > 0 {
|
|
return normalizerErrorf("construct locations fallback: registry identity is not usable downstream")
|
|
}
|
|
index := source.NewDocumentIndex(document)
|
|
for _, location := range value.Locations {
|
|
for _, reference := range location.SourceRefs {
|
|
if err := index.ValidateRef(reference); err != nil {
|
|
return normalizerErrorf("construct locations fallback: %w", err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func cloneRegistry(value dnd.LocationRegistry) dnd.LocationRegistry {
|
|
if value.Locations == nil {
|
|
return dnd.LocationRegistry{}
|
|
}
|
|
result := dnd.LocationRegistry{Locations: make([]dnd.Location, len(value.Locations))}
|
|
for i, location := range value.Locations {
|
|
result.Locations[i] = cloneLocation(location)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func semanticFallbackFinding(discarded int, details ...string) diagnostics.Finding {
|
|
message := "semantic reconciliation did not obtain ordinary acceptance; using supported fallback output"
|
|
if discarded > 0 {
|
|
message = fmt.Sprintf("%d proposal group(s) omitted from the final semantic proposal; all affected candidates were preserved separately", discarded)
|
|
}
|
|
if len(details) > 0 {
|
|
message = diagnostics.Aggregate(message, details)
|
|
}
|
|
return diagnostics.Finding{Scope: "locations", ReasonCode: ReasonCodeLocationSemanticReconciliationExhausted, Message: message}
|
|
}
|
|
|
|
func normalizationResult(value dnd.LocationRegistry, findings []diagnostics.Finding) (contracts.TypedNormalizeResult[dnd.LocationRegistry], error) {
|
|
groups, err := diagnostics.Collect(findings, contracts.DiagnosticDispositionObservation, contracts.DiagnosticCategoryNormalization)
|
|
if err != nil {
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("collect normalization diagnostics: %w", err)
|
|
}
|
|
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{Value: value, Diagnostics: groups}, nil
|
|
}
|
|
|
|
type normalizedRecord struct {
|
|
location dnd.Location
|
|
inputIndexes []int
|
|
earliest int
|
|
}
|
|
|
|
func preprocessRecords(input dnd.LocationRegistry, order shared.SourceRefOrder) ([]normalizedRecord, []diagnostics.Finding) {
|
|
if input.Locations == nil {
|
|
return nil, nil
|
|
}
|
|
records := make([]normalizedRecord, len(input.Locations))
|
|
findings := make([]diagnostics.Finding, 0)
|
|
for index, inputLocation := range input.Locations {
|
|
location, fieldsChanged, refsChanged := normalizeRecord(inputLocation, order)
|
|
records[index] = normalizedRecord{location: location, inputIndexes: []int{index}, earliest: index}
|
|
if fieldsChanged {
|
|
findings = append(findings, diagnostics.Finding{Scope: locationScope(index), ReasonCode: ReasonCodeLocationFieldsNormalized, Message: fmt.Sprintf("input index %d: location name normalized for %s", index, diagnostics.Quote(inputLocation.Name))})
|
|
}
|
|
if refsChanged {
|
|
findings = append(findings, diagnostics.Finding{Scope: locationScope(index), ReasonCode: ReasonCodeSourceReferencesNormalized, Message: fmt.Sprintf("input index %d: source references normalized (original count %d, final count %d)", index, len(inputLocation.SourceRefs), len(location.SourceRefs))})
|
|
}
|
|
if inputLocation.ID != location.ID {
|
|
findings = append(findings, diagnostics.Finding{Scope: locationScope(index), ReasonCode: ReasonCodeLocationIDRecomputed, Message: fmt.Sprintf("input index %d: location ID recomputed from %s", index, diagnostics.Quote(location.Name))})
|
|
}
|
|
}
|
|
groups := exactDuplicateGroups(records)
|
|
output := make([]normalizedRecord, 0, len(groups))
|
|
for _, members := range groups {
|
|
retained := cloneRecord(records[members[0]])
|
|
for _, member := range members[1:] {
|
|
retained.inputIndexes = append(retained.inputIndexes, records[member].inputIndexes...)
|
|
}
|
|
retained.inputIndexes = sortedUniqueIndexes(retained.inputIndexes)
|
|
output = append(output, retained)
|
|
if len(members) > 1 {
|
|
findings = append(findings, duplicateFinding(retained.earliest, memberInputIndexes(records, members[1:])))
|
|
}
|
|
}
|
|
return output, findings
|
|
}
|
|
|
|
func normalizeRecord(input dnd.Location, order shared.SourceRefOrder) (dnd.Location, bool, bool) {
|
|
output := cloneLocation(input)
|
|
output.Name = identity.NormalizeDisplay(input.Name)
|
|
output.SourceRefs = order.Canonicalize(input.SourceRefs)
|
|
output.ID = identity.DeriveID(output.Name, output.SourceRefs)
|
|
return output, input.Name != output.Name, !reflect.DeepEqual(input.SourceRefs, output.SourceRefs)
|
|
}
|
|
|
|
func exactDuplicateGroups(records []normalizedRecord) [][]int {
|
|
groups := make([][]int, 0, len(records))
|
|
groupPositions := make(map[string][]int, len(records))
|
|
for index, record := range records {
|
|
key := exactDuplicateKey(record.location)
|
|
comparisonName := identity.ComparisonKey(record.location.Name)
|
|
matched := false
|
|
for _, groupIndex := range groupPositions[key] {
|
|
first := records[groups[groupIndex][0]]
|
|
if identity.ComparisonKey(first.location.Name) == comparisonName && reflect.DeepEqual(first.location.SourceRefs, record.location.SourceRefs) {
|
|
groups[groupIndex] = append(groups[groupIndex], index)
|
|
matched = true
|
|
break
|
|
}
|
|
}
|
|
if matched {
|
|
continue
|
|
}
|
|
groupPositions[key] = append(groupPositions[key], len(groups))
|
|
groups = append(groups, []int{index})
|
|
}
|
|
return groups
|
|
}
|
|
|
|
type duplicateKey struct {
|
|
ComparisonName string `json:"comparison_name"`
|
|
SourceRefs []source.SourceRef `json:"source_refs"`
|
|
}
|
|
|
|
func exactDuplicateKey(location dnd.Location) string {
|
|
encoded, err := json.Marshal(duplicateKey{
|
|
ComparisonName: identity.ComparisonKey(location.Name),
|
|
SourceRefs: location.SourceRefs,
|
|
})
|
|
if err != nil {
|
|
panic("encode location duplicate key")
|
|
}
|
|
return string(encoded)
|
|
}
|
|
|
|
func cloneLocation(input dnd.Location) dnd.Location {
|
|
input.SourceRefs = cloneSourceRefs(input.SourceRefs)
|
|
return input
|
|
}
|
|
func cloneRecord(input normalizedRecord) normalizedRecord {
|
|
input.location = cloneLocation(input.location)
|
|
input.inputIndexes = append([]int(nil), input.inputIndexes...)
|
|
return input
|
|
}
|
|
func cloneSourceRefs(input []source.SourceRef) []source.SourceRef {
|
|
if input == nil {
|
|
return nil
|
|
}
|
|
return append([]source.SourceRef(nil), input...)
|
|
}
|
|
|
|
func sortedUniqueIndexes(indexes []int) []int {
|
|
if len(indexes) == 0 {
|
|
return nil
|
|
}
|
|
out := append([]int(nil), indexes...)
|
|
sort.Ints(out)
|
|
write := 1
|
|
for _, index := range out[1:] {
|
|
if index != out[write-1] {
|
|
out[write] = index
|
|
write++
|
|
}
|
|
}
|
|
return out[:write]
|
|
}
|
|
func memberInputIndexes(records []normalizedRecord, members []int) []int {
|
|
indexes := make([]int, 0, len(members))
|
|
for _, member := range members {
|
|
indexes = append(indexes, records[member].inputIndexes...)
|
|
}
|
|
return sortedUniqueIndexes(indexes)
|
|
}
|
|
func recordValues(records []normalizedRecord) []dnd.Location {
|
|
if records == nil {
|
|
return nil
|
|
}
|
|
values := make([]dnd.Location, len(records))
|
|
for index, record := range records {
|
|
values[index] = cloneLocation(record.location)
|
|
}
|
|
return values
|
|
}
|
|
func recordList(records []normalizedRecord) dnd.LocationRegistry {
|
|
if records == nil {
|
|
return dnd.LocationRegistry{}
|
|
}
|
|
return dnd.LocationRegistry{Locations: recordValues(records)}
|
|
}
|
|
|
|
func duplicateFinding(retainedIndex int, removed []int) diagnostics.Finding {
|
|
const maxDisplayedIndices = 20
|
|
displayed := removed
|
|
if len(displayed) > maxDisplayedIndices {
|
|
displayed = displayed[:maxDisplayedIndices]
|
|
}
|
|
indices := make([]string, len(displayed))
|
|
for index, removedIndex := range displayed {
|
|
indices[index] = strconv.Itoa(removedIndex)
|
|
}
|
|
message := fmt.Sprintf("retained input index %d; removed input indices [%s]", retainedIndex, strings.Join(indices, ", "))
|
|
if omitted := len(removed) - len(displayed); omitted > 0 {
|
|
message += fmt.Sprintf("; %d additional removed input indices omitted", omitted)
|
|
}
|
|
return diagnostics.Finding{Scope: locationScope(retainedIndex), ReasonCode: ReasonCodeDuplicateLocationCollapsed, Message: message}
|
|
}
|
|
func locationScope(index int) string { return fmt.Sprintf("locations[%d]", index) }
|
|
|
|
func ModuleSpec() pipeline.ModuleSpec {
|
|
return pipeline.ModuleSpec{Key: Key, Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: append([]string(nil), requiredCapabilities...), Provides: append([]string(nil), providedCapabilities...), ArtifactKind: dnd.LocationRegistryKind}
|
|
}
|
|
func Register(registry *pipeline.NormalizerRegistry) error {
|
|
return pipeline.RegisterNormalizerBuilder(registry, ModuleSpec(), validateOptions, func(request pipeline.BuildRequest) (contracts.Normalizer[dnd.LocationRegistry], error) {
|
|
options, err := DecodeOptions(request.Options)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return New(request.Dependencies.Engine, options)
|
|
})
|
|
}
|
|
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
|
func DecodeOptions(options map[string]any) (Options, error) {
|
|
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
|
return Options{}, normalizerErrorf("%w", err)
|
|
}
|
|
return Options{}, nil
|
|
}
|
|
func normalizerErrorf(format string, args ...any) error {
|
|
return fmt.Errorf("dnd location registry normalizer: "+format, args...)
|
|
}
|