Normalize projected collection history evidence

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Interface Contract Remediation Implementation Plan
 
-Status: in progress. Stages 1–4 are complete; Stages 5–18 remain pending.
+Status: in progress. Stages 1–5 are complete; Stages 6–18 remain pending.
 Baseline: `f6d54a094a2b5ada5748686620659399dce7ebf0`.
 Audience: a Luna coding agent executing the stages sequentially.
 
@@ -395,7 +395,10 @@ provenance, and explanation cases; focused check:
 ## Stage 5 — Normalize Projected Collection Evidence
 
 Owners: `sequence_history_projection.go`, sequence combination evidence,
-`history_source.go`, and `history_sequence_test.go`. Status: pending.
+`history_source.go`, and `history_sequence_test.go`. Status: complete. Validation:
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'SequenceHistory|Mapped.*History|Tier.*History|ProjectedSequenceHistory|Destination|HistorySourceSequence' -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+and `git diff --check`.
 
 1. Extend source normalization to assigned sequence coordinates using
    `walkHistoryApplication`, `historyApplicationLocalPath`, and existing sequence

internal/engine/history_sequence_test.go

@@ -577,6 +577,197 @@ func TestMappedSequenceHistoryReplacesWholeRecordsAndKeepsDisplacedEvidence(t *t
 	}
 }
 
+func TestHistorySourceSequenceApplicationsUseAssignedCoordinates(t *testing.T) {
+	t.Parallel()
+
+	policy := mustCompileSequenceIdentityPolicy(t, []sequenceIdentityRegistration{{
+		selector:       sequenceIdentitySelector{literalSequenceSelector("collections"), anyMappingSequenceSelector(), literalSequenceSelector("records")},
+		identityFields: []string{"id"},
+	}})
+	first := parseCompositionDocument(t, "source-1", "collections: {north: {records: [{id: a, value: first}]}}\n")
+	second := parseCompositionDocument(t, "source-2", "collections: {north: {records: [{id: b, value: second}]}}\n")
+	composed, diagnostics := composeDocumentsWithPolicy(policy, first, second)
+	if len(diagnostics) != 0 {
+		t.Fatalf("compose diagnostics = %#v", diagnostics)
+	}
+	builder := newHistoryBuilder()
+	builder.observeSourceEvidence(composed.root, composed.origins, composed.applications, composed.overlays)
+	model := builder.freeze()
+
+	appendedPath := Keys("collections", "north", "records").Index(1).Key("value")
+	selections := model.selectHistory(appendedPath)
+	foundSecond := false
+	for _, selection := range selections {
+		if selection.role == historyRoleContribution && selection.attribution.origin.Locations[0].SourceID == "source-2" {
+			foundSecond = true
+		}
+	}
+	if !foundSecond {
+		t.Fatalf("appended source was not attributed at its assigned index: %#v", selections)
+	}
+	for _, selection := range model.selectHistory(Keys("collections", "north", "records").Index(0).Key("value")) {
+		if selection.role == historyRoleContribution && selection.attribution.origin.Locations[0].SourceID == "source-2" {
+			t.Fatalf("later source acquired an earlier append index: %#v", selection)
+		}
+	}
+	local, ok := historyApplicationLocalPath(composed.applications[1], appendedPath)
+	if !ok || local.String() != "/collections/north/records/0/value" {
+		t.Fatalf("assigned append path localizes to %s, %v", local.String(), ok)
+	}
+}
+
+func TestHistorySourceSequenceReplacementTracksMappedAndWholeSides(t *testing.T) {
+	t.Parallel()
+
+	lowerRoot := parseCompositionDocument(t, "lower", `records:
+  - id: first
+  - id: middle
+  - id: last
+    nested: {removed: lower}
+`)
+	higherRoot := parseCompositionDocument(t, "higher", `records:
+  - id: last
+    nested: {kept: higher}
+  - id: first
+  - id: added
+`)
+	lower, _ := documentAtPath(lowerRoot, Keys("records"))
+	higher, _ := documentAtPath(higherRoot, Keys("records"))
+	combined, diagnostics := combineSequence(
+		lower, higher, sequenceCombinationPolicy(sequenceModeUpsert, "id"),
+		sequenceCombinationContext{destination: Keys("records")},
+	)
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	finalRoot := document.Clone(lowerRoot)
+	finalSequence, _ := documentAtPath(finalRoot, Keys("records"))
+	*finalSequence = *document.Clone(combined.value)
+	lowerOrigins := make(map[string]Origin)
+	collectOverlayOrigins(lowerOrigins, lowerRoot, Path{}, LayerBase)
+	higherOrigins := make(map[string]Origin)
+	collectOverlayOrigins(higherOrigins, higherRoot, Path{}, LayerOverlay)
+	assembly := newAssemblyAccumulator(&composedDocument{root: finalRoot, origins: lowerOrigins})
+	applyOverlaySequenceEvidence(
+		assembly, Keys("records"), lower, higher, combined,
+		collectionPolicyResolution{operation: sequenceModeUpsert}, higherOrigins, collectionPolicyScope{}, LayerOverlay,
+	)
+	higherApplication := historyApplication{
+		layer: LayerOverlay, value: higherRoot,
+		sequences: []sequenceHistoryContribution{{
+			path: Keys("records"), mode: sequenceHistoryMapped,
+			items: combined.higherItems, replacements: combined.replacements,
+		}},
+	}
+	higherApplication.sequencesByPath = indexSequenceHistoryContributions(higherApplication.sequences)
+	applications := []historyApplication{
+		{layer: LayerBase, value: lowerRoot}, higherApplication,
+	}
+	builder := newHistoryBuilder()
+	builder.observeSourceEvidence(finalRoot, assembly.current.origins, applications, assembly.current.overlays)
+	model := builder.freeze()
+
+	removedPath := Keys("records").Index(2).Key("nested").Key("removed")
+	removed := model.selectHistory(removedPath)
+	var lowerContribution, removal bool
+	for _, selection := range removed {
+		if selection.role == historyRoleContribution && selection.attribution.origin.Locations[0].SourceID == "lower" {
+			lowerContribution = true
+		}
+		if selection.role == historyRoleOperation {
+			removal = selection.before && !selection.after
+		}
+	}
+	if !lowerContribution || !removal {
+		t.Fatalf("omitted descendant evidence = %#v", removed)
+	}
+
+	introducedPath := Keys("records").Index(2).Key("nested").Key("kept")
+	introduced := model.selectHistory(introducedPath)
+	var higherContribution, introduction bool
+	for _, selection := range introduced {
+		if selection.role == historyRoleContribution && selection.attribution.origin.Locations[0].SourceID == "higher" {
+			higherContribution = true
+		}
+		if selection.role == historyRoleOperation {
+			introduction = !selection.before && selection.after
+		}
+	}
+	if !higherContribution || !introduction {
+		t.Fatalf("mapped introduced descendant evidence = %#v", introduced)
+	}
+	if local, ok := historyApplicationLocalPath(higherApplication, introducedPath); !ok || local.String() != "/records/0/nested/kept" {
+		t.Fatalf("mapped path localizes to %s, %v", local.String(), ok)
+	}
+	if selections := model.selectHistory(Keys("records").Index(2).Key("nested").Key("removed")); len(selections) > 0 {
+		for _, selection := range selections {
+			if selection.role == historyRoleContribution && selection.attribution.origin.Locations[0].SourceID == "higher" {
+				t.Fatalf("unaddressed source acquired removed-record attribution: %#v", selections)
+			}
+		}
+	}
+
+	wholeLower := parseCompositionDocument(t, "whole-lower", "records: [{id: old, nested: {removed: old}}]\n")
+	wholeHigher := parseCompositionDocument(t, "whole-higher", "records: [{id: new, nested: {added: new}}]\n")
+	wholeLowerSequence, _ := documentAtPath(wholeLower, Keys("records"))
+	wholeHigherSequence, _ := documentAtPath(wholeHigher, Keys("records"))
+	whole, diagnostics := combineSequence(
+		wholeLowerSequence, wholeHigherSequence, sequenceCombinationPolicy(sequenceModeReplace),
+		sequenceCombinationContext{destination: Keys("records")},
+	)
+	if len(diagnostics) != 0 {
+		t.Fatal(diagnostics)
+	}
+	wholeFinal := document.Clone(wholeLower)
+	wholeFinalSequence, _ := documentAtPath(wholeFinal, Keys("records"))
+	*wholeFinalSequence = *document.Clone(whole.value)
+	wholeLowerOrigins := make(map[string]Origin)
+	collectOverlayOrigins(wholeLowerOrigins, wholeLower, Path{}, LayerBase)
+	wholeHigherOrigins := make(map[string]Origin)
+	collectOverlayOrigins(wholeHigherOrigins, wholeHigher, Path{}, LayerOverlay)
+	wholeAssembly := newAssemblyAccumulator(&composedDocument{root: wholeFinal, origins: wholeLowerOrigins})
+	applyOverlaySequenceEvidence(
+		wholeAssembly, Keys("records"), wholeLowerSequence, wholeHigherSequence, whole,
+		collectionPolicyResolution{operation: sequenceModeReplace}, wholeHigherOrigins, collectionPolicyScope{}, LayerOverlay,
+	)
+	wholeHigherApplication := historyApplication{
+		layer: LayerOverlay, value: wholeHigher,
+		sequences: []sequenceHistoryContribution{sequenceHistoryContributionForCombination(Keys("records"), sequenceModeReplace, wholeHigherSequence, whole)},
+	}
+	wholeHigherApplication.sequencesByPath = indexSequenceHistoryContributions(wholeHigherApplication.sequences)
+	wholeBuilder := newHistoryBuilder()
+	wholeBuilder.observeSourceEvidence(wholeFinal, wholeAssembly.current.origins, []historyApplication{
+		{layer: LayerBase, value: wholeLower}, wholeHigherApplication,
+	}, wholeAssembly.current.overlays)
+	wholeModel := wholeBuilder.freeze()
+	if len(wholeAssembly.current.overlays) != 1 {
+		t.Fatalf("whole-sequence replacement records = %#v", wholeAssembly.current.overlays)
+	}
+	wholeIntroductions := wholeAssembly.current.overlays[0].introduced
+	if len(wholeIntroductions) != 5 {
+		t.Fatalf("whole-sequence introduced paths = %#v", wholeIntroductions)
+	}
+	var introducedNested bool
+	for _, fact := range wholeIntroductions {
+		if sameConcreteHistoryPath(fact.path, Keys("records").Index(0).Key("nested").Key("added")) {
+			introducedNested = true
+		}
+	}
+	if !introducedNested {
+		t.Fatalf("whole-sequence nested introduction missing: %#v", wholeIntroductions)
+	}
+	oldWholePath := Keys("records").Index(0).Key("nested").Key("removed")
+	newWholePath := Keys("records").Index(0).Key("nested").Key("added")
+	oldWhole := wholeModel.selectHistory(oldWholePath)
+	newWhole := wholeModel.selectHistory(newWholePath)
+	if len(oldWhole) < 2 || oldWhole[len(oldWhole)-1].role != historyRoleOperation || !oldWhole[len(oldWhole)-1].before || oldWhole[len(oldWhole)-1].after {
+		t.Fatalf("whole-sequence removed descendant selections = %#v", oldWhole)
+	}
+	if len(newWhole) < 2 || newWhole[len(newWhole)-1].role != historyRoleOperation || newWhole[len(newWhole)-1].before || !newWhole[len(newWhole)-1].after {
+		t.Fatalf("whole-sequence introduced descendant selections = %#v", newWhole)
+	}
+}
+
 func mustParsePath(t *testing.T, value string) Path {
 	t.Helper()
 	path, err := ParsePath(value)

internal/engine/history_source.go

@@ -151,6 +151,20 @@ func (builder *historyBuilder) sourceContributions(path Path, summary Origin) []
 		if !ok || !sourceLocationsContainedBy(attribution.origin, summary) {
 			continue
 		}
+		if observation.payload.id < 0 || observation.payload.id >= len(builder.model.sourcePayloads) {
+			continue
+		}
+		payload := builder.model.sourcePayloads[observation.payload.id]
+		if payload.kind != historySourceApplicationPayload {
+			continue
+		}
+		local, ok := historyApplicationLocalPath(payload.application, path)
+		if !ok {
+			continue
+		}
+		if _, exists := documentAtPath(payload.application.value, local); !exists {
+			continue
+		}
 		if _, exists := seen[observation.payload]; exists {
 			continue
 		}

internal/engine/overlay.go

@@ -445,11 +445,12 @@ func applyOverlaySequenceEvidence(
 			}
 		}
 		if lower != nil {
+			introduced := overlayPathOrigins(higher, incomingOrigins, path, layer)
 			result.overlays = append(result.overlays, overlayRecord{
 				change: Change{
 					Path: path.clone(), Origin: incomingOrigin(incomingOrigins, path, higher, layer),
 					Previous: previousOrigins(previous),
-				}, displaced: displaced,
+				}, displaced: displaced, introduced: introduced,
 			})
 		}
 	} else {
@@ -468,12 +469,13 @@ func applyOverlaySequenceEvidence(
 			previous := cloneOrigin(result.origins[lowerPath.String()])
 			deleteOriginSubtree(result.origins, lowerPath, lowerItem)
 			collectOverlayOriginsAt(result.origins, incomingOrigins, higherItem, higherPath, outputPath, layer)
+			introduced := overlayPathOriginsAt(higherItem, incomingOrigins, higherPath, outputPath, layer)
 			result.overlays = append(result.overlays, overlayRecord{
 				change: Change{
 					Path: outputPath.clone(), Origin: incomingOrigin(incomingOrigins, higherPath, higherItem, layer),
 					Previous: previousOrigins(previous),
 				},
-				displaced: displaced,
+				displaced: displaced, introduced: introduced,
 			})
 		}
 		for _, projection := range combined.higherItems {
@@ -521,10 +523,16 @@ func collectOverlayOriginsFrom(origins, incomingOrigins map[string]Origin, node
 }
 
 func overlayPathOrigins(node *document.Node, incomingOrigins map[string]Origin, path Path, layer Layer) []historyPathOrigin {
+	return overlayPathOriginsAt(node, incomingOrigins, path, path, layer)
+}
+
+func overlayPathOriginsAt(node *document.Node, incomingOrigins map[string]Origin, from, to Path, layer Layer) []historyPathOrigin {
 	var introduced []historyPathOrigin
-	walkDocument(node, path, func(affected Path, value *document.Node) {
+	walkDocument(node, Path{}, func(relative Path, value *document.Node) {
+		source := appendDestination(from, relative)
+		affected := appendDestination(to, relative)
 		introduced = append(introduced, historyPathOrigin{
-			path: affected.clone(), origin: incomingOrigin(incomingOrigins, affected, value, layer),
+			path: affected.clone(), origin: incomingOrigin(incomingOrigins, source, value, layer),
 			sensitive: value.Sensitive,
 		})
 	})