Normalize original resolution history evidence

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Interface Contract Remediation Implementation Plan
 
-Status: in progress. Stages 1–7 are complete; Stages 8–18 remain pending.
+Status: in progress. Stages 1–8 are complete; Stages 9–18 remain pending.
 Baseline: `f6d54a094a2b5ada5748686620659399dce7ebf0`.
 Audience: a Luna coding agent executing the stages sequentially.
 
@@ -475,7 +475,11 @@ pre-CLI input, raw CLI application, and typed post-CLI entry state. Focused chec
 ## Stage 8 — Normalize Original Resolution And Net Replay Groups
 
 Owners: new `history_resolution.go`, `history.go`, original resolution snapshots
-and coverage indexes. Status: pending.
+and coverage indexes. Status: complete. Validation:
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'Resolution|Coalesced|Overlapping|Explain' -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./...`,
+and `git diff --check`.
 
 1. Store the ordered original snapshots after successful audit. Normalize their
    before/after subtrees into concrete observations and retain every original

internal/engine/explain.go

@@ -126,13 +126,17 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) {
 		})
 	}
 	working = document.Clone(r.input)
-	for _, snapshot := range r.resolutionReplay {
+	for _, group := range r.resolutionReplay {
-		relevant := pathsIntersect(snapshot.change.Path, path)
+		if group.coveringRecordID < 0 || group.coveringRecordID >= len(r.resolution) {
+			continue
+		}
+		covering := r.resolution[group.coveringRecordID]
+		relevant := pathsIntersect(group.change.Path, path)
 		var before ValueSnapshot
 		if relevant {
 			before = valueSnapshot(working, path, r.secrets)
 		}
-		replaceHistoryPath(&working, snapshot.change.Path, snapshot.after)
+		replaceHistoryPath(&working, group.change.Path, covering.after)
 		if !relevant {
 			continue
 		}
@@ -143,9 +147,9 @@ func (r *Inspection) Explain(path Path) (Explanation, bool) {
 		explanation.History = append(explanation.History, ValueChange{
 			Before:       before,
 			After:        after,
-			Origin:       cloneOrigin(snapshot.change.Origin),
+			Origin:       cloneOrigin(group.change.Origin),
-			Reasons:      append([]string(nil), snapshot.change.Reasons...),
+			Reasons:      append([]string(nil), group.change.Reasons...),
-			Dependencies: clonePaths(snapshot.change.Dependencies),
+			Dependencies: clonePaths(group.change.Dependencies),
 		})
 	}
 

internal/engine/history.go

@@ -103,11 +103,9 @@ func addInspectionHistory(counter *retainedNodeCounter, inspection *Inspection)
 			return false
 		}
 	}
-	for _, snapshots := range [][]resolutionSnapshot{inspection.resolution, inspection.resolutionReplay} {
+	for _, snapshot := range inspection.resolution {
-		for _, snapshot := range snapshots {
+		if !counter.add(snapshot.before) || !counter.add(snapshot.after) {
-			if !counter.add(snapshot.before) || !counter.add(snapshot.after) {
+			return false
-				return false
-			}
 		}
 	}
 	for _, operations := range [][]inheritanceOperation{inspection.inheritance, inspection.metadata.inheritance} {
@@ -492,12 +490,12 @@ func mergeHistoryAtPath(root **document.Node, path Path, incoming *document.Node
 // Resolution records describe one before/final transformation, not assignment
 // order. A parent already carries its descendants' final values; merge overlapping
 // intent into that event rather than replay a child assignment a second time.
-func coalescedResolutionSnapshots(snapshots []resolutionSnapshot) []resolutionSnapshot {
+func coalescedResolutionSnapshots(snapshots []resolutionSnapshot) []resolutionReplayGroup {
-	groups := make([]resolutionSnapshot, 0, len(snapshots))
+	groups := make([]resolutionReplayGroup, 0, len(snapshots))
 	reasons := make([]map[string]struct{}, 0, len(snapshots))
 	dependencies := make([]map[string]struct{}, 0, len(snapshots))
 	var coverage pathPrefixIndex
-	for _, snapshot := range snapshots {
+	for recordID, snapshot := range snapshots {
 		pointer := snapshot.change.Path.String()
 		groupIndex := -1
 		coverage.visitAncestors(pointer, func(id int) {
@@ -507,6 +505,7 @@ func coalescedResolutionSnapshots(snapshots []resolutionSnapshot) []resolutionSn
 		})
 		if groupIndex >= 0 {
 			group := &groups[groupIndex]
+			group.originalRecordIDs = append(group.originalRecordIDs, recordID)
 			for _, reason := range snapshot.change.Reasons {
 				if _, found := reasons[groupIndex][reason]; !found {
 					reasons[groupIndex][reason] = struct{}{}
@@ -525,15 +524,19 @@ func coalescedResolutionSnapshots(snapshots []resolutionSnapshot) []resolutionSn
 			}
 			continue
 		}
-		snapshot.change = cloneChange(snapshot.change)
+		change := cloneChange(snapshot.change)
-		groups = append(groups, snapshot)
+		groups = append(groups, resolutionReplayGroup{
-		reasonSet := make(map[string]struct{}, len(snapshot.change.Reasons))
+			originalRecordIDs: []int{recordID},
-		for _, reason := range snapshot.change.Reasons {
+			coveringRecordID:  recordID,
+			change:            change,
+		})
+		reasonSet := make(map[string]struct{}, len(change.Reasons))
+		for _, reason := range change.Reasons {
 			reasonSet[reason] = struct{}{}
 		}
 		reasons = append(reasons, reasonSet)
-		dependencySet := make(map[string]struct{}, len(snapshot.change.Dependencies))
+		dependencySet := make(map[string]struct{}, len(change.Dependencies))
-		for _, dependency := range snapshot.change.Dependencies {
+		for _, dependency := range change.Dependencies {
 			dependencySet[dependency.String()] = struct{}{}
 		}
 		dependencies = append(dependencies, dependencySet)

internal/engine/history_budget_test.go

@@ -32,9 +32,6 @@ func retainedInspectionNodeCountForTest(inspection *Inspection) int {
 	for _, snapshot := range inspection.resolution {
 		stack = append(stack, snapshot.before, snapshot.after)
 	}
-	for _, snapshot := range inspection.resolutionReplay {
-		stack = append(stack, snapshot.before, snapshot.after)
-	}
 	for _, operation := range inspection.inheritance {
 		stack = append(stack, operation.parent, operation.effective)
 	}

internal/engine/history_model.go

@@ -61,6 +61,7 @@ type historyModel struct {
 	sourcePayloads      []historySourcePayload
 	inheritancePayloads []historyInheritancePayload
 	cliPayloads         []historySourcePayload
+	resolutionPayload   *historyResolutionPayload
 	preCLICheckpoint    *historyCheckpointPayload
 	postCLICheckpoint   *historyCheckpointPayload
 

internal/engine/history_resolution.go

@@ -0,0 +1,127 @@
+package engine
+
+import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+
+// resolutionReplayGroup is the explanation projection of one or more ordered
+// resolution records. The covering record identifies the before/after roots;
+// originalRecordIDs keep every coalesced record available to other projections.
+type resolutionReplayGroup struct {
+	originalRecordIDs []int
+	coveringRecordID  int
+	change            Change
+}
+
+type historyResolutionPayload struct {
+	snapshots    []resolutionSnapshot
+	replayGroups []resolutionReplayGroup
+}
+
+// observeResolutionEvidence retains audited records in producer order and
+// indexes the concrete shapes in every original before/after subtree. Snapshot
+// roots are already detached by runResolution and are shared with the replay
+// group that refers to their covering record.
+func (builder *historyBuilder) observeResolutionEvidence(snapshots []resolutionSnapshot) {
+	builder.requireOpen()
+	if len(snapshots) == 0 {
+		return
+	}
+	if builder.model.resolutionPayload != nil {
+		panic("configkit: resolution history already registered")
+	}
+	if builder.scratch == nil {
+		panic("configkit: resolution history requires the pre-resolution document")
+	}
+
+	payload := &historyResolutionPayload{
+		snapshots:    make([]resolutionSnapshot, len(snapshots)),
+		replayGroups: coalescedResolutionSnapshots(snapshots),
+	}
+	attributions := make([]historyAttributionRef, len(snapshots))
+	for recordID, snapshot := range snapshots {
+		snapshot.change = cloneChange(snapshot.change)
+		payload.snapshots[recordID] = snapshot
+
+		ref := builder.nextPayload(historyPhaseResolution)
+		if ref.id != recordID {
+			panic("configkit: resolution payload identifiers are out of order")
+		}
+		attribution := builder.addAttribution(snapshot.change.Origin, snapshot.change.Reasons, snapshot.change.Dependencies)
+		attributions[recordID] = attribution
+		beforePath, beforeFound := historyConcretePathAt(builder.scratch, snapshot.change.Path)
+		if beforeFound {
+			observeResolutionSubtree(builder, snapshot.before, beforePath, snapshot.change.Path, ref, attribution, true, false)
+		}
+	}
+	for _, group := range payload.replayGroups {
+		if group.coveringRecordID < 0 || group.coveringRecordID >= len(payload.snapshots) {
+			continue
+		}
+		covering := payload.snapshots[group.coveringRecordID]
+		replaceHistoryPath(&builder.scratch, group.change.Path, covering.after)
+	}
+	for recordID, snapshot := range payload.snapshots {
+		ref := historyPayloadRef{phase: historyPhaseResolution, id: recordID}
+		afterPath, afterFound := historyConcretePathAt(builder.scratch, snapshot.change.Path)
+		if afterFound {
+			observeResolutionSubtree(builder, snapshot.after, afterPath, snapshot.change.Path, ref, attributions[recordID], false, true)
+		}
+	}
+	builder.model.resolutionPayload = payload
+}
+
+func observeResolutionSubtree(builder *historyBuilder, root *document.Node, path, reportedPath Path, ref historyPayloadRef, attribution historyAttributionRef, before, after bool) {
+	walkDocument(root, path, func(path Path, node *document.Node) {
+		builder.addObservation(historyObservation{
+			path:           path,
+			phase:          historyPhaseResolution,
+			payload:        ref,
+			reportedPath:   reportedPath,
+			role:           historyRoleOperation,
+			beforePresent:  before,
+			afterPresent:   after,
+			sensitive:      node.Sensitive,
+			attributionRef: attribution,
+		})
+	})
+}
+
+func historyConcretePathAt(root *document.Node, path Path) (Path, bool) {
+	if root == nil || !path.valid() {
+		return Path{}, false
+	}
+	current := root
+	var concrete Path
+	for _, segment := range path.segments {
+		switch current.Kind {
+		case document.Mapping:
+			if segment.kind == pathSegmentIndex {
+				return Path{}, false
+			}
+			found := false
+			for _, entry := range current.Mapping {
+				if entry.Key == segment.key {
+					current = entry.Value
+					found = true
+					break
+				}
+			}
+			if !found {
+				return Path{}, false
+			}
+			concrete = concrete.Key(segment.key)
+		case document.Sequence:
+			index, ok := schemaIndex(segment)
+			if !ok || index >= len(current.Sequence) {
+				return Path{}, false
+			}
+			current = current.Sequence[index]
+			concrete = concrete.Index(index)
+		default:
+			return Path{}, false
+		}
+		if current == nil {
+			return Path{}, false
+		}
+	}
+	return concrete, true
+}

internal/engine/history_resolution_test.go

@@ -0,0 +1,94 @@
+package engine
+
+import (
+	"reflect"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
+
+func TestHistoryResolutionRetainsOriginalShapeAttributionBehindReplayGroup(t *testing.T) {
+	before := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+		Key: "0", Value: &document.Node{Kind: document.String, Text: "old"},
+	}}}
+	after := &document.Node{Kind: document.Sequence, Sequence: []*document.Node{{
+		Kind: document.String, Text: "new", Sensitive: true,
+	}}}
+	baseline := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+		Key: "settings", Value: before,
+	}}}
+	parentDependency := Keys("parent-dependency")
+	childDependency := Keys("child-dependency")
+	snapshots := []resolutionSnapshot{
+		{
+			change: Change{
+				Path: Keys("settings"), Origin: Origin{Layer: LayerResolution, Operation: ChangeDerivedDefault},
+				Reasons: []string{"parent normalization"}, Dependencies: []Path{parentDependency},
+			},
+			before: before, after: after,
+		},
+		{
+			change: Change{
+				Path: parseOverrideTestPath(t, "/settings/0"), Origin: Origin{Layer: LayerResolution, Operation: ChangeNormalization},
+				Reasons: []string{"child normalization"}, Dependencies: []Path{childDependency},
+			},
+			before: before.Mapping[0].Value,
+			after:  after.Sequence[0],
+		},
+	}
+
+	builder := newHistoryBuilder()
+	builder.observeBaseline(baseline)
+	builder.setScratch(baseline)
+	builder.observeResolutionEvidence(snapshots)
+	model := builder.freeze()
+
+	if got := model.resolutionPayload.replayGroups; len(got) != 1 ||
+		got[0].coveringRecordID != 0 || !reflect.DeepEqual(got[0].originalRecordIDs, []int{0, 1}) ||
+		!reflect.DeepEqual(got[0].change.Reasons, []string{"parent normalization", "child normalization"}) ||
+		!reflect.DeepEqual(got[0].change.Dependencies, []Path{parentDependency, childDependency}) ||
+		got[0].change.Origin.Operation != ChangeNormalization {
+		t.Fatalf("replay group lost original references or merged annotations: %#v", got)
+	}
+	if len(model.resolutionPayload.snapshots) != 2 ||
+		model.resolutionPayload.snapshots[1].change.Reasons[0] != "child normalization" ||
+		model.resolutionPayload.snapshots[1].change.Dependencies[0].String() != childDependency.String() ||
+		model.resolutionPayload.snapshots[0].before != before || model.resolutionPayload.snapshots[1].after != after.Sequence[0] {
+		t.Fatalf("original resolution records or their roots were not retained: %#v", model.resolutionPayload.snapshots)
+	}
+
+	keyPath := Keys("settings", "0")
+	keySelections := model.selectHistory(keyPath)
+	if len(keySelections) != 3 || !keySelections[0].after || keySelections[1].phase != historyPhaseResolution ||
+		!keySelections[1].before || keySelections[1].after || !keySelections[2].before || keySelections[2].after {
+		t.Fatalf("mapping-key history did not retain its removed side: %#v", keySelections)
+	}
+	if got := keySelections[1].previous; len(got) != 1 || got[0].Layer != LayerDefault {
+		t.Fatalf("removed key previous origin = %#v, want baseline origin", got)
+	}
+
+	indexPath := Keys("settings").Index(0)
+	indexSelections := model.selectHistory(indexPath)
+	if len(indexSelections) != 2 || indexSelections[0].phase != historyPhaseResolution ||
+		!indexSelections[0].after || indexSelections[0].before || !indexSelections[1].after || indexSelections[1].before {
+		t.Fatalf("sequence-index history did not retain introduced sides: %#v", indexSelections)
+	}
+	if len(indexSelections[0].previous) != 0 {
+		t.Fatalf("introduced index inherited a different key's previous origin: %#v", indexSelections[0].previous)
+	}
+	if got := indexSelections[1].attribution; got.origin.Operation != ChangeNormalization ||
+		!reflect.DeepEqual(got.reasons, []string{"child normalization"}) ||
+		!reflect.DeepEqual(got.dependencies, []Path{childDependency}) {
+		t.Fatalf("covered original attribution = %#v", got)
+	}
+
+	foundSensitive := false
+	for _, observation := range model.observations {
+		if observation.phase == historyPhaseResolution && sameConcreteHistoryPath(observation.path, indexPath) && observation.sensitive {
+			foundSensitive = true
+		}
+	}
+	if !foundSensitive {
+		t.Fatal("original resolution subtree sensitivity was omitted from observations")
+	}
+}

internal/engine/load.go

@@ -99,7 +99,7 @@ type Inspection struct {
 	final            *document.Node
 	immutableLookup  *documentLookup
 	resolution       []resolutionSnapshot
-	resolutionReplay []resolutionSnapshot
+	resolutionReplay []resolutionReplayGroup
 	applications     []historyApplication
 	inheritance      []inheritanceOperation
 	secrets          map[string]bool

internal/engine/resolution_index_test.go

@@ -23,23 +23,26 @@ func TestCoalescedResolutionSnapshotsMatchOrderedScan(t *testing.T) {
 	want := coalescedResolutionSnapshotsByScan(snapshots)
 	got := coalescedResolutionSnapshots(snapshots)
 	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("coalesced snapshots differ from ordered scan:\n got: %#v\nwant: %#v", got, want)
+		t.Fatalf("coalesced replay groups differ from ordered scan:\n got: %#v\nwant: %#v", got, want)
 	}
 	if len(got) != 4 || got[0].change.Origin.Operation != ChangeNormalization ||
 		!reflect.DeepEqual(got[0].change.Reasons, []string{"child", "shared", "child", "deep", "normalized"}) ||
-		len(got[0].change.Dependencies) != 2 || got[3].change.Path.String() != "" {
+		len(got[0].change.Dependencies) != 2 || got[3].change.Path.String() != "" ||
-		t.Fatalf("coalesced ordering or unions changed: %#v", got)
+		got[0].coveringRecordID != 0 || !reflect.DeepEqual(got[0].originalRecordIDs, []int{0, 3, 6}) ||
+		!reflect.DeepEqual(got[3].originalRecordIDs, []int{4, 5}) {
+		t.Fatalf("coalesced ordering, references, or unions changed: %#v", got)
 	}
 }
 
-func coalescedResolutionSnapshotsByScan(snapshots []resolutionSnapshot) []resolutionSnapshot {
+func coalescedResolutionSnapshotsByScan(snapshots []resolutionSnapshot) []resolutionReplayGroup {
-	groups := make([]resolutionSnapshot, 0, len(snapshots))
+	groups := make([]resolutionReplayGroup, 0, len(snapshots))
-	for _, snapshot := range snapshots {
+	for recordID, snapshot := range snapshots {
 		merged := false
 		for i := range groups {
 			if !pathContainsByPointer(groups[i].change.Path, snapshot.change.Path) {
 				continue
 			}
+			groups[i].originalRecordIDs = append(groups[i].originalRecordIDs, recordID)
 			for _, reason := range snapshot.change.Reasons {
 				groups[i].change.Reasons = appendUniqueStringByScan(groups[i].change.Reasons, reason)
 			}
@@ -53,8 +56,11 @@ func coalescedResolutionSnapshotsByScan(snapshots []resolutionSnapshot) []resolu
 			break
 		}
 		if !merged {
-			snapshot.change = cloneChange(snapshot.change)
+			groups = append(groups, resolutionReplayGroup{
-			groups = append(groups, snapshot)
+				originalRecordIDs: []int{recordID},
+				coveringRecordID:  recordID,
+				change:            cloneChange(snapshot.change),
+			})
 		}
 	}
 	return groups