Add shape-aware history selection core

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Interface Contract Remediation Implementation Plan
 
-Status: in progress. Stages 1–2 are complete; Stages 3–18 remain pending.
+Status: in progress. Stages 1–3 are complete; Stages 4–18 remain pending.
 Baseline: `f6d54a094a2b5ada5748686620659399dce7ebf0`.
 Audience: a Luna coding agent executing the stages sequentially.
 
@@ -337,7 +337,11 @@ input remains explicit. Focused check:
 ## Stage 3 — Introduce The Shared Evidence And Selection Core
 
 Finding 3 architecture. Owners: new `history_model.go` and `history_select.go`,
-existing path/lookup helpers, focused history-selection tests. Status: pending.
+existing path/lookup helpers, focused history-selection tests. Status: complete.
+Validation:
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'History.*Select|History.*Shape|Path' -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+and `git diff --check`.
 
 1. Implement the selected model, builder, phase/role enums, payload ID references,
    and semantic observation types. Keep phase producers in later stages; do not

internal/engine/history_model.go

@@ -0,0 +1,225 @@
+package engine
+
+import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+
+type historyPhase uint8
+
+const (
+	historyPhaseBaseline historyPhase = iota
+	historyPhaseSources
+	historyPhaseInheritance
+	historyPhasePreCLI
+	historyPhaseCLI
+	historyPhasePostCLI
+	historyPhaseResolution
+)
+
+type historyRole uint8
+
+const (
+	historyRoleDefinition historyRole = iota
+	historyRoleCompositeDefinition
+	historyRoleOperation
+	historyRoleControl
+	historyRoleCheckpoint
+	historyRoleInventory
+)
+
+type historyPayloadRef struct {
+	phase   historyPhase
+	id      int
+	substep int
+}
+
+type historyAttributionRef int
+
+const noHistoryAttribution historyAttributionRef = -1
+
+type historyAttribution struct {
+	origin       Origin
+	reasons      []string
+	dependencies []Path
+}
+
+type historyObservation struct {
+	path           Path
+	phase          historyPhase
+	payload        historyPayloadRef
+	reportedPath   Path
+	role           historyRole
+	beforePresent  bool
+	afterPresent   bool
+	sensitive      bool
+	attributionRef historyAttributionRef
+}
+
+type historyModel struct {
+	observations []historyObservation
+	attributions []historyAttribution
+
+	shapesByPointer       map[string][]Path
+	shapeOrder            []Path
+	observationsByPointer map[string][]int
+	reportedPathIndex     pathPrefixIndex
+
+	frozen bool
+}
+
+// historyBuilder owns the model while loading. Its scratch document is one
+// mutable working projection used to resolve concrete sides of relative
+// producer effects; it is not retained by the published model.
+type historyBuilder struct {
+	model          *historyModel
+	nextPayloadIDs map[historyPhase]int
+	scratch        *document.Node
+}
+
+func newHistoryBuilder() *historyBuilder {
+	return &historyBuilder{
+		model: &historyModel{
+			shapesByPointer:       make(map[string][]Path),
+			observationsByPointer: make(map[string][]int),
+		},
+		nextPayloadIDs: make(map[historyPhase]int),
+	}
+}
+
+func (builder *historyBuilder) nextPayload(phase historyPhase) historyPayloadRef {
+	builder.requireOpen()
+	id := builder.nextPayloadIDs[phase]
+	builder.nextPayloadIDs[phase] = id + 1
+	return historyPayloadRef{phase: phase, id: id}
+}
+
+func (builder *historyBuilder) addAttribution(origin Origin, reasons []string, dependencies []Path) historyAttributionRef {
+	builder.requireOpen()
+	ref := historyAttributionRef(len(builder.model.attributions))
+	builder.model.attributions = append(builder.model.attributions, historyAttribution{
+		origin:       cloneOrigin(origin),
+		reasons:      append([]string(nil), reasons...),
+		dependencies: clonePaths(dependencies),
+	})
+	return ref
+}
+
+func (builder *historyBuilder) addObservation(observation historyObservation) int {
+	builder.requireOpen()
+	if !observation.path.valid() || !concreteHistoryPath(observation.path) || !observation.reportedPath.valid() {
+		return -1
+	}
+	if observation.attributionRef < noHistoryAttribution || int(observation.attributionRef) >= len(builder.model.attributions) {
+		return -1
+	}
+	observation.path = observation.path.clone()
+	observation.reportedPath = observation.reportedPath.clone()
+	observationID := len(builder.model.observations)
+	builder.model.observations = append(builder.model.observations, observation)
+	builder.addShape(observation.path)
+	switch observation.role {
+	case historyRoleDefinition, historyRoleCompositeDefinition, historyRoleCheckpoint:
+		pointer := observation.path.String()
+		builder.model.observationsByPointer[pointer] = append(builder.model.observationsByPointer[pointer], observationID)
+	case historyRoleOperation, historyRoleControl:
+		builder.model.reportedPathIndex.insert(observation.reportedPath.String(), observationID)
+	}
+	return observationID
+}
+
+func (builder *historyBuilder) observeDocument(root *document.Node, phase historyPhase) {
+	builder.requireOpen()
+	if root == nil {
+		return
+	}
+	payload := builder.nextPayload(phase)
+	walkDocument(root, Path{}, func(path Path, node *document.Node) {
+		builder.addObservation(historyObservation{
+			path:           path,
+			phase:          phase,
+			payload:        payload,
+			reportedPath:   path,
+			role:           historyRoleInventory,
+			sensitive:      node.Sensitive,
+			attributionRef: noHistoryAttribution,
+		})
+	})
+}
+
+func (builder *historyBuilder) setScratch(root *document.Node) {
+	builder.requireOpen()
+	builder.scratch = document.Clone(root)
+}
+
+func (builder *historyBuilder) scratchAt(path Path) (*document.Node, bool) {
+	builder.requireOpen()
+	return documentAtPath(builder.scratch, path)
+}
+
+// advanceScratch transfers the next working projection to the builder. Callers
+// use it after applying existing value-replay helpers to the current scratch
+// document, avoiding a retained tree copy for each operation.
+func (builder *historyBuilder) advanceScratch(root *document.Node) {
+	builder.requireOpen()
+	builder.scratch = root
+}
+
+func (builder *historyBuilder) freeze() *historyModel {
+	builder.requireOpen()
+	builder.scratch = nil
+	builder.model.frozen = true
+	model := builder.model
+	builder.model = nil
+	builder.nextPayloadIDs = nil
+	return model
+}
+
+func (builder *historyBuilder) addShape(path Path) {
+	pointer := path.String()
+	for _, existing := range builder.model.shapesByPointer[pointer] {
+		if sameConcreteHistoryPath(existing, path) {
+			return
+		}
+	}
+	owned := path.clone()
+	builder.model.shapesByPointer[pointer] = append(builder.model.shapesByPointer[pointer], owned)
+	builder.model.shapeOrder = append(builder.model.shapeOrder, owned.clone())
+}
+
+func (builder *historyBuilder) requireOpen() {
+	if builder == nil || builder.model == nil || builder.model.frozen {
+		panic("configkit: history builder is no longer active")
+	}
+}
+
+func concreteHistoryPath(path Path) bool {
+	for _, segment := range path.segments {
+		if segment.kind == pathSegmentParsed {
+			return false
+		}
+	}
+	return true
+}
+
+func sameConcreteHistoryPath(left, right Path) bool {
+	if len(left.segments) != len(right.segments) {
+		return false
+	}
+	for index, leftSegment := range left.segments {
+		rightSegment := right.segments[index]
+		if leftSegment.kind != rightSegment.kind {
+			return false
+		}
+		switch leftSegment.kind {
+		case pathSegmentKey:
+			if leftSegment.key != rightSegment.key {
+				return false
+			}
+		case pathSegmentIndex:
+			if leftSegment.index != rightSegment.index {
+				return false
+			}
+		default:
+			return false
+		}
+	}
+	return true
+}

internal/engine/history_select.go

@@ -0,0 +1,158 @@
+package engine
+
+import (
+	"slices"
+	"sort"
+)
+
+type historySelection struct {
+	path           Path
+	phase          historyPhase
+	payload        historyPayloadRef
+	reportedPath   Path
+	role           historyRole
+	before         bool
+	after          bool
+	attribution    historyAttribution
+	hasAttribution bool
+	previous       []Origin
+}
+
+func (model *historyModel) selectHistory(path Path) []historySelection {
+	if model == nil || !path.valid() || !model.hasObservedShape(path) {
+		return nil
+	}
+
+	candidateSet := make(map[int]struct{})
+	for _, observationID := range model.observationsByPointer[path.String()] {
+		candidateSet[observationID] = struct{}{}
+	}
+	model.reportedPathIndex.visitAncestors(path.String(), func(observationID int) {
+		candidateSet[observationID] = struct{}{}
+	})
+	candidates := make([]int, 0, len(candidateSet))
+	for observationID := range candidateSet {
+		candidates = append(candidates, observationID)
+	}
+	slices.Sort(candidates)
+
+	selections := make([]historySelection, 0, len(candidates))
+	selectionByPayload := make(map[historyPayloadRef]int)
+	for _, observationID := range candidates {
+		observation := model.observations[observationID]
+		if observation.role == historyRoleInventory || !matchesPathShape(path, observation.path) {
+			continue
+		}
+		selectionIndex, found := selectionByPayload[observation.payload]
+		if !found {
+			selectionIndex = len(selections)
+			selectionByPayload[observation.payload] = selectionIndex
+			selection := historySelection{
+				path:         path.clone(),
+				phase:        observation.phase,
+				payload:      observation.payload,
+				reportedPath: observation.reportedPath.clone(),
+				role:         observation.role,
+			}
+			if attribution, ok := model.resolveAttribution(observation.attributionRef); ok {
+				selection.attribution = attribution
+				selection.hasAttribution = true
+			}
+			selections = append(selections, selection)
+		}
+		selection := &selections[selectionIndex]
+		selection.before = selection.before || observation.beforePresent
+		selection.after = selection.after || observation.afterPresent
+	}
+	if len(selections) == 0 {
+		return nil
+	}
+
+	var previous *Origin
+	for index := range selections {
+		selection := &selections[index]
+		if selection.hasAttribution && metadataHistoryRole(selection.role) {
+			if previous != nil {
+				selection.previous = []Origin{cloneOrigin(*previous)}
+			}
+			origin := cloneOrigin(selection.attribution.origin)
+			previous = &origin
+		}
+	}
+	return selections
+}
+
+func (model *historyModel) hasObservedShape(path Path) bool {
+	if model == nil || !path.valid() {
+		return false
+	}
+	for _, candidate := range model.shapesByPointer[path.String()] {
+		if matchesPathShape(path, candidate) {
+			return true
+		}
+	}
+	return false
+}
+
+func (model *historyModel) concreteShapes(pointer string) []Path {
+	if model == nil {
+		return nil
+	}
+	shapes := model.shapesByPointer[pointer]
+	cloned := make([]Path, len(shapes))
+	for index, path := range shapes {
+		cloned[index] = path.clone()
+	}
+	return cloned
+}
+
+func (model *historyModel) observedPaths() []Path {
+	if model == nil {
+		return nil
+	}
+	paths := make([]Path, len(model.shapeOrder))
+	for index, path := range model.shapeOrder {
+		paths[index] = path.clone()
+	}
+	return paths
+}
+
+func (model *historyModel) historicalPaths() []Path {
+	if model == nil {
+		return nil
+	}
+	pointers := make([]string, 0, len(model.shapesByPointer))
+	for pointer := range model.shapesByPointer {
+		pointers = append(pointers, pointer)
+	}
+	sort.Strings(pointers)
+	paths := make([]Path, 0, len(pointers))
+	for _, pointer := range pointers {
+		path, err := ParsePath(pointer)
+		if err == nil {
+			paths = append(paths, path)
+		}
+	}
+	return paths
+}
+
+func (model *historyModel) resolveAttribution(ref historyAttributionRef) (historyAttribution, bool) {
+	if model == nil || ref < 0 || int(ref) >= len(model.attributions) {
+		return historyAttribution{}, false
+	}
+	attribution := model.attributions[ref]
+	return historyAttribution{
+		origin:       cloneOrigin(attribution.origin),
+		reasons:      append([]string(nil), attribution.reasons...),
+		dependencies: clonePaths(attribution.dependencies),
+	}, true
+}
+
+func metadataHistoryRole(role historyRole) bool {
+	switch role {
+	case historyRoleDefinition, historyRoleCompositeDefinition, historyRoleOperation, historyRoleControl:
+		return true
+	default:
+		return false
+	}
+}

internal/engine/history_select_test.go

@@ -0,0 +1,184 @@
+package engine
+
+import (
+	"reflect"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
+)
+
+func TestHistorySelectionSeparatesNumericShapesAndGroupsParsedSides(t *testing.T) {
+	t.Parallel()
+
+	builder := newHistoryBuilder()
+	mapping := parseDocumentForDecode(t, "shape: {'0': old}\n")
+	sequence := parseDocumentForDecode(t, "shape: [new]\n")
+	shapeNode, _ := documentAtPath(mapping, Keys("shape"))
+	shapeNode.Sensitive = true
+	builder.observeDocument(mapping, historyPhaseSources)
+	builder.observeDocument(mapping, historyPhaseBaseline)
+	builder.observeDocument(sequence, historyPhaseResolution)
+
+	baseOrigin := Origin{Layer: LayerBase}
+	baseAttribution := builder.addAttribution(baseOrigin, nil, nil)
+	keyPath := Keys("shape", "0")
+	indexPath := Keys("shape").Index(0)
+	definition := builder.nextPayload(historyPhaseSources)
+	builder.addObservation(historyObservation{
+		path: keyPath, phase: historyPhaseSources, payload: definition,
+		reportedPath: keyPath, role: historyRoleDefinition,
+		afterPresent: true, attributionRef: baseAttribution,
+	})
+
+	replacementAttribution := builder.addAttribution(
+		Origin{Layer: LayerResolution, Operation: ChangeNormalization},
+		[]string{"replace dynamic shape"}, []Path{Keys("shape")},
+	)
+	replacement := builder.nextPayload(historyPhaseResolution)
+	for _, observation := range []historyObservation{
+		{
+			path: keyPath, phase: historyPhaseResolution, payload: replacement,
+			reportedPath: Keys("shape"), role: historyRoleOperation,
+			beforePresent: true, attributionRef: replacementAttribution,
+		},
+		{
+			path: indexPath, phase: historyPhaseResolution, payload: replacement,
+			reportedPath: Keys("shape"), role: historyRoleOperation,
+			afterPresent: true, attributionRef: replacementAttribution,
+		},
+	} {
+		if got := builder.addObservation(observation); got < 0 {
+			t.Fatal("failed to add concrete history observation")
+		}
+	}
+
+	model := builder.freeze()
+	shapes := model.concreteShapes("/shape/0")
+	if len(shapes) != 2 || shapes[0].segments[1].kind != pathSegmentKey || shapes[1].segments[1].kind != pathSegmentIndex {
+		t.Fatalf("concrete shapes = %#v", shapes)
+	}
+	if !reflect.DeepEqual(model.observedPaths()[:2], []Path{Path{}, Keys("shape")}) {
+		t.Fatalf("first observed paths = %#v", model.observedPaths()[:2])
+	}
+	historicalPaths := model.historicalPaths()
+	var historicalPointers []string
+	for _, path := range historicalPaths {
+		historicalPointers = append(historicalPointers, path.String())
+	}
+	if !reflect.DeepEqual(historicalPointers, []string{"", "/shape", "/shape/0"}) {
+		t.Fatalf("historical paths = %#v", historicalPointers)
+	}
+	if len(model.concreteShapes("/shape/0")) != 2 {
+		t.Fatalf("repeated inventory added a duplicate concrete shape: %#v", model.concreteShapes("/shape/0"))
+	}
+	foundSensitive := false
+	for _, observation := range model.observations {
+		if observation.role == historyRoleInventory && observation.path.String() == "/shape" && observation.sensitive {
+			foundSensitive = true
+			break
+		}
+	}
+	if !foundSensitive {
+		t.Fatal("sensitive document fact was not retained with inventory evidence")
+	}
+
+	keySelections := model.selectHistory(keyPath)
+	if len(keySelections) != 2 || keySelections[0].role != historyRoleDefinition || keySelections[1].before != true || keySelections[1].after {
+		t.Fatalf("mapping-key selections = %#v", keySelections)
+	}
+	if len(keySelections[1].previous) != 1 || keySelections[1].previous[0].Layer != LayerBase {
+		t.Fatalf("mapping-key previous origins = %#v", keySelections[1].previous)
+	}
+
+	indexSelections := model.selectHistory(indexPath)
+	if len(indexSelections) != 1 || indexSelections[0].before || !indexSelections[0].after || len(indexSelections[0].previous) != 0 {
+		t.Fatalf("sequence-index selections = %#v", indexSelections)
+	}
+
+	parsed, err := ParsePath("/shape/0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	parsedSelections := model.selectHistory(parsed)
+	if len(parsedSelections) != 2 || !parsedSelections[1].before || !parsedSelections[1].after {
+		t.Fatalf("parsed-path selections = %#v", parsedSelections)
+	}
+	if !reflect.DeepEqual(parsedSelections[1].attribution.reasons, []string{"replace dynamic shape"}) ||
+		len(parsedSelections[1].attribution.dependencies) != 1 || parsedSelections[1].attribution.dependencies[0].String() != "/shape" {
+		t.Fatalf("resolved attribution = %#v", parsedSelections[1].attribution)
+	}
+	parsedSelections[1].attribution.reasons[0] = "mutated"
+	parsedSelections[1].attribution.dependencies[0] = Path{}
+	again := model.selectHistory(parsed)
+	if again[1].attribution.reasons[0] != "replace dynamic shape" || again[1].attribution.dependencies[0].String() != "/shape" {
+		t.Fatalf("selection mutation changed model attribution: %#v", again[1].attribution)
+	}
+	if selections := model.selectHistory(Keys("shape").Index(9)); len(selections) != 0 {
+		t.Fatalf("never-observed index selections = %#v", selections)
+	}
+}
+
+func TestHistorySelectionMatchesDeepSidesOfAncestorShapeChange(t *testing.T) {
+	t.Parallel()
+
+	builder := newHistoryBuilder()
+	builder.observeDocument(parseDocumentForDecode(t, "shape: {'0': {leaf: old}}\n"), historyPhaseSources)
+	builder.observeDocument(parseDocumentForDecode(t, "shape: [{leaf: new}]\n"), historyPhaseResolution)
+	base := builder.addAttribution(Origin{Layer: LayerBase}, nil, nil)
+	oldPath := Keys("shape", "0", "leaf")
+	newPath := Keys("shape").Index(0).Key("leaf")
+	definition := builder.nextPayload(historyPhaseSources)
+	builder.addObservation(historyObservation{
+		path: oldPath, phase: historyPhaseSources, payload: definition,
+		reportedPath: oldPath, role: historyRoleDefinition,
+		afterPresent: true, attributionRef: base,
+	})
+	operation := builder.nextPayload(historyPhaseResolution)
+	operationAttribution := builder.addAttribution(Origin{Layer: LayerResolution, Operation: ChangeNormalization}, nil, nil)
+	for _, observation := range []historyObservation{
+		{path: oldPath, phase: historyPhaseResolution, payload: operation, reportedPath: Keys("shape"), role: historyRoleOperation, beforePresent: true, attributionRef: operationAttribution},
+		{path: newPath, phase: historyPhaseResolution, payload: operation, reportedPath: Keys("shape"), role: historyRoleOperation, afterPresent: true, attributionRef: operationAttribution},
+	} {
+		builder.addObservation(observation)
+	}
+	model := builder.freeze()
+
+	newSelections := model.selectHistory(newPath)
+	if len(newSelections) != 1 || newSelections[0].role != historyRoleOperation || newSelections[0].before || !newSelections[0].after || len(newSelections[0].previous) != 0 {
+		t.Fatalf("new descendant selections = %#v", newSelections)
+	}
+	oldSelections := model.selectHistory(oldPath)
+	if len(oldSelections) != 2 || !oldSelections[1].before || oldSelections[1].after || len(oldSelections[1].previous) != 1 || oldSelections[1].previous[0].Layer != LayerBase {
+		t.Fatalf("removed descendant selections = %#v", oldSelections)
+	}
+	if selections := model.selectHistory(Keys("shape").Index(0).Key("missing")); len(selections) != 0 {
+		t.Fatalf("never-observed descendant selections = %#v", selections)
+	}
+}
+
+func TestHistoryBuilderScratchUsesOneDetachedWorkingDocument(t *testing.T) {
+	t.Parallel()
+
+	input := parseDocumentForDecode(t, "shape: {value: original}\n")
+	builder := newHistoryBuilder()
+	builder.setScratch(input)
+	inputValue, _ := documentAtPath(input, Keys("shape", "value"))
+	inputValue.Text = "producer mutation"
+	scratchValue, ok := builder.scratchAt(Keys("shape", "value"))
+	if !ok || scratchValue.Text != "original" {
+		t.Fatalf("scratch value after producer mutation = %#v, %v", scratchValue, ok)
+	}
+
+	next := document.Clone(builder.scratch)
+	nextValue, _ := documentAtPath(next, Keys("shape", "value"))
+	nextValue.Text = "replayed"
+	builder.advanceScratch(next)
+	scratchValue, ok = builder.scratchAt(Keys("shape", "value"))
+	if !ok || scratchValue.Text != "replayed" {
+		t.Fatalf("advanced scratch value = %#v, %v", scratchValue, ok)
+	}
+	model := builder.freeze()
+	if !model.frozen || builder.scratch != nil {
+		t.Fatal("freeze retained mutable scratch state")
+	}
+}