Integrate history prefixes into load handoffs

docs/roadmap/implementation.md

@@ -1,6 +1,6 @@
 # Interface Contract Remediation Implementation Plan
 
-Status: in progress. Stages 1–11 are complete; Stages 12–18 remain pending.
+Status: in progress. Stages 1–12 are complete; Stages 13–18 remain pending.
 Baseline: `f6d54a094a2b5ada5748686620659399dce7ebf0`.
 Audience: a Luna coding agent executing the stages sequentially.
 
@@ -582,7 +582,10 @@ the same payload storage with old independent interpretation. Focused check:
 
 Owners: `preparePostDecodeOverrideContext`, `resolveLoadInput`,
 `documentWithMetadataLocations`, `metadataFieldLocation`, and load handoffs.
-Status: pending.
+Status: complete. Validation:
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'Location|Provenance|ResolutionInput|CLISecret|Validation' -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./...`, and `git diff --check`.
 
 1. Thread one builder through `postDecodeOverrideContext` and `resolvedLoad` using
    the lifecycle specified above. Create the pre-resolution prefix once; append

internal/engine/history_model.go

@@ -83,6 +83,7 @@ type historyBuilder struct {
 	model          *historyModel
 	nextPayloadIDs map[historyPhase]int
 	scratch        *document.Node
+	prefixReady    bool
 }
 
 func newHistoryBuilder() *historyBuilder {

internal/engine/load_result.go

@@ -12,6 +12,7 @@ type resolvedLoad[T any] struct {
 	configuration   T
 	decoded         decodedConfiguration
 	metadata        Metadata
+	historyBuilder  *historyBuilder
 	final           *document.Node
 	snapshots       []resolutionSnapshot
 	applications    []historyApplication
@@ -56,9 +57,14 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
 	decoded := prepared.decoded
 	configuration := decoded.value.Interface().(T)
 	metadata := prepared.metadata
+	builder := prepared.historyBuilder
+	if builder == nil {
+		builder = metadata.historyBuilder
+	}
 	final := decoded.final
 	referenceInput := prepared.referenceInput
 	var resolutionSnapshots []resolutionSnapshot
+	refreshReferenceInput := false
 	if definition.Resolve != nil {
 		resolved := runResolution(&configuration, definition.Resolve, schema, decoded, referenceInput, prepared.resolution)
 		diagnostics = append(diagnostics, resolved.diagnostics...)
@@ -67,7 +73,18 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
 		}
 		final = resolved.final
 		resolutionSnapshots = resolved.snapshots
+		if builder != nil && len(resolutionSnapshots) != 0 {
+			builder.observeResolutionEvidence(resolutionSnapshots)
+		}
 		applyResolutionMetadata(&metadata, resolved, prepared.resolution)
+		finalizeEffectiveMetadataPaths(&metadata, final)
+		refreshReferenceInput = true
+	}
+	if builder != nil {
+		builder.model.final = final
+		builder.registerDocumentRoot(final)
+	}
+	if refreshReferenceInput {
 		referenceInput = documentWithMetadataLocations(final, metadata, input.document)
 	}
 	applications := []historyApplication(nil)
@@ -81,7 +98,7 @@ func resolveLoadInput[T any](schema *schemaDescriptor, definition Definition[T],
 		versionEvidence = append(versionEvidence, input.versionState.evidence...)
 	}
 	return resolvedLoad[T]{
-		configuration: configuration, decoded: decoded, metadata: metadata, final: final,
+		configuration: configuration, decoded: decoded, metadata: metadata, historyBuilder: builder, final: final,
 		snapshots: resolutionSnapshots, applications: applications,
 		postDecodeApps: prepared.postDecodeApps, structuralInput: document.Clone(prepared.structuralInput),
 		preCLI: prepared.preCLI, preCLIOrigins: prepared.preCLIOrigins, referenceInput: referenceInput,
@@ -101,6 +118,9 @@ func detachLoadConfiguration[T any](schema *schemaDescriptor, configuration T, m
 
 func prepareInspectionHistory[T any](resolved resolvedLoad[T], counter *retainedNodeCounter) (*Inspection, bool) {
 	metadata := resolved.metadata
+	if resolved.historyBuilder != nil {
+		metadata.historyBuilder = resolved.historyBuilder
+	}
 	metadata.inheritance = nil
 	inspection := &Inspection{
 		metadata:         metadata,

internal/engine/metadata.go

@@ -13,6 +13,16 @@ type Metadata struct {
 	historyModel   *historyModel
 }
 
+func (m Metadata) currentHistoryModel() *historyModel {
+	if m.historyModel != nil {
+		return m.historyModel
+	}
+	if m.historyBuilder != nil {
+		return m.historyBuilder.model
+	}
+	return nil
+}
+
 // Lookup returns information for an effective path. Unknown and invalid paths
 // return false.
 func (m Metadata) Lookup(path Path) (FieldInfo, bool) {
@@ -152,6 +162,21 @@ func finalizeMetadataPaths(m *Metadata, final *document.Node, applications []his
 func finalizeMetadataPathsWithResolution(m *Metadata, final *document.Node, applications []historyApplication, snapshots []resolutionSnapshot, earlier ...*document.Node) {
 	finalizeEffectiveMetadataPaths(m, final)
 	builder := m.historyBuilder
+	if builder != nil && builder.prefixReady {
+		if len(snapshots) != 0 && builder.model.resolutionPayload == nil {
+			if len(earlier) != 0 {
+				builder.setScratch(earlier[len(earlier)-1])
+			}
+			builder.observeResolutionEvidence(snapshots)
+		}
+		if builder.model.final != final {
+			builder.model.final = final
+			builder.registerDocumentRoot(final)
+		}
+		m.historyModel = builder.freeze()
+		m.historyBuilder = nil
+		return
+	}
 	if builder == nil {
 		builder = newHistoryBuilder()
 	}

internal/engine/metadata_locations_test.go

@@ -71,9 +71,68 @@ func TestIndexedNearestLocationsPreserveMissingAndSyntheticCoordinates(t *testin
 	}
 	final := parseDocumentForDecode(t, "value: cli\n")
 	synthetic := Location{SourceID: "cli-1"}
-	metadata := Metadata{fields: map[string]FieldInfo{"/value": {Origin: Origin{Layer: LayerCLI, Locations: []Location{synthetic}}}}}
+	metadata := Metadata{
+		effectivePaths: effectivePathIndex(final),
+		fields:         map[string]FieldInfo{"/value": {Origin: Origin{Layer: LayerCLI, Locations: []Location{synthetic}}}},
+	}
 	located := documentWithMetadataLocations(final, metadata, root)
 	if location := located.Mapping[0].Value.Location; location.SourceID != "cli-1" || location.Line != 0 || location.Column != 0 {
 		t.Fatalf("synthetic CLI location was replaced by fallback: %#v", location)
 	}
 }
+
+func TestMetadataLocationFallbackDoesNotCrossKeyIndexShapes(t *testing.T) {
+	fallback := parseDocumentForDecode(t, "settings:\n  - value:\n      child: input\n")
+	final := parseDocumentForDecode(t, "settings:\n  '0':\n    value: final\n")
+	metadata := Metadata{
+		effectivePaths: effectivePathIndex(final),
+		fields:         map[string]FieldInfo{"/settings/0/value": {Origin: Origin{Layer: LayerDefault}}},
+	}
+	located := documentWithMetadataLocations(final, metadata, fallback)
+	settings, _ := documentAtPath(located, Keys("settings"))
+	value, _ := documentAtPath(located, Keys("settings", "0", "value"))
+	if settings == nil || value == nil || value.Location != settings.Location {
+		t.Fatalf("fallback crossed from a mapping key to an input sequence item: settings %#v, value %#v", settings, value)
+	}
+	inputChild, _ := documentAtPath(fallback, Keys("settings").Index(0).Key("value").Key("child"))
+	if inputChild == nil || value.Location == inputChild.Location {
+		t.Fatalf("fallback fabricated a key location from an index: input child %#v, value %#v", inputChild, value)
+	}
+}
+
+func TestHistoricalLocationSelectionSeparatesNumericKeyAndIndex(t *testing.T) {
+	mappingRoot := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+		Key: "shape", Value: &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+			Key: "0", Value: &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+				Key: "value", Value: &document.Node{Kind: document.String, Text: "mapping", Location: document.Location{SourceID: "mapping", Line: 3, Column: 5}},
+			}}},
+		}}},
+	}}}
+	sequenceRoot := &document.Node{Kind: document.Mapping, Mapping: []document.Entry{{
+		Key: "shape", Value: &document.Node{Kind: document.Sequence, Sequence: []*document.Node{{
+			Kind: document.Mapping, Mapping: []document.Entry{{
+				Key: "value", Value: &document.Node{Kind: document.String, Text: "sequence", Location: document.Location{SourceID: "sequence", Line: 3, Column: 5}},
+			}},
+		}}},
+	}}}
+	builder := newHistoryBuilder()
+	builder.observeSourceEvidence(sequenceRoot, nil, []historyApplication{
+		{layer: LayerBase, value: mappingRoot},
+		{layer: LayerOverlay, value: sequenceRoot},
+	}, nil)
+	model := builder.freeze()
+	metadata := Metadata{historyModel: model}
+	resolutionOrigin := Origin{Layer: LayerResolution, Operation: ChangeNormalization}
+	for _, test := range []struct {
+		path Path
+		want string
+	}{
+		{Keys("shape", "0", "value"), "mapping"},
+		{Keys("shape").Index(0).Key("value"), "sequence"},
+	} {
+		location, ok := metadataFieldLocation(metadata, test.path, resolutionOrigin)
+		if !ok || location.SourceID != test.want {
+			t.Fatalf("historical location for %s = %#v, %v; want %q", test.path, location, ok, test.want)
+		}
+	}
+}

internal/engine/override_context.go

@@ -10,6 +10,7 @@ type postDecodeOverrideContext struct {
 	preCLIOrigins   map[string]Origin
 	decoded         decodedConfiguration
 	metadata        Metadata
+	historyBuilder  *historyBuilder
 	resolution      resolutionInput
 	applications    []historyApplication
 	postDecodeApps  []historyApplication
@@ -60,6 +61,7 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig
 		appendPostDecodeCLIHistory(&metadata, cliRecords, postDecodeApps, before.final)
 	}
 	metadata.historyBuilder.observePostCLICheckpoint(after.final, after.origins)
+	metadata.historyBuilder.prefixReady = true
 
 	sources := cloneSourceInfos(input.sources)
 	knownSources := make(map[string]bool, len(sources)+len(assigned.overrides))
@@ -85,6 +87,7 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig
 		preCLIOrigins:   cloneOrigins(before.origins),
 		decoded:         after,
 		metadata:        metadata,
+		historyBuilder:  metadata.historyBuilder,
 		resolution:      resolution,
 		applications:    applications,
 		postDecodeApps:  postDecodeApps,
@@ -197,13 +200,12 @@ func documentWithMetadataLocations(root *document.Node, metadata Metadata, fallb
 		if lookup == nil {
 			lookup = buildDocumentLookup(fallbackRoot)
 		}
-		return nearestDocumentLocationWithLookup(fallbackRoot, path, lookup)
+		return nearestConcreteDocumentLocationWithLookup(fallbackRoot, path, lookup)
 	}
 	var visit func(*document.Node, Path)
 	visit = func(node *document.Node, path Path) {
-		pointer := path.String()
+		if info, exists := metadata.effectiveField(path); exists {
-		if info, exists := metadata.fields[pointer]; exists {
+			if location, ok := metadataFieldLocation(metadata, path, info.Origin); ok {
-			if location, ok := metadataFieldLocation(metadata, pointer, info.Origin); ok {
 				node.Location = location
 			} else if location, ok := fallbackLocation(path); ok {
 				node.Location = location
@@ -216,8 +218,8 @@ func documentWithMetadataLocations(root *document.Node, metadata Metadata, fallb
 			for index := range node.Mapping {
 				entry := &node.Mapping[index]
 				childPath := path.Key(entry.Key)
-				if info, exists := metadata.fields[childPath.String()]; exists {
+				if info, exists := metadata.effectiveField(childPath); exists {
-					if location, ok := metadataFieldLocation(metadata, childPath.String(), info.Origin); ok {
+					if location, ok := metadataFieldLocation(metadata, childPath, info.Origin); ok {
 						entry.KeyLocation = location
 						entry.ValueLocation = location
 					} else if location, ok := fallbackLocation(childPath); ok {
@@ -240,19 +242,57 @@ func documentWithMetadataLocations(root *document.Node, metadata Metadata, fallb
 	return located
 }
 
-func metadataFieldLocation(metadata Metadata, pointer string, origin Origin) (document.Location, bool) {
+func nearestConcreteDocumentLocationWithLookup(root *document.Node, path Path, lookup *documentLookup) (document.Location, bool) {
+	if root == nil || !path.valid() {
+		return document.Location{}, false
+	}
+	current := root
+	nearest := current.Location
+	for _, segment := range path.segments {
+		switch current.Kind {
+		case document.Mapping:
+			if segment.kind != pathSegmentKey {
+				return nearest, locationKnown(nearest)
+			}
+			position := -1
+			if lookup != nil {
+				position = lookup.find(current, segment.key)
+			} else {
+				for index, entry := range current.Mapping {
+					if entry.Key == segment.key {
+						position = index
+						break
+					}
+				}
+			}
+			if position < 0 || position >= len(current.Mapping) {
+				return nearest, locationKnown(nearest)
+			}
+			current = current.Mapping[position].Value
+		case document.Sequence:
+			if segment.kind != pathSegmentIndex || segment.index < 0 || segment.index >= len(current.Sequence) {
+				return nearest, locationKnown(nearest)
+			}
+			current = current.Sequence[segment.index]
+		default:
+			return nearest, locationKnown(nearest)
+		}
+		if current == nil {
+			return nearest, locationKnown(nearest)
+		}
+		nearest = current.Location
+	}
+	return nearest, locationKnown(nearest)
+}
+
+func metadataFieldLocation(metadata Metadata, path Path, origin Origin) (document.Location, bool) {
 	if location, ok := latestOriginDocumentLocation(origin); ok {
 		return location, true
 	}
-	for index := len(metadata.history[pointer]) - 1; index >= 0; index-- {
+	if model := metadata.currentHistoryModel(); model != nil {
-		change := metadata.history[pointer][index]
+		location, ok := model.latestHistoricalLocation(path)
-		if location, ok := latestOriginDocumentLocation(change.Origin); ok {
+		if ok {
-			return location, true
+			return document.Location{SourceID: location.SourceID, Line: location.Line, Column: location.Column}, true
-		}
-		for prior := len(change.Previous) - 1; prior >= 0; prior-- {
-			if location, ok := latestOriginDocumentLocation(change.Previous[prior]); ok {
-				return location, true
-			}
 		}
 	}
 	return document.Location{}, false

internal/engine/override_context_test.go

@@ -50,6 +50,12 @@ func TestPostDecodeOverrideContextOrdersDecodeCLIAndResolutionEvidence(t *testin
 		t.Fatalf("applyDecodedOverrides() issues = %#v", issues)
 	}
 	prepared := preparePostDecodeOverrideContext(input, decoded, assigned)
+	if prepared.historyBuilder == nil || prepared.historyBuilder != prepared.metadata.historyBuilder || !prepared.historyBuilder.prefixReady {
+		t.Fatal("post-decode handoff did not retain the completed history prefix")
+	}
+	if prepared.resolution.metadata.historyBuilder != nil || prepared.resolution.metadata.historyModel != nil {
+		t.Fatal("resolution entry view borrowed mutable or historical state")
+	}
 	if prepared.structuralInput == structural || prepared.preCLI == decoded.final ||
 		prepared.decoded.final == assigned.final || prepared.referenceInput == assigned.final {
 		t.Fatal("structural, pre-CLI, post-CLI, and located snapshots are not detached")
@@ -78,11 +84,37 @@ func TestPostDecodeOverrideContextOrdersDecodeCLIAndResolutionEvidence(t *testin
 	if info, ok := prepared.resolution.lookup(Keys("count")); !ok || !info.Explicit || info.Origin.Layer != LayerCLI {
 		t.Fatalf("resolution entry count = %#v, %v", info, ok)
 	}
+	countSource := batch.overrides[1].source.ID
+	if location, ok := metadataFieldLocation(prepared.metadata, Keys("count"), Origin{Layer: LayerDefault}); !ok || location.SourceID != countSource {
+		t.Fatalf("pre-resolution historical location = %#v, %v", location, ok)
+	}
 	cliSource := batch.overrides[3].source.ID
 	if source, ok := prepared.resolution.source(cliSource); !ok || source.Layer != LayerCLI || source.Label != "--api-key" {
 		t.Fatalf("resolution CLI source = %#v, %v", source, ok)
 	}
 
+	resolvedContext, resolveDiagnostics := resolveLoadInput(schema, Definition[config]{Resolve: func(value *config, resolution *Resolution) error {
+		value.Count = 6
+		resolution.Record(Keys("count"), ChangeNormalization, "normalize count")
+		return nil
+	}}, input, preparePostDecodeOverrideContext(input, decoded, assigned))
+	if len(resolveDiagnostics) != 0 {
+		t.Fatalf("resolveLoadInput() diagnostics = %#v", resolveDiagnostics)
+	}
+	if resolvedContext.historyBuilder == nil || resolvedContext.historyBuilder != resolvedContext.metadata.historyBuilder ||
+		resolvedContext.historyBuilder.model.resolutionPayload == nil || len(resolvedContext.historyBuilder.model.resolutionPayload.snapshots) != 1 {
+		t.Fatal("resolution evidence was not appended once to the handed-off model")
+	}
+	if location, ok := metadataFieldLocation(resolvedContext.metadata, Keys("count"), Origin{Layer: LayerResolution}); !ok || location.SourceID != countSource {
+		t.Fatalf("post-resolution historical location = %#v, %v", location, ok)
+	}
+	if info, ok := prepared.resolution.lookup(Keys("count")); !ok || !info.Explicit || info.Origin.Layer != LayerCLI {
+		t.Fatalf("resolution entry view changed after resolution: %#v, %v", info, ok)
+	}
+	if value, ok := documentAtPath(resolvedContext.referenceInput, Keys("count")); !ok || value.Location.SourceID != countSource {
+		t.Fatalf("post-resolution reference location = %#v, %v", value, ok)
+	}
+
 	finalizeMetadataPaths(&prepared.metadata, assigned.final, prepared.historyApplications(), decoded.baseline, prepared.structuralInput, prepared.preCLI, assigned.final)
 	inspection := Inspection{
 		metadata: prepared.metadata, baseline: decoded.baseline, input: assigned.final, inputOrigins: assigned.origins,