Move sensitivity classification to shared history

docs/roadmap/implementation.md

@@ -610,7 +610,11 @@ Reuse metadata-location, secret-location, CLI, and provenance tests. Focused che
 ## Stage 13 — Migrate Historical Sensitivity Readers
 
 Owners: `secret.go`, `secret_propagation.go`, model candidates/dependencies.
-Status: pending.
+Status: complete. Validation:
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'Secret|Sensitive|Redact|Diff' -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test . -run 'Secret|Sensitive|Redact|Diff' -count=1`,
+`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./...`, and `git diff --check`.
 
 1. Replace reads of metadata history and private inheritance copies with the
    complete model candidate and dependency services. Preserve original resolution

internal/engine/history_select.go

@@ -3,6 +3,8 @@ package engine
 import (
 	"slices"
 	"sort"
+
+	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
 )
 
 type historySelection struct {
@@ -21,6 +23,7 @@ type historySelection struct {
 type historySensitivityCandidate struct {
 	path         Path
 	sensitive    bool
+	classify     bool
 	dependencies []Path
 }
 
@@ -200,16 +203,123 @@ func (model *historyModel) sensitivityCandidates() []historySensitivityCandidate
 		return nil
 	}
 	candidates := make([]historySensitivityCandidate, 0, len(model.observations))
-	for _, observation := range model.observations {
+	indices := make(map[string][]int)
-		candidate := historySensitivityCandidate{path: observation.path.clone(), sensitive: observation.sensitive}
+	add := func(path Path, sensitive, classify bool) int {
-		if attribution, found := model.resolveAttribution(observation.attributionRef); found {
+		if !path.valid() {
-			candidate.dependencies = attribution.dependencies
+			return -1
 		}
-		candidates = append(candidates, candidate)
+		pointer := path.String()
+		for _, index := range indices[pointer] {
+			if sameConcreteHistoryPath(candidates[index].path, path) {
+				candidates[index].sensitive = candidates[index].sensitive || sensitive
+				candidates[index].classify = candidates[index].classify || classify
+				return index
+			}
+		}
+		index := len(candidates)
+		candidates = append(candidates, historySensitivityCandidate{path: path.clone(), sensitive: sensitive, classify: classify})
+		indices[pointer] = append(indices[pointer], index)
+		return index
+	}
+	for _, observation := range model.observations {
+		concreteCandidate := observation.role == historyRoleControl
+		if observation.role == historyRoleContribution {
+			if application, found := model.applicationPayload(observation.payload); found && len(application.sequences) != 0 {
+				concreteCandidate = true
+			}
+		}
+		index := add(observation.path, observation.sensitive, concreteCandidate)
+		if parsed, err := ParsePath(observation.path.String()); err == nil {
+			add(parsed, observation.sensitive, true)
+		}
+		if index < 0 {
+			continue
+		}
+		if attribution, found := model.resolveAttribution(observation.attributionRef); found {
+			for _, dependency := range attribution.dependencies {
+				if dependency.valid() {
+					appendDistinctHistoryPath(&candidates[index].dependencies, dependency)
+					add(dependency, false, true)
+				}
+			}
+		}
+	}
+	_, inherited := model.sensitivityDependencies()
+	for _, link := range inherited {
+		add(link.target, false, true)
+		add(link.dependency, false, true)
 	}
 	return candidates
 }
 
+func appendDistinctHistoryPath(paths *[]Path, path Path) {
+	for _, existing := range *paths {
+		if sameConcreteHistoryPath(existing, path) {
+			return
+		}
+	}
+	*paths = append(*paths, path.clone())
+}
+
+// sensitivityDependencies returns the original attribution edges and the
+// bidirectional links represented by inheritance payloads. It intentionally
+// does not derive edges from the public metadata projection, which may omit
+// evidence needed to keep historical values redacted.
+func (model *historyModel) sensitivityDependencies() (dependencies, inherited []secretDependency) {
+	if model == nil {
+		return nil, nil
+	}
+	for _, observation := range model.observations {
+		if observation.phase == historyPhaseInheritance {
+			continue
+		}
+		attribution, found := model.resolveAttribution(observation.attributionRef)
+		if !found {
+			continue
+		}
+		for _, dependency := range attribution.dependencies {
+			if observation.reportedPath.valid() && dependency.valid() {
+				dependencies = append(dependencies, secretDependency{
+					target: observation.reportedPath.clone(), dependency: dependency.clone(),
+				})
+			}
+		}
+	}
+	for _, payload := range model.inheritancePayloads {
+		for _, operation := range payload.operations {
+			if operation.control != nil {
+				control := operation.control.path
+				for _, contributor := range operation.controlOrigin.contributors {
+					if control.valid() && contributor.source.valid() {
+						inherited = append(inherited, secretDependency{
+							target: control.clone(), dependency: contributor.source.clone(),
+						})
+					}
+				}
+				continue
+			}
+			if operation.transfer == nil || operation.parent == nil {
+				continue
+			}
+			transfer := operation.transfer
+			walkDocument(operation.parent, transfer.to, func(destination Path, _ *document.Node) {
+				relative, ok := relativeHistoryPath(transfer.to, destination)
+				if !ok {
+					return
+				}
+				source := appendInheritancePath(transfer.from, relative)
+				if _, found := operation.parentOrigins[source.String()]; !found {
+					return
+				}
+				inherited = append(inherited, secretDependency{
+					target: destination.clone(), dependency: source,
+				})
+			})
+		}
+	}
+	return dependencies, inherited
+}
+
 func (model *historyModel) resolveAttribution(ref historyAttributionRef) (historyAttribution, bool) {
 	if model == nil || ref < 0 || int(ref) >= len(model.attributions) {
 		return historyAttribution{}, false

internal/engine/secret.go

@@ -40,58 +40,37 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa
 			addSecretPath(paths, path.String())
 		})
 	}
-	var dependencies []secretDependency
+	secret := make(map[string]bool)
-	for pointer, history := range metadata.history {
+	var dependencies, inherited []secretDependency
-		addSecretPath(paths, pointer)
+	if model := metadata.currentHistoryModel(); model != nil {
-		if path, err := ParsePath(pointer); err == nil {
+		dependencies, inherited = model.sensitivityDependencies()
-			addCandidate(path)
+		for _, candidate := range model.sensitivityCandidates() {
-		}
+			pointer := candidate.path.String()
-		for _, change := range history {
+			paths[pointer] = candidate.path.clone()
-			paths[change.Path.String()] = change.Path.clone()
+			if candidate.classify {
-			for _, dependency := range change.Dependencies {
+				addCandidate(candidate.path)
+			}
+			if candidate.sensitive {
+				secret[pointer] = true
+			}
+			for _, dependency := range candidate.dependencies {
 				paths[dependency.String()] = dependency.clone()
 				addCandidate(dependency)
-				dependencies = append(dependencies, secretDependency{
-					target: change.Path.clone(), dependency: dependency.clone(),
-				})
 			}
 		}
-	}
+		for _, edge := range dependencies {
-	var inherited []secretDependency
+			paths[edge.target.String()] = edge.target.clone()
-	for _, operation := range metadata.inheritance {
+			paths[edge.dependency.String()] = edge.dependency.clone()
-		if operation.control != nil {
+			addCandidate(edge.dependency)
-			control := operation.control.path
-			paths[control.String()] = control.clone()
-			addCandidate(control)
-			for _, contributor := range operation.controlOrigin.contributors {
-				paths[contributor.source.String()] = contributor.source.clone()
-				addCandidate(contributor.source)
-				inherited = append(inherited, secretDependency{target: control.clone(), dependency: contributor.source.clone()})
-			}
-			continue
 		}
-		if operation.transfer == nil {
+		for _, edge := range inherited {
-			continue
+			paths[edge.target.String()] = edge.target.clone()
-		}
+			paths[edge.dependency.String()] = edge.dependency.clone()
-		for pointer := range operation.parentOrigins {
+			addCandidate(edge.target)
-			source, err := ParsePath(pointer)
+			addCandidate(edge.dependency)
-			if err != nil {
-				continue
-			}
-			relative, ok := relativeHistoryPath(operation.transfer.from, source)
-			if !ok {
-				continue
-			}
-			destination := appendInheritancePath(operation.transfer.to, relative)
-			paths[pointer] = source
-			paths[destination.String()] = destination.clone()
-			addCandidate(source)
-			addCandidate(destination)
-			inherited = append(inherited, secretDependency{target: destination, dependency: source})
 		}
 	}
 
-	secret := make(map[string]bool)
 	classifyProjection := func(tree *document.Node) {
 		walkDocument(tree, Path{}, func(path Path, node *document.Node) {
 			pointer := path.String()

internal/engine/secret_classification_test.go

@@ -40,7 +40,9 @@ func TestSecretClassificationParsesTypedHistoryAfterNumericShapeChanges(t *testi
 			earlier := parseCompositionDocument(t, "earlier", test.earlier)
 			final := parseCompositionDocument(t, "final", test.final)
 			pointer := test.historical.String()
-			metadata := Metadata{history: map[string][]Change{pointer: {{Path: test.historical}}}}
+			builder := newHistoryBuilder()
+			builder.observeDocument(earlier, historyPhaseSources)
+			metadata := Metadata{historyModel: builder.freeze()}
 			seen := false
 			classified := classifySecretPaths(nil, func(path Path) bool {
 				if path.String() != pointer {
@@ -63,9 +65,10 @@ func TestSecretClassificationParsesTypedHistoryAfterNumericShapeChanges(t *testi
 
 func TestSecretPredicateReceivesDetachedHistoricalPath(t *testing.T) {
 	path := Keys("records").Key("1")
-	metadata := Metadata{history: map[string][]Change{
+	root := parseCompositionDocument(t, "history", "records: {'1': value}\n")
-		path.String(): {{Path: path}},
+	builder := newHistoryBuilder()
-	}}
+	builder.observeDocument(root, historyPhaseSources)
+	metadata := Metadata{historyModel: builder.freeze()}
 	seen := false
 	classified := classifySecretPaths(nil, func(candidate Path) bool {
 		if candidate.String() != path.String() {