From 8155465322f322e89205675b8bac74319f79ee09 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 26 Oct 2026 17:25:11 +0000 Subject: [PATCH] Move sensitivity classification to shared history --- docs/roadmap/implementation.md | 6 +- internal/engine/history_select.go | 120 +++++++++++++++++- internal/engine/secret.go | 67 ++++------ internal/engine/secret_classification_test.go | 11 +- 4 files changed, 150 insertions(+), 54 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 492d2ef..cdacaa8 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -610,7 +610,11 @@ Reuse metadata-location, secret-location, CLI, and provenance tests. Focused che ## Stage 13 — Migrate Historical Sensitivity Readers Owners: `secret.go`, `secret_propagation.go`, model candidates/dependencies. -Status: pending. +Status: complete. Validation: +`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -run 'Secret|Sensitive|Redact|Diff' -count=1`, +`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./internal/engine -count=1`, +`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test . -run 'Secret|Sensitive|Redact|Diff' -count=1`, +`GOTOOLCHAIN=local GOPROXY=off GOTELEMETRY=off go test ./...`, and `git diff --check`. 1. Replace reads of metadata history and private inheritance copies with the complete model candidate and dependency services. Preserve original resolution diff --git a/internal/engine/history_select.go b/internal/engine/history_select.go index e90a91f..83646ba 100644 --- a/internal/engine/history_select.go +++ b/internal/engine/history_select.go @@ -3,6 +3,8 @@ package engine import ( "slices" "sort" + + "gitea.maximumdirect.net/eric/ConfigKit/internal/document" ) type historySelection struct { @@ -21,6 +23,7 @@ type historySelection struct { type historySensitivityCandidate struct { path Path sensitive bool + classify bool dependencies []Path } @@ -200,16 +203,123 @@ func (model *historyModel) sensitivityCandidates() []historySensitivityCandidate return nil } candidates := make([]historySensitivityCandidate, 0, len(model.observations)) - for _, observation := range model.observations { - candidate := historySensitivityCandidate{path: observation.path.clone(), sensitive: observation.sensitive} - if attribution, found := model.resolveAttribution(observation.attributionRef); found { - candidate.dependencies = attribution.dependencies + indices := make(map[string][]int) + add := func(path Path, sensitive, classify bool) int { + if !path.valid() { + return -1 } - candidates = append(candidates, candidate) + pointer := path.String() + for _, index := range indices[pointer] { + if sameConcreteHistoryPath(candidates[index].path, path) { + candidates[index].sensitive = candidates[index].sensitive || sensitive + candidates[index].classify = candidates[index].classify || classify + return index + } + } + index := len(candidates) + candidates = append(candidates, historySensitivityCandidate{path: path.clone(), sensitive: sensitive, classify: classify}) + indices[pointer] = append(indices[pointer], index) + return index + } + for _, observation := range model.observations { + concreteCandidate := observation.role == historyRoleControl + if observation.role == historyRoleContribution { + if application, found := model.applicationPayload(observation.payload); found && len(application.sequences) != 0 { + concreteCandidate = true + } + } + index := add(observation.path, observation.sensitive, concreteCandidate) + if parsed, err := ParsePath(observation.path.String()); err == nil { + add(parsed, observation.sensitive, true) + } + if index < 0 { + continue + } + if attribution, found := model.resolveAttribution(observation.attributionRef); found { + for _, dependency := range attribution.dependencies { + if dependency.valid() { + appendDistinctHistoryPath(&candidates[index].dependencies, dependency) + add(dependency, false, true) + } + } + } + } + _, inherited := model.sensitivityDependencies() + for _, link := range inherited { + add(link.target, false, true) + add(link.dependency, false, true) } return candidates } +func appendDistinctHistoryPath(paths *[]Path, path Path) { + for _, existing := range *paths { + if sameConcreteHistoryPath(existing, path) { + return + } + } + *paths = append(*paths, path.clone()) +} + +// sensitivityDependencies returns the original attribution edges and the +// bidirectional links represented by inheritance payloads. It intentionally +// does not derive edges from the public metadata projection, which may omit +// evidence needed to keep historical values redacted. +func (model *historyModel) sensitivityDependencies() (dependencies, inherited []secretDependency) { + if model == nil { + return nil, nil + } + for _, observation := range model.observations { + if observation.phase == historyPhaseInheritance { + continue + } + attribution, found := model.resolveAttribution(observation.attributionRef) + if !found { + continue + } + for _, dependency := range attribution.dependencies { + if observation.reportedPath.valid() && dependency.valid() { + dependencies = append(dependencies, secretDependency{ + target: observation.reportedPath.clone(), dependency: dependency.clone(), + }) + } + } + } + for _, payload := range model.inheritancePayloads { + for _, operation := range payload.operations { + if operation.control != nil { + control := operation.control.path + for _, contributor := range operation.controlOrigin.contributors { + if control.valid() && contributor.source.valid() { + inherited = append(inherited, secretDependency{ + target: control.clone(), dependency: contributor.source.clone(), + }) + } + } + continue + } + if operation.transfer == nil || operation.parent == nil { + continue + } + transfer := operation.transfer + walkDocument(operation.parent, transfer.to, func(destination Path, _ *document.Node) { + relative, ok := relativeHistoryPath(transfer.to, destination) + if !ok { + return + } + source := appendInheritancePath(transfer.from, relative) + if _, found := operation.parentOrigins[source.String()]; !found { + return + } + inherited = append(inherited, secretDependency{ + target: destination.clone(), dependency: source, + }) + }) + } + } + return dependencies, inherited +} + func (model *historyModel) resolveAttribution(ref historyAttributionRef) (historyAttribution, bool) { if model == nil || ref < 0 || int(ref) >= len(model.attributions) { return historyAttribution{}, false diff --git a/internal/engine/secret.go b/internal/engine/secret.go index 0f4bcc1..9f85171 100644 --- a/internal/engine/secret.go +++ b/internal/engine/secret.go @@ -40,58 +40,37 @@ func classifySecretPathsWithEvidence(schema *schemaDescriptor, predicate func(Pa addSecretPath(paths, path.String()) }) } - var dependencies []secretDependency - for pointer, history := range metadata.history { - addSecretPath(paths, pointer) - if path, err := ParsePath(pointer); err == nil { - addCandidate(path) - } - for _, change := range history { - paths[change.Path.String()] = change.Path.clone() - for _, dependency := range change.Dependencies { + secret := make(map[string]bool) + var dependencies, inherited []secretDependency + if model := metadata.currentHistoryModel(); model != nil { + dependencies, inherited = model.sensitivityDependencies() + for _, candidate := range model.sensitivityCandidates() { + pointer := candidate.path.String() + paths[pointer] = candidate.path.clone() + if candidate.classify { + addCandidate(candidate.path) + } + if candidate.sensitive { + secret[pointer] = true + } + for _, dependency := range candidate.dependencies { paths[dependency.String()] = dependency.clone() addCandidate(dependency) - dependencies = append(dependencies, secretDependency{ - target: change.Path.clone(), dependency: dependency.clone(), - }) } } - } - var inherited []secretDependency - for _, operation := range metadata.inheritance { - if operation.control != nil { - control := operation.control.path - paths[control.String()] = control.clone() - addCandidate(control) - for _, contributor := range operation.controlOrigin.contributors { - paths[contributor.source.String()] = contributor.source.clone() - addCandidate(contributor.source) - inherited = append(inherited, secretDependency{target: control.clone(), dependency: contributor.source.clone()}) - } - continue + for _, edge := range dependencies { + paths[edge.target.String()] = edge.target.clone() + paths[edge.dependency.String()] = edge.dependency.clone() + addCandidate(edge.dependency) } - if operation.transfer == nil { - continue - } - for pointer := range operation.parentOrigins { - source, err := ParsePath(pointer) - if err != nil { - continue - } - relative, ok := relativeHistoryPath(operation.transfer.from, source) - if !ok { - continue - } - destination := appendInheritancePath(operation.transfer.to, relative) - paths[pointer] = source - paths[destination.String()] = destination.clone() - addCandidate(source) - addCandidate(destination) - inherited = append(inherited, secretDependency{target: destination, dependency: source}) + for _, edge := range inherited { + paths[edge.target.String()] = edge.target.clone() + paths[edge.dependency.String()] = edge.dependency.clone() + addCandidate(edge.target) + addCandidate(edge.dependency) } } - secret := make(map[string]bool) classifyProjection := func(tree *document.Node) { walkDocument(tree, Path{}, func(path Path, node *document.Node) { pointer := path.String() diff --git a/internal/engine/secret_classification_test.go b/internal/engine/secret_classification_test.go index 1fed508..a4177a9 100644 --- a/internal/engine/secret_classification_test.go +++ b/internal/engine/secret_classification_test.go @@ -40,7 +40,9 @@ func TestSecretClassificationParsesTypedHistoryAfterNumericShapeChanges(t *testi earlier := parseCompositionDocument(t, "earlier", test.earlier) final := parseCompositionDocument(t, "final", test.final) pointer := test.historical.String() - metadata := Metadata{history: map[string][]Change{pointer: {{Path: test.historical}}}} + builder := newHistoryBuilder() + builder.observeDocument(earlier, historyPhaseSources) + metadata := Metadata{historyModel: builder.freeze()} seen := false classified := classifySecretPaths(nil, func(path Path) bool { if path.String() != pointer { @@ -63,9 +65,10 @@ func TestSecretClassificationParsesTypedHistoryAfterNumericShapeChanges(t *testi func TestSecretPredicateReceivesDetachedHistoricalPath(t *testing.T) { path := Keys("records").Key("1") - metadata := Metadata{history: map[string][]Change{ - path.String(): {{Path: path}}, - }} + root := parseCompositionDocument(t, "history", "records: {'1': value}\n") + builder := newHistoryBuilder() + builder.observeDocument(root, historyPhaseSources) + metadata := Metadata{historyModel: builder.freeze()} seen := false classified := classifySecretPaths(nil, func(candidate Path) bool { if candidate.String() != path.String() {