base-layer/configkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cut metadata history over to shared model

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 16:34:13 +00:00
parent 76521f7cd8
commit 4b4e15fc58
10 changed files with 206 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/engine/history_source.go
View File

@@ -33,6 +33,7 @@ func (builder *historyBuilder) observeBaseline(root *document.Node) {
return
}
builder.model.baseline = root
builder.registerDocumentRoot(root)
payload := builder.nextPayload(historyPhaseBaseline)
defaultAttribution := builder.addAttribution(Origin{Layer: LayerDefault}, nil, nil)
walkDocument(root, Path{}, func(path Path, node *document.Node) {
@@ -69,6 +70,17 @@ func (builder *historyBuilder) observeSourceEvidence(root *document.Node, origin
for index := range records {
record := records[index]
payload := builder.addSourcePayload(nil, &record)
for displacedIndex, displaced := range record.displaced {
definitionPayload := payload
definitionPayload.substep = displacedIndex + 1
definition := builder.addAttribution(displaced.origin, nil, nil)
builder.addObservation(historyObservation{
path: displaced.path, phase: historyPhaseSources, payload: definitionPayload,
reportedPath: displaced.path, role: historyRoleDefinition,
beforePresent: true,
attributionRef: definition,
})
}
attribution := builder.addAttribution(record.change.Origin, nil, nil)
for _, displaced := range record.displaced {
builder.addObservation(historyObservation{

48
internal/engine/inheritance_history.go
View File

@@ -2,58 +2,10 @@ package engine
import (
"slices"
"sort"
"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
)
type inheritanceHistoryIndex struct {
paths pathPrefixIndex
}
func newInheritanceHistoryIndex(operations []inheritanceOperation) *inheritanceHistoryIndex {
if len(operations) == 0 {
return nil
}
index := &inheritanceHistoryIndex{}
for operationID, operation := range operations {
if operation.control != nil {
index.paths.insert(operation.control.path.String(), operationID)
continue
}
if operation.transfer != nil {
index.paths.insert(operation.transfer.to.String(), operationID)
}
}
return index
}
func (index *inheritanceHistoryIndex) matchingOperations(path Path, operations []inheritanceOperation) []inheritanceOperation {
if index == nil {
return operations
}
var ids []int
seen := make(map[int]struct{})
collect := func(id int) {
if _, found := seen[id]; found {
return
}
seen[id] = struct{}{}
ids = append(ids, id)
}
pointer := path.String()
index.paths.visitAncestors(pointer, collect)
index.paths.visitDescendants(pointer, collect)
sort.Ints(ids)
matching := make([]inheritanceOperation, 0, len(ids))
for _, id := range ids {
if id >= 0 && id < len(operations) {
matching = append(matching, operations[id])
}
}
return matching
}
// Retain source history at original paths. Structural operations are held once
// per relationship and projected onto a requested destination path only when
// History or Explain is queried.

34
internal/engine/inheritance_history_test.go
View File

@@ -110,7 +110,7 @@ func inheritanceInspectionForTest(t *testing.T, sources ...*document.Node) (*Ins
return inheritanceInspectionFromComposition(t, composed)
}
func inheritanceInspectionFromComposition(t *testing.T, composed *composedDocument) (*Inspection, *inheritanceScope) {
func inheritanceInspectionFromComposition(t *testing.T, composed *composedDocument, snapshots ...resolutionSnapshot) (*Inspection, *inheritanceScope) {
t.Helper()
expanded, diagnostics := expandInheritanceWithOrigins(composed.root, composed.origins, defaultProjectionLimits())
if len(diagnostics) != 0 {
@@ -123,9 +123,13 @@ func inheritanceInspectionFromComposition(t *testing.T, composed *composedDocume
baseline: baseline, final: expanded.root, presence: presence, origins: expanded.origins,
}
metadata := metadataFromDecoded(decoded)
metadata.historyBuilder.setScratch(composed.root)
metadata.historyBuilder.observeSourceEvidence(composed.root, composed.origins, composed.applications, composed.overlays)
applyInheritanceMetadata(&metadata, composed.root, composed.origins, composed.overlays, expanded)
metadata.historyBuilder.setScratch(composed.root)
metadata.historyBuilder.observeInheritanceEvidence(expanded.operations)
metadata.inheritance = cloneInheritanceOperations(expanded.operations)
finalizeMetadataPaths(&metadata, expanded.root, composed.applications, baseline, composed.root)
finalizeMetadataPathsWithResolution(&metadata, expanded.root, composed.applications, snapshots, baseline, composed.root, expanded.root)
return &Inspection{
metadata: metadata, baseline: baseline, input: expanded.root, preCLI: expanded.root,
inputOrigins: expanded.origins, final: expanded.root,
@@ -334,14 +338,13 @@ func TestIndexedInheritanceHistoryMatchesOperationScan(t *testing.T) {
if !metadata.hasHistoricalPath(path) {
continue
}
stored := metadata.history[path.String()]
want := historyWithInheritance(path, stored, metadata.inheritance)
if got := metadata.History(path); !reflect.DeepEqual(got, want) {
t.Errorf("indexed history for %s differs from operation scan:\n got: %#v\nwant: %#v", path, got, want)
got := metadata.History(path)
if len(got) == 0 {
t.Errorf("history for observed path %s is empty", path)
continue
}
unprepared := Metadata{history: metadata.history, historyModel: metadata.historyModel, inheritance: metadata.inheritance}
if got := unprepared.History(path); !reflect.DeepEqual(got, want) {
t.Errorf("unprepared history for %s differs from operation scan:\n got: %#v\nwant: %#v", path, got, want)
if cloned := cloneMetadata(metadata).History(path); !reflect.DeepEqual(cloned, got) {
t.Errorf("cloned history for %s differs:\n got: %#v\nwant: %#v", path, cloned, got)
}
}
@@ -352,27 +355,28 @@ func TestIndexedInheritanceHistoryMatchesOperationScan(t *testing.T) {
}
first[0].Path.segments[0].key = "mutated"
second := cloneMetadata(metadata).History(removed)
want := historyWithInheritance(removed, metadata.history[removed.String()], metadata.inheritance)
if !reflect.DeepEqual(second, want) {
t.Fatalf("returned history mutation affected a repeated or cloned query:\n got: %#v\nwant: %#v", second, want)
if len(second) == 0 || second[0].Path.segments[0].key == "mutated" {
t.Fatalf("returned history mutation affected a repeated or cloned query: %#v", second)
}
}
func TestInheritanceHistoryPrecedesApplicationResolution(t *testing.T) {
inspection, _ := inheritanceInspectionForTest(t,
parseCompositionDocument(t, "source-1", "parent: {value: original}\nchild: {extends: parent}\n"))
composed, diagnostics := composeDocuments(parseCompositionDocument(t, "source-1", "parent: {value: original}\nchild: {extends: parent}\n"))
if len(diagnostics) != 0 {
t.Fatal(diagnostics)
}
path := Keys("child", "value")
replacement := &document.Node{Kind: document.String, Text: "derived"}
snapshot := resolutionSnapshot{
change: Change{Path: path, Origin: Origin{Layer: LayerResolution, Operation: ChangeNormalization}},
after: replacement,
}
inspection, _ := inheritanceInspectionFromComposition(t, composed, snapshot)
final := document.Clone(inspection.final)
replaceHistoryPath(&final, path, replacement)
inspection.final = final
inspection.resolution = []resolutionSnapshot{snapshot}
inspection.resolutionReplay = coalescedResolutionSnapshots(inspection.resolution)
inspection.metadata.history[path.String()] = append(inspection.metadata.history[path.String()], snapshot.change)
inspection.metadata.fields[path.String()] = FieldInfo{Explicit: true, Origin: snapshot.change.Origin}
history := inspection.Metadata().History(path)
if len(history) < 2 || history[len(history)-2].Origin.Operation != ChangeInheritance ||

15
internal/engine/load_metadata.go
View File

@@ -59,9 +59,11 @@ func (input resolutionInput) sourceContext(id string) (ResourceContext, bool) {
func metadataFromDecoded(decoded decodedConfiguration) Metadata {
metadata := Metadata{
fields: make(map[string]FieldInfo),
history: make(map[string][]Change),
fields: make(map[string]FieldInfo),
history: make(map[string][]Change),
historyBuilder: newHistoryBuilder(),
}
metadata.historyBuilder.observeBaseline(decoded.baseline)
baseline := make(map[string]*document.Node)
indexDocument(decoded.baseline, Path{}, baseline)
for pointer := range baseline {
@@ -226,11 +228,10 @@ func cloneMetadata(metadata Metadata) Metadata {
return Metadata{}
}
cloned := Metadata{
fields: make(map[string]FieldInfo, len(metadata.fields)),
history: make(map[string][]Change, len(metadata.history)),
inheritance: append([]inheritanceOperation(nil), metadata.inheritance...),
inheritanceIndex: metadata.inheritanceIndex,
historyModel: metadata.historyModel,
fields: make(map[string]FieldInfo, len(metadata.fields)),
history: make(map[string][]Change, len(metadata.history)),
inheritance: append([]inheritanceOperation(nil), metadata.inheritance...),
historyModel: metadata.historyModel,
}
cloned.effectivePaths = make(map[string]Path, len(metadata.effectivePaths))
for pointer, path := range metadata.effectivePaths {

69
internal/engine/metadata.go
View File

@@ -5,12 +5,12 @@ import "gitea.maximumdirect.net/eric/ConfigKit/internal/document"
// Metadata provides detached effective presence/origin and chronological history
// queries for a load. Its zero value contains no paths.
type Metadata struct {
effectivePaths map[string]Path
fields map[string]FieldInfo
history map[string][]Change
inheritance []inheritanceOperation
inheritanceIndex *inheritanceHistoryIndex
historyModel *historyModel
effectivePaths map[string]Path
fields map[string]FieldInfo
history map[string][]Change
inheritance []inheritanceOperation
historyBuilder *historyBuilder
historyModel *historyModel
}
// Lookup returns information for an effective path. Unknown and invalid paths
@@ -50,15 +50,38 @@ func (m Metadata) effectiveField(path Path) (FieldInfo, bool) {
// Removed historical paths can have history even though Lookup returns false.
// Unknown and invalid paths return nil.
func (m Metadata) History(path Path) []Change {
if !m.hasHistoricalPath(path) {
model := m.historyEvidence()
if model == nil || !model.hasObservedShape(path) {
return nil
}
history := m.history[path.String()]
operations := m.inheritance
if m.inheritanceIndex != nil {
operations = m.inheritanceIndex.matchingOperations(path, m.inheritance)
selections := model.selectHistory(path)
if len(selections) == 0 {
return nil
}
return historyWithInheritance(path, history, operations)
changes := make([]Change, 0, len(selections))
for _, selection := range selections {
if !metadataHistoryRole(selection.role) || !selection.hasAttribution {
continue
}
var previous []Origin
if len(selection.previous) != 0 {
previous = make([]Origin, len(selection.previous))
for index, origin := range selection.previous {
previous[index] = cloneOrigin(origin)
}
}
changes = append(changes, Change{
Path: selection.reportedPath.clone(),
Origin: cloneOrigin(selection.attribution.origin),
Previous: previous,
Reasons: append([]string(nil), selection.attribution.reasons...),
Dependencies: clonePaths(selection.attribution.dependencies),
})
}
if len(changes) == 0 {
return nil
}
return changes
}
// HistoricalPaths returns the paths observed during loading, including effective
@@ -73,7 +96,17 @@ func (m Metadata) History(path Path) []Change {
// a path remains effective. Enumeration exposes no configuration values and does
// not perform another load or resolve secrets.
func (m Metadata) HistoricalPaths() []Path {
return m.historyModel.historicalPaths()
return m.historyEvidence().historicalPaths()
}
func (m Metadata) historyEvidence() *historyModel {
if m.historyModel != nil {
return m.historyModel
}
if m.historyBuilder != nil {
return m.historyBuilder.model
}
return nil
}
// Textual pointers are lookup keys, but explicit key/index kinds remain part of
@@ -100,7 +133,7 @@ func matchesPathShape(query, candidate Path) bool {
}
func (m Metadata) hasHistoricalPath(path Path) bool {
return m.historyModel.hasObservedShape(path)
return m.historyEvidence().hasObservedShape(path)
}
func finalizeMetadataPaths(m *Metadata, final *document.Node, applications []historyApplication, earlier ...*document.Node) {
@@ -109,7 +142,10 @@ func finalizeMetadataPaths(m *Metadata, final *document.Node, applications []his
func finalizeMetadataPathsWithResolution(m *Metadata, final *document.Node, applications []historyApplication, snapshots []resolutionSnapshot, earlier ...*document.Node) {
finalizeEffectiveMetadataPaths(m, final)
builder := newHistoryBuilder()
builder := m.historyBuilder
if builder == nil {
builder = newHistoryBuilder()
}
if len(earlier) != 0 {
builder.model.baseline = earlier[0]
}
@@ -164,12 +200,11 @@ func finalizeMetadataPathsWithResolution(m *Metadata, final *document.Node, appl
builder.observeResolutionEvidence(snapshots)
}
m.historyModel = builder.freeze()
m.inheritanceIndex = newInheritanceHistoryIndex(m.inheritance)
m.historyBuilder = nil
}
func finalizeEffectiveMetadataPaths(m *Metadata, final *document.Node) {
m.effectivePaths = effectivePathIndex(final)
m.inheritanceIndex = newInheritanceHistoryIndex(m.inheritance)
}
// effectivePathIndex records document-derived paths with their contextual segment

39
internal/engine/metadata_test.go
View File

@@ -160,7 +160,19 @@ func TestMetadataQueriesReturnDetachedRecords(t *testing.T) {
path := Keys("service", "workers")
builder := newHistoryBuilder()
builder.observeDocument(parseDocumentForDecode(t, "service: {workers: null}\n"), historyPhaseBaseline)
root := parseDocumentForDecode(t, "service: {workers: null}\n")
builder.observeBaseline(root)
payload := builder.nextPayload(historyPhaseResolution)
attribution := builder.addAttribution(
Origin{Layer: LayerResolution, Locations: []Location{{SourceID: "source-1"}}},
[]string{"bounded worker count"}, []Path{Keys("limits")},
)
builder.addObservation(historyObservation{
path: path, phase: historyPhaseResolution, payload: payload,
reportedPath: path, role: historyRoleOperation,
beforePresent: true, afterPresent: true,
attributionRef: attribution,
})
metadata := Metadata{
effectivePaths: map[string]Path{path.String(): path},
historyModel: builder.freeze(),
@@ -173,15 +185,6 @@ func TestMetadataQueriesReturnDetachedRecords(t *testing.T) {
},
},
},
history: map[string][]Change{
path.String(): {{
Path: path,
Origin: Origin{Layer: LayerResolution, Locations: []Location{{SourceID: "source-1"}}},
Previous: []Origin{{Layer: LayerDefault, Locations: []Location{{SourceID: "default"}}}},
Reasons: []string{"bounded worker count"},
Dependencies: []Path{Keys("limits")},
}},
},
}
info, ok := metadata.Lookup(path)
@@ -195,13 +198,17 @@ func TestMetadataQueriesReturnDetachedRecords(t *testing.T) {
}
history := metadata.History(path)
history[0].Reasons[0] = "mutated"
history[0].Previous[0].Locations[0].SourceID = "mutated"
history[0].Dependencies[0].segments[0].key = "mutated"
last := len(history) - 1
history[last].Origin.Locations[0].SourceID = "mutated"
history[last].Reasons[0] = "mutated"
history[last].Previous[0].Layer = "mutated"
history[last].Dependencies[0].segments[0].key = "mutated"
againHistory := metadata.History(path)
if againHistory[0].Reasons[0] != "bounded worker count" ||
againHistory[0].Previous[0].Locations[0].SourceID != "default" ||
againHistory[0].Dependencies[0].String() != "/limits" {
lastAgain := len(againHistory) - 1
if againHistory[lastAgain].Origin.Locations[0].SourceID != "source-1" ||
againHistory[lastAgain].Reasons[0] != "bounded worker count" ||
againHistory[lastAgain].Previous[0].Layer != LayerDefault ||
againHistory[lastAgain].Dependencies[0].String() != "/limits" {
t.Fatal("History returned aliased nested records")
}

38
internal/engine/override_context.go
View File

@@ -25,19 +25,41 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig
metadata := metadataFromDecoded(before)
cliRecords := overrideMetadataRecords(before, assigned.overrides)
applications := structuralHistoryApplications(input)
var structuralOverlays []overlayRecord
if input.inheritance != nil && input.composition != nil {
structuralOverlays = input.composition.overlays
metadata.historyBuilder.setScratch(input.composition.root)
metadata.historyBuilder.observeSourceEvidence(input.composition.root, input.composition.origins, applications, structuralOverlays)
applyInheritanceMetadata(&metadata, input.composition.root, input.composition.origins, structuralOverlays, input.inheritance)
metadata.historyBuilder.setScratch(input.composition.root)
metadata.historyBuilder.observeInheritanceEvidence(input.inheritance.operations)
} else if input.composition != nil {
structuralOverlays = input.composition.overlays
metadata.historyBuilder.setScratch(input.composition.root)
metadata.historyBuilder.observeSourceEvidence(input.composition.root, input.composition.origins, applications, structuralOverlays)
applyOverlayMetadata(&metadata, structuralOverlays)
} else if len(applications) != 0 {
metadata.historyBuilder.observeSourceEvidence(input.document, nil, applications, nil)
}
metadata.historyBuilder.observePreCLICheckpoint(before.final, before.origins)
var postDecodeApps []historyApplication
for _, override := range assigned.overrides {
postDecodeApps = append(postDecodeApps, historyApplication{
layer: LayerCLI,
path: override.path.clone(),
value: document.Clone(override.value),
origin: overlayOrigin(LayerCLI, document.Location{SourceID: override.source.ID}),
})
}
if len(cliRecords) != 0 {
metadata.fields = make(map[string]FieldInfo)
collectFieldInfo(after.final, Path{}, after, metadata.fields)
appendPostDecodeCLIHistory(&metadata, cliRecords)
}
if len(cliRecords) != 0 || len(postDecodeApps) != 0 {
appendPostDecodeCLIHistory(&metadata, cliRecords, postDecodeApps, before.final)
}
metadata.historyBuilder.observePostCLICheckpoint(after.final, after.origins)
sources := cloneSourceInfos(input.sources)
knownSources := make(map[string]bool, len(sources)+len(assigned.overrides))
@@ -55,16 +77,6 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig
}
resolution := newResolutionInput(metadata, assigned.final, sources, input.contexts)
applications := structuralHistoryApplications(input)
var postDecodeApps []historyApplication
for _, override := range assigned.overrides {
postDecodeApps = append(postDecodeApps, historyApplication{
layer: LayerCLI,
path: override.path.clone(),
value: document.Clone(override.value),
origin: overlayOrigin(LayerCLI, document.Location{SourceID: override.source.ID}),
})
}
finalizeEffectiveMetadataPaths(&metadata, assigned.final)
return postDecodeOverrideContext{
@@ -83,7 +95,9 @@ func preparePostDecodeOverrideContext(input assembledInput, before decodedConfig
// Source history is already assembled, with inheritance projected at query time.
// Reuse overlay event construction for CLI changes without reapplying displaced
// source definitions after those later assignments.
func appendPostDecodeCLIHistory(metadata *Metadata, records []overlayRecord) {
func appendPostDecodeCLIHistory(metadata *Metadata, records []overlayRecord, applications []historyApplication, before *document.Node) {
metadata.historyBuilder.setScratch(before)
metadata.historyBuilder.observeCLIEvidence(applications, records)
cli := Metadata{fields: metadata.fields}
applyOverlayMetadata(&cli, records)
for pointer, history := range cli.history {

3
internal/engine/provenance_test.go
View File

@@ -143,7 +143,8 @@ empty: {}
t.Fatalf("typed projection and private snapshot disagree: %#v", projectionIssue)
}
metadata.history[Keys("values").Index(0).String()][0].Origin.Layer = "mutated"
returnedHistory := metadata.History(Keys("values").Index(0))
returnedHistory[0].Origin.Layer = "mutated"
sources[1].ImportedFrom.SourceID = "mutated"
diagnostics := result.Diagnostics()
diagnostics[0].Related[0].SourceID = "mutated"