Preserve source paths during field normalization

This commit is contained in:
2026-11-17 20:44:07 +00:00
parent 6da4d070e5
commit 2d1fa57d9b
7 changed files with 80 additions and 59 deletions

View File

@@ -191,6 +191,11 @@ Prompt size: API/handoff migration; no discovery implementation or broad fixture
## Stage 3 — Simplify Recorded Normalization And Preserve Source Bases ## Stage 3 — Simplify Recorded Normalization And Preserve Source Bases
**Complete.** Leaf edits are recorded directly and reference paths are resolved
before key normalization moves their source coordinates. Focused path,
inheritance, discovered-main, and leaf-history tests pass. The full config
suite's remaining override-order expectation is owned by fixture migration.
Goal: use v1.1 field-level auditing instead of old blanket sequence coverage. Goal: use v1.1 field-level auditing instead of old blanket sequence coverage.
Inspect document_normalizer.go, document_paths.go, pipeline_document.go, Inspect document_normalizer.go, document_paths.go, pipeline_document.go,

View File

@@ -1,26 +0,0 @@
package config
import configkit "go.maximumdirect.net/configkit"
// documentNormalizer records leaf provenance and, while visiting a sequence,
// covers its atomic mutation boundary. ConfigKit audits sequences as whole
// values, so a record for only a descendant is insufficient. The outermost
// sequence covers nested sequences as well; unchanged visits record nothing.
type documentNormalizer struct {
*configkit.Resolution
sequence *configkit.Path
}
func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer {
if n.sequence == nil {
n.sequence = &path
}
return n
}
func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) {
n.Resolution.RecordNormalization(path, reason, dependencies...)
if n.sequence != nil {
n.Resolution.RecordNormalization(*n.sequence, reason, append([]configkit.Path{path}, dependencies...)...)
}
}

View File

@@ -2,12 +2,12 @@ package config
import ( import (
"path/filepath" "path/filepath"
"strings"
configkit "go.maximumdirect.net/configkit" configkit "go.maximumdirect.net/configkit"
) )
func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) { func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
resolution := documentNormalizer{Resolution: context}
if document == nil { if document == nil {
return return
} }
@@ -45,13 +45,12 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state) normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
profile.Artifacts[laneID] = lane profile.Artifacts[laneID] = lane
} }
stepResolution := resolution.inSequence(path.Key("steps"))
for index := range profile.Steps { for index := range profile.Steps {
step := &profile.Steps[index] step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index) stepPath := path.Key("steps").Index(index)
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state) normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), resolution, state)
for laneID, lane := range step.Artifacts { for laneID, lane := range step.Artifacts {
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state) normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
step.Artifacts[laneID] = lane step.Artifacts[laneID] = lane
} }
} }
@@ -59,7 +58,7 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
} }
} }
func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) { func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *documentLoadState) {
if state == nil || state.mainDirectory != "" { if state == nil || state.mainDirectory != "" {
return return
} }
@@ -76,13 +75,13 @@ func initializeDocumentMainDirectory(resolution documentNormalizer, state *docum
state.mainDirectory = cleanInvocationDirectory(".") state.mainDirectory = cleanInvocationDirectory(".")
} }
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
if binding != nil { if binding != nil {
normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state) normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state)
} }
} }
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
if lane == nil { if lane == nil {
return return
} }
@@ -91,10 +90,17 @@ func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path config
normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state) normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state)
} }
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
for _, name := range sortedReferenceDocumentKeys(references) { for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name] source := references[name]
sourcePath := path.Key(name).Key("path") sourcePath := path.Key(name).Key("path")
if source.Path != nil && resolution.IsExplicit(sourcePath) {
value := strings.TrimSpace(*source.Path)
if value != *source.Path {
resolution.RecordNormalization(sourcePath, "trimmed reference path")
source.Path = documentValue(value)
}
}
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) { if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
base, ok := documentPathBase(resolution, state, sourcePath) base, ok := documentPathBase(resolution, state, sourcePath)
if !ok { if !ok {
@@ -106,12 +112,12 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa
source.Path = documentValue(value) source.Path = documentValue(value)
} }
} }
references[name] = source
} }
references[name] = source
} }
} }
func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) { func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) { if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
return return
} }
@@ -125,11 +131,11 @@ func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadSta
return return
} }
if *target != nil && **target != value { if *target != nil && **target != value {
configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason) configkit.Normalize(resolution, path, target, documentValue(value), reason)
} }
} }
func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) { func documentPathBase(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path) (string, bool) {
original := path original := path
if state != nil { if state != nil {
original = state.originalPath(path) original = state.originalPath(path)

View File

@@ -27,7 +27,7 @@ func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if err := os.WriteFile(main, []byte(`version: 5 if err := os.WriteFile(main, []byte(`version: 5
configkit: config:
imports: imports:
- path: fragments/pipelines.yml - path: fragments/pipelines.yml
output: {directory: ./output} output: {directory: ./output}
@@ -56,6 +56,47 @@ pipelines:
} }
} }
func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) {
root := t.TempDir()
fragmentDir := filepath.Join(root, "fragments")
if err := os.Mkdir(fragmentDir, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(root, "notarius.yml")
fragment := filepath.Join(fragmentDir, "pipeline.yml")
if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragments/pipeline.yml\n required: true\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fragment, []byte(`pipelines:
session:
input: {module: input}
steps:
- id: " first "
references:
" source ": {path: " ./source.txt "}
artifacts:
result:
extract: {module: " extract "}
`), 0600); err != nil {
t.Fatal(err)
}
loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(main), configkit.SearchOptions{Required: true}), LoadOptions{})
if err != nil {
t.Fatal(err)
}
if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(root, "notarius-output") {
t.Fatalf("discovered main or default directory: %q, %q", loaded.MainSourcePath(), loaded.Config.Output.Directory)
}
step := loaded.Config.Pipelines["session"].Steps[0]
if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") {
t.Fatalf("normalized step = %#v", step)
}
history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(0).Key("artifacts").Key("result").Key("extract").Key("module"))
if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("module leaf history = %#v", history)
}
}
func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) { func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
t.Parallel() t.Parallel()

View File

@@ -391,7 +391,7 @@ func TestInheritedReferencePathsKeepDeclaringSource(t *testing.T) {
} }
main := filepath.Join(root, "notarius.yml") main := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(main, []byte(`version: 5 if err := os.WriteFile(main, []byte(`version: 5
configkit: config:
imports: imports:
- path: parents/pipelines.yml - path: parents/pipelines.yml
- path: children/pipelines.yml - path: children/pipelines.yml
@@ -488,7 +488,7 @@ func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) {
} }
main := filepath.Join(root, "notarius.yml") main := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(main, []byte(`version: 5 if err := os.WriteFile(main, []byte(`version: 5
configkit: config:
imports: imports:
- path: parents/pipelines.yml - path: parents/pipelines.yml
- path: children/pipelines.yml - path: children/pipelines.yml

View File

@@ -16,12 +16,12 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
} }
normalizeDocumentPromptKit(document.PromptKit, resolution) normalizeDocumentPromptKit(document.PromptKit, resolution)
normalizeDocumentState(document, resolution) normalizeDocumentState(document, resolution)
normalizeDocumentPipelines(document, resolution, state)
normalizeDocumentPaths(document, resolution, state) normalizeDocumentPaths(document, resolution, state)
normalizeDocumentPipelines(document, resolution, state)
if document.Concurrency != nil { if document.Concurrency != nil {
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers, document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier", configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
documentNormalizer{Resolution: resolution}, state) resolution, state)
} }
deriveDocumentExtractWorkers(document, resolution, state) deriveDocumentExtractWorkers(document, resolution, state)
return nil return nil

View File

@@ -64,15 +64,13 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu
return return
} }
validateDocumentPipelinePresence(document, resolution) validateDocumentPipelinePresence(document, resolution)
normalizer := documentNormalizer{Resolution: resolution}
pipelinesPath := configkit.Keys("pipelines") pipelinesPath := configkit.Keys("pipelines")
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) { for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id] profile := document.Pipelines[id]
normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state) normalizePipelineDocument(&profile, pipelinesPath.Key(id), resolution, state)
document.Pipelines[id] = profile document.Pipelines[id] = profile
} }
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state) document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", resolution, state)
} }
func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) { func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) {
@@ -176,7 +174,7 @@ func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path confi
rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil) rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
} }
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state) normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
if profile.Chunk != nil { if profile.Chunk != nil {
@@ -192,7 +190,6 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
profile.Artifacts[laneID] = lane profile.Artifacts[laneID] = lane
} }
profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state) profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
resolution = resolution.inSequence(path.Key("steps"))
for index := range profile.Steps { for index := range profile.Steps {
step := &profile.Steps[index] step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index) stepPath := path.Key("steps").Index(index)
@@ -207,7 +204,7 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
} }
} }
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state) normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state)
if lane.Merge != nil { if lane.Merge != nil {
normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state) normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state)
@@ -215,28 +212,26 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
if lane.Normalize != nil { if lane.Normalize != nil {
normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state) normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
} }
resolution = resolution.inSequence(path.Key("validators"))
for index := range lane.Validators { for index := range lane.Validators {
normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution) normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
} }
} }
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) { func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier") normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state) binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
resolution = resolution.inSequence(path.Key("validators"))
for index := range binding.Validators { for index := range binding.Validators {
normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution) normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
} }
} }
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) { func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution *configkit.Resolution) {
normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier") normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile") normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile")
} }
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument { func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) map[string]ReferenceDocument {
for _, name := range sortedReferenceDocumentKeys(references) { for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name] source := references[name]
sourcePath := path.Key(name) sourcePath := path.Key(name)
@@ -250,7 +245,7 @@ func normalizeDocumentReferences(references map[string]ReferenceDocument, path c
return normalizeDocumentMap(references, path, "reference name", resolution, state) return normalizeDocumentMap(references, path, "reference name", resolution, state)
} }
func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) { func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path configkit.Path, target *T, value T, reason string) {
if target == nil || !resolution.IsExplicit(path) { if target == nil || !resolution.IsExplicit(path) {
return return
} }
@@ -260,7 +255,7 @@ func normalizeDocumentValue[T comparable](resolution documentNormalizer, path co
} }
} }
func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) { func normalizePipelineDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) {
if target == nil || *target == nil || !resolution.IsExplicit(path) { if target == nil || *target == nil || !resolution.IsExplicit(path) {
return return
} }
@@ -271,7 +266,7 @@ func normalizePipelineDocumentString(resolution documentNormalizer, path configk
} }
} }
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T { func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution *configkit.Resolution, state *documentLoadState) map[string]T {
if len(values) == 0 { if len(values) == 0 {
return values return values
} }