Preserve source paths during field normalization

This commit is contained in:
2026-11-17 20:44:07 +00:00
parent 6da4d070e5
commit 2d1fa57d9b
7 changed files with 80 additions and 59 deletions

View File

@@ -1,26 +0,0 @@
package config
import configkit "go.maximumdirect.net/configkit"
// documentNormalizer records leaf provenance and, while visiting a sequence,
// covers its atomic mutation boundary. ConfigKit audits sequences as whole
// values, so a record for only a descendant is insufficient. The outermost
// sequence covers nested sequences as well; unchanged visits record nothing.
type documentNormalizer struct {
*configkit.Resolution
sequence *configkit.Path
}
func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer {
if n.sequence == nil {
n.sequence = &path
}
return n
}
func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) {
n.Resolution.RecordNormalization(path, reason, dependencies...)
if n.sequence != nil {
n.Resolution.RecordNormalization(*n.sequence, reason, append([]configkit.Path{path}, dependencies...)...)
}
}

View File

@@ -2,12 +2,12 @@ package config
import (
"path/filepath"
"strings"
configkit "go.maximumdirect.net/configkit"
)
func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) {
resolution := documentNormalizer{Resolution: context}
func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
if document == nil {
return
}
@@ -45,13 +45,12 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
profile.Artifacts[laneID] = lane
}
stepResolution := resolution.inSequence(path.Key("steps"))
for index := range profile.Steps {
step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index)
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state)
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), resolution, state)
for laneID, lane := range step.Artifacts {
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state)
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
step.Artifacts[laneID] = lane
}
}
@@ -59,7 +58,7 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
}
}
func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) {
func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *documentLoadState) {
if state == nil || state.mainDirectory != "" {
return
}
@@ -76,13 +75,13 @@ func initializeDocumentMainDirectory(resolution documentNormalizer, state *docum
state.mainDirectory = cleanInvocationDirectory(".")
}
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
if binding != nil {
normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state)
}
}
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
if lane == nil {
return
}
@@ -91,10 +90,17 @@ func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path config
normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state)
}
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name).Key("path")
if source.Path != nil && resolution.IsExplicit(sourcePath) {
value := strings.TrimSpace(*source.Path)
if value != *source.Path {
resolution.RecordNormalization(sourcePath, "trimmed reference path")
source.Path = documentValue(value)
}
}
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
base, ok := documentPathBase(resolution, state, sourcePath)
if !ok {
@@ -106,12 +112,12 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa
source.Path = documentValue(value)
}
}
references[name] = source
}
references[name] = source
}
}
func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
return
}
@@ -125,11 +131,11 @@ func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadSta
return
}
if *target != nil && **target != value {
configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason)
configkit.Normalize(resolution, path, target, documentValue(value), reason)
}
}
func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) {
func documentPathBase(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path) (string, bool) {
original := path
if state != nil {
original = state.originalPath(path)

View File

@@ -27,7 +27,7 @@ func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
t.Fatal(err)
}
if err := os.WriteFile(main, []byte(`version: 5
configkit:
config:
imports:
- path: fragments/pipelines.yml
output: {directory: ./output}
@@ -56,6 +56,47 @@ pipelines:
}
}
func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) {
root := t.TempDir()
fragmentDir := filepath.Join(root, "fragments")
if err := os.Mkdir(fragmentDir, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(root, "notarius.yml")
fragment := filepath.Join(fragmentDir, "pipeline.yml")
if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragments/pipeline.yml\n required: true\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fragment, []byte(`pipelines:
session:
input: {module: input}
steps:
- id: " first "
references:
" source ": {path: " ./source.txt "}
artifacts:
result:
extract: {module: " extract "}
`), 0600); err != nil {
t.Fatal(err)
}
loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(main), configkit.SearchOptions{Required: true}), LoadOptions{})
if err != nil {
t.Fatal(err)
}
if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(root, "notarius-output") {
t.Fatalf("discovered main or default directory: %q, %q", loaded.MainSourcePath(), loaded.Config.Output.Directory)
}
step := loaded.Config.Pipelines["session"].Steps[0]
if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") {
t.Fatalf("normalized step = %#v", step)
}
history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(0).Key("artifacts").Key("result").Key("extract").Key("module"))
if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("module leaf history = %#v", history)
}
}
func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
t.Parallel()

View File

@@ -391,7 +391,7 @@ func TestInheritedReferencePathsKeepDeclaringSource(t *testing.T) {
}
main := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(main, []byte(`version: 5
configkit:
config:
imports:
- path: parents/pipelines.yml
- path: children/pipelines.yml
@@ -488,7 +488,7 @@ func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) {
}
main := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(main, []byte(`version: 5
configkit:
config:
imports:
- path: parents/pipelines.yml
- path: children/pipelines.yml

View File

@@ -16,12 +16,12 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
}
normalizeDocumentPromptKit(document.PromptKit, resolution)
normalizeDocumentState(document, resolution)
normalizeDocumentPipelines(document, resolution, state)
normalizeDocumentPaths(document, resolution, state)
normalizeDocumentPipelines(document, resolution, state)
if document.Concurrency != nil {
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
documentNormalizer{Resolution: resolution}, state)
resolution, state)
}
deriveDocumentExtractWorkers(document, resolution, state)
return nil

View File

@@ -64,15 +64,13 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu
return
}
validateDocumentPipelinePresence(document, resolution)
normalizer := documentNormalizer{Resolution: resolution}
pipelinesPath := configkit.Keys("pipelines")
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]
normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state)
normalizePipelineDocument(&profile, pipelinesPath.Key(id), resolution, state)
document.Pipelines[id] = profile
}
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state)
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", resolution, state)
}
func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) {
@@ -176,7 +174,7 @@ func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path confi
rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
}
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
if profile.Chunk != nil {
@@ -192,7 +190,6 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
profile.Artifacts[laneID] = lane
}
profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
resolution = resolution.inSequence(path.Key("steps"))
for index := range profile.Steps {
step := &profile.Steps[index]
stepPath := path.Key("steps").Index(index)
@@ -207,7 +204,7 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
}
}
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state)
if lane.Merge != nil {
normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state)
@@ -215,28 +212,26 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
if lane.Normalize != nil {
normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
}
resolution = resolution.inSequence(path.Key("validators"))
for index := range lane.Validators {
normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
}
}
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
resolution = resolution.inSequence(path.Key("validators"))
for index := range binding.Validators {
normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
}
}
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) {
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution *configkit.Resolution) {
normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile")
}
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument {
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) map[string]ReferenceDocument {
for _, name := range sortedReferenceDocumentKeys(references) {
source := references[name]
sourcePath := path.Key(name)
@@ -250,7 +245,7 @@ func normalizeDocumentReferences(references map[string]ReferenceDocument, path c
return normalizeDocumentMap(references, path, "reference name", resolution, state)
}
func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) {
func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path configkit.Path, target *T, value T, reason string) {
if target == nil || !resolution.IsExplicit(path) {
return
}
@@ -260,7 +255,7 @@ func normalizeDocumentValue[T comparable](resolution documentNormalizer, path co
}
}
func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) {
func normalizePipelineDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) {
if target == nil || *target == nil || !resolution.IsExplicit(path) {
return
}
@@ -271,7 +266,7 @@ func normalizePipelineDocumentString(resolution documentNormalizer, path configk
}
}
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T {
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution *configkit.Resolution, state *documentLoadState) map[string]T {
if len(values) == 0 {
return values
}