Preserve source paths during field normalization
This commit is contained in:
@@ -1,26 +0,0 @@
|
||||
package config
|
||||
|
||||
import configkit "go.maximumdirect.net/configkit"
|
||||
|
||||
// documentNormalizer records leaf provenance and, while visiting a sequence,
|
||||
// covers its atomic mutation boundary. ConfigKit audits sequences as whole
|
||||
// values, so a record for only a descendant is insufficient. The outermost
|
||||
// sequence covers nested sequences as well; unchanged visits record nothing.
|
||||
type documentNormalizer struct {
|
||||
*configkit.Resolution
|
||||
sequence *configkit.Path
|
||||
}
|
||||
|
||||
func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer {
|
||||
if n.sequence == nil {
|
||||
n.sequence = &path
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (n documentNormalizer) RecordNormalization(path configkit.Path, reason string, dependencies ...configkit.Path) {
|
||||
n.Resolution.RecordNormalization(path, reason, dependencies...)
|
||||
if n.sequence != nil {
|
||||
n.Resolution.RecordNormalization(*n.sequence, reason, append([]configkit.Path{path}, dependencies...)...)
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,12 @@ package config
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) {
|
||||
resolution := documentNormalizer{Resolution: context}
|
||||
func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
if document == nil {
|
||||
return
|
||||
}
|
||||
@@ -45,13 +45,12 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
|
||||
normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
|
||||
profile.Artifacts[laneID] = lane
|
||||
}
|
||||
stepResolution := resolution.inSequence(path.Key("steps"))
|
||||
for index := range profile.Steps {
|
||||
step := &profile.Steps[index]
|
||||
stepPath := path.Key("steps").Index(index)
|
||||
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state)
|
||||
normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), resolution, state)
|
||||
for laneID, lane := range step.Artifacts {
|
||||
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state)
|
||||
normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
|
||||
step.Artifacts[laneID] = lane
|
||||
}
|
||||
}
|
||||
@@ -59,7 +58,7 @@ func normalizeDocumentPaths(document *Document, context *configkit.Resolution, s
|
||||
}
|
||||
}
|
||||
|
||||
func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) {
|
||||
func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *documentLoadState) {
|
||||
if state == nil || state.mainDirectory != "" {
|
||||
return
|
||||
}
|
||||
@@ -76,13 +75,13 @@ func initializeDocumentMainDirectory(resolution documentNormalizer, state *docum
|
||||
state.mainDirectory = cleanInvocationDirectory(".")
|
||||
}
|
||||
|
||||
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
|
||||
func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
if binding != nil {
|
||||
normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
|
||||
func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
if lane == nil {
|
||||
return
|
||||
}
|
||||
@@ -91,10 +90,17 @@ func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path config
|
||||
normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state)
|
||||
}
|
||||
|
||||
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
|
||||
func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
for _, name := range sortedReferenceDocumentKeys(references) {
|
||||
source := references[name]
|
||||
sourcePath := path.Key(name).Key("path")
|
||||
if source.Path != nil && resolution.IsExplicit(sourcePath) {
|
||||
value := strings.TrimSpace(*source.Path)
|
||||
if value != *source.Path {
|
||||
resolution.RecordNormalization(sourcePath, "trimmed reference path")
|
||||
source.Path = documentValue(value)
|
||||
}
|
||||
}
|
||||
if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
|
||||
base, ok := documentPathBase(resolution, state, sourcePath)
|
||||
if !ok {
|
||||
@@ -106,12 +112,12 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa
|
||||
source.Path = documentValue(value)
|
||||
}
|
||||
}
|
||||
references[name] = source
|
||||
}
|
||||
references[name] = source
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
|
||||
func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
|
||||
if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
|
||||
return
|
||||
}
|
||||
@@ -125,11 +131,11 @@ func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadSta
|
||||
return
|
||||
}
|
||||
if *target != nil && **target != value {
|
||||
configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason)
|
||||
configkit.Normalize(resolution, path, target, documentValue(value), reason)
|
||||
}
|
||||
}
|
||||
|
||||
func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) {
|
||||
func documentPathBase(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path) (string, bool) {
|
||||
original := path
|
||||
if state != nil {
|
||||
original = state.originalPath(path)
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestDocumentPathsUseDeclaringFilesAcrossImports(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
configkit:
|
||||
config:
|
||||
imports:
|
||||
- path: fragments/pipelines.yml
|
||||
output: {directory: ./output}
|
||||
@@ -56,6 +56,47 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveredMainAndStepReferenceNormalization(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fragmentDir := filepath.Join(root, "fragments")
|
||||
if err := os.Mkdir(fragmentDir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
fragment := filepath.Join(fragmentDir, "pipeline.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 5\nconfig:\n imports:\n - path: fragments/pipeline.yml\n required: true\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(fragment, []byte(`pipelines:
|
||||
session:
|
||||
input: {module: input}
|
||||
steps:
|
||||
- id: " first "
|
||||
references:
|
||||
" source ": {path: " ./source.txt "}
|
||||
artifacts:
|
||||
result:
|
||||
extract: {module: " extract "}
|
||||
`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := LoadSource(configkit.SearchConfig("", configkit.SearchPaths(main), configkit.SearchOptions{Required: true}), LoadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.MainSourcePath() != main || loaded.Config.Output.Directory != filepath.Join(root, "notarius-output") {
|
||||
t.Fatalf("discovered main or default directory: %q, %q", loaded.MainSourcePath(), loaded.Config.Output.Directory)
|
||||
}
|
||||
step := loaded.Config.Pipelines["session"].Steps[0]
|
||||
if step.ID != "first" || step.Artifacts["result"].Extract.Module != "extract" || step.References["source"].Path != filepath.Join(fragmentDir, "source.txt") {
|
||||
t.Fatalf("normalized step = %#v", step)
|
||||
}
|
||||
history := loaded.result.Metadata().History(configkit.Keys("pipelines", "session", "steps").Index(0).Key("artifacts").Key("result").Key("extract").Key("module"))
|
||||
if len(history) < 2 || history[len(history)-1].Origin.Operation != configkit.ChangeNormalization {
|
||||
t.Fatalf("module leaf history = %#v", history)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -391,7 +391,7 @@ func TestInheritedReferencePathsKeepDeclaringSource(t *testing.T) {
|
||||
}
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
configkit:
|
||||
config:
|
||||
imports:
|
||||
- path: parents/pipelines.yml
|
||||
- path: children/pipelines.yml
|
||||
@@ -488,7 +488,7 @@ func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) {
|
||||
}
|
||||
main := filepath.Join(root, "notarius.yml")
|
||||
if err := os.WriteFile(main, []byte(`version: 5
|
||||
configkit:
|
||||
config:
|
||||
imports:
|
||||
- path: parents/pipelines.yml
|
||||
- path: children/pipelines.yml
|
||||
|
||||
@@ -16,12 +16,12 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
|
||||
}
|
||||
normalizeDocumentPromptKit(document.PromptKit, resolution)
|
||||
normalizeDocumentState(document, resolution)
|
||||
normalizeDocumentPipelines(document, resolution, state)
|
||||
normalizeDocumentPaths(document, resolution, state)
|
||||
normalizeDocumentPipelines(document, resolution, state)
|
||||
if document.Concurrency != nil {
|
||||
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
|
||||
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
|
||||
documentNormalizer{Resolution: resolution}, state)
|
||||
resolution, state)
|
||||
}
|
||||
deriveDocumentExtractWorkers(document, resolution, state)
|
||||
return nil
|
||||
|
||||
@@ -64,15 +64,13 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu
|
||||
return
|
||||
}
|
||||
validateDocumentPipelinePresence(document, resolution)
|
||||
normalizer := documentNormalizer{Resolution: resolution}
|
||||
|
||||
pipelinesPath := configkit.Keys("pipelines")
|
||||
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
|
||||
profile := document.Pipelines[id]
|
||||
normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state)
|
||||
normalizePipelineDocument(&profile, pipelinesPath.Key(id), resolution, state)
|
||||
document.Pipelines[id] = profile
|
||||
}
|
||||
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state)
|
||||
document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", resolution, state)
|
||||
}
|
||||
|
||||
func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) {
|
||||
@@ -176,7 +174,7 @@ func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path confi
|
||||
rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
|
||||
}
|
||||
|
||||
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
|
||||
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
|
||||
normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
|
||||
if profile.Chunk != nil {
|
||||
@@ -192,7 +190,6 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
|
||||
profile.Artifacts[laneID] = lane
|
||||
}
|
||||
profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
|
||||
resolution = resolution.inSequence(path.Key("steps"))
|
||||
for index := range profile.Steps {
|
||||
step := &profile.Steps[index]
|
||||
stepPath := path.Key("steps").Index(index)
|
||||
@@ -207,7 +204,7 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
|
||||
func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state)
|
||||
if lane.Merge != nil {
|
||||
normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state)
|
||||
@@ -215,28 +212,26 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
|
||||
if lane.Normalize != nil {
|
||||
normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
|
||||
}
|
||||
resolution = resolution.inSequence(path.Key("validators"))
|
||||
for index := range lane.Validators {
|
||||
normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
|
||||
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
|
||||
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
|
||||
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
|
||||
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
|
||||
resolution = resolution.inSequence(path.Key("validators"))
|
||||
for index := range binding.Validators {
|
||||
normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) {
|
||||
func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution *configkit.Resolution) {
|
||||
normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier")
|
||||
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile")
|
||||
}
|
||||
|
||||
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument {
|
||||
func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) map[string]ReferenceDocument {
|
||||
for _, name := range sortedReferenceDocumentKeys(references) {
|
||||
source := references[name]
|
||||
sourcePath := path.Key(name)
|
||||
@@ -250,7 +245,7 @@ func normalizeDocumentReferences(references map[string]ReferenceDocument, path c
|
||||
return normalizeDocumentMap(references, path, "reference name", resolution, state)
|
||||
}
|
||||
|
||||
func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) {
|
||||
func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path configkit.Path, target *T, value T, reason string) {
|
||||
if target == nil || !resolution.IsExplicit(path) {
|
||||
return
|
||||
}
|
||||
@@ -260,7 +255,7 @@ func normalizeDocumentValue[T comparable](resolution documentNormalizer, path co
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) {
|
||||
func normalizePipelineDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) {
|
||||
if target == nil || *target == nil || !resolution.IsExplicit(path) {
|
||||
return
|
||||
}
|
||||
@@ -271,7 +266,7 @@ func normalizePipelineDocumentString(resolution documentNormalizer, path configk
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T {
|
||||
func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution *configkit.Resolution, state *documentLoadState) map[string]T {
|
||||
if len(values) == 0 {
|
||||
return values
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user