Preserve composite ConfigKit provenance

This commit is contained in:
2026-10-04 13:34:27 +00:00
parent 428f435734
commit f1672f6f95
13 changed files with 200 additions and 145 deletions

View File

@@ -20,7 +20,7 @@ type EffectiveArtifactRecord struct {
Dependencies []string
FamilyKey string
CharacterID string
FamilySource string
FamilySources []string
PartySource string
}
@@ -53,7 +53,7 @@ func EffectiveArtifactCatalog(cfg *PipelineConfig, party ResolvedParty) ([]Effec
record.DefinitionSources = artifactFamilyDefinitionSources(cfg, member.Family)
record.FamilyKey = member.Family
record.CharacterID = member.CharacterID
record.FamilySource = member.Source
record.FamilySources = append([]string(nil), cfg.familyCatalog.Families[member.Family].Sources...)
record.PartySource = party.Source.Path
} else {
record.DefinitionSources = artifactDefinitionSources(cfg, key)
@@ -67,60 +67,52 @@ func EffectiveArtifactCatalog(cfg *PipelineConfig, party ResolvedParty) ([]Effec
func artifactDefinitionSources(cfg *PipelineConfig, key string) []string {
return declarationDefinitionSources(cfg,
configkit.Keys("scriptorium", "artifacts", key),
"scriptorium.artifacts."+key,
)
}
func artifactFamilyDefinitionSources(cfg *PipelineConfig, key string) []string {
return declarationDefinitionSources(cfg,
configkit.Keys("scriptorium", "artifact_families", key),
"scriptorium.artifact_families."+key,
)
}
func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Path, prefix string) []string {
func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Path) []string {
if cfg == nil || cfg.resolution == nil {
return nil
}
pathsByID := make(map[string]string, len(cfg.resolution.configkitSources))
for _, source := range cfg.resolution.configkitSources {
path := source.Path
if path == "" {
path = source.Label
}
pathsByID[source.ID] = path
}
pathsByID := pipelineSourcePaths(cfg)
seen := map[string]struct{}{}
for _, change := range cfg.resolution.configkitMetadata.History(declaration) {
if change.Origin.Layer != configkit.LayerBase && change.Origin.Layer != configkit.LayerOverlay {
continue
}
for _, location := range change.Origin.Locations {
source := pathsByID[location.SourceID]
if source == "" {
source = location.SourceID
seen[location.SourceID] = struct{}{}
}
for _, inheritance := range change.Origin.Inheritance {
for _, location := range inheritance.Locations {
seen[location.SourceID] = struct{}{}
}
seen[source] = struct{}{}
}
}
if len(seen) == 0 {
for _, ownership := range cfg.resolution.ownership {
if ownership.path != prefix && !strings.HasPrefix(ownership.path, prefix+".") {
continue
if info, ok := cfg.resolution.configkitMetadata.Lookup(declaration); ok {
for _, location := range info.Origin.Locations {
seen[location.SourceID] = struct{}{}
}
for _, source := range ownership.sources {
if source != pipelineDefaultOwnershipSource && source != "resolution" {
seen[source] = struct{}{}
for _, inheritance := range info.Origin.Inheritance {
for _, location := range inheritance.Locations {
seen[location.SourceID] = struct{}{}
}
}
}
}
if len(seen) == 0 && cfg.resolution.rootPath != "" {
seen[cfg.resolution.rootPath] = struct{}{}
}
result := make([]string, 0, len(seen))
for source := range seen {
for sourceID := range seen {
source := pathsByID[sourceID]
if source == "" {
source = sourceID
}
result = append(result, source)
}
sort.Strings(result)

View File

@@ -83,7 +83,7 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
Members: make([]string, 0, len(characters)),
MemberDependencies: append([]string(nil), family.MemberDependencies...),
Publish: cloneArtifactFamilyPublish(family.Publish),
Source: artifactFamilySource(cfg, familyKey),
Sources: artifactFamilyDefinitionSources(cfg, familyKey),
}
for _, character := range characters {
key := familyKey + "_" + character.ID
@@ -116,7 +116,7 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
}
outputOwners[outputPath] = key
familyOrigin.Members = append(familyOrigin.Members, key)
catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Source: familyOrigin.Source, Dependencies: append([]string(nil), dependencies...), Inputs: inputSourceMap(inputs)}
catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Dependencies: append([]string(nil), dependencies...), Inputs: inputSourceMap(inputs)}
}
catalog.Families[familyKey] = familyOrigin
}
@@ -351,31 +351,6 @@ func sortedFamilyKeys(families map[string]ScriptoriumArtifactFamilyConfig) []str
return keys
}
func artifactFamilySource(cfg *PipelineConfig, family string) string {
if cfg == nil || cfg.resolution == nil {
return ""
}
prefix := "scriptorium.artifact_families." + family + "."
var fallback string
for _, ownership := range cfg.resolution.ownership {
if !strings.HasPrefix(ownership.path, prefix) || len(ownership.sources) == 0 {
continue
}
for _, source := range ownership.sources {
if source != pipelineDefaultOwnershipSource {
return source
}
if fallback == "" {
fallback = source
}
}
}
if fallback != "" {
return fallback
}
return pipelineDefaultOwnershipSource
}
func cloneArtifactDefinitions(in map[string]ScriptoriumArtifactConfig) map[string]ScriptoriumArtifactConfig {
if len(in) == 0 {
return map[string]ScriptoriumArtifactConfig{}
@@ -447,6 +422,7 @@ func cloneArtifactFamilyPublish(in *ScriptoriumArtifactFamilyPublishConfig) *Scr
func cloneArtifactFamilyCatalog(in ArtifactFamilyCatalog) ArtifactFamilyCatalog {
out := ArtifactFamilyCatalog{Families: make(map[string]ArtifactFamilyOrigin, len(in.Families)), Members: make(map[string]ArtifactFamilyMemberOrigin, len(in.Members))}
for key, family := range in.Families {
family.Sources = append([]string(nil), family.Sources...)
family.Members = append([]string(nil), family.Members...)
family.MemberDependencies = append([]string(nil), family.MemberDependencies...)
family.Publish = cloneArtifactFamilyPublish(family.Publish)

View File

@@ -90,13 +90,13 @@ characters:
t.Fatalf("zeta class summary = %#v", got)
}
catalog := ArtifactFamilies(cfg.Pipeline)
if got, want := catalog.Families["character_meta"].Source, absolutePath(t, pipelinePath); got != want {
t.Fatalf("family source = %q, want %q", got, want)
if got, want := catalog.Families["character_meta"].Sources, []string{absolutePath(t, pipelinePath)}; !reflect.DeepEqual(got, want) {
t.Fatalf("family sources = %#v, want %#v", got, want)
}
if got := catalog.Families["character_meta"].Members; !reflect.DeepEqual(got, []string{"character_meta_alpha", "character_meta_zeta"}) {
t.Fatalf("meta members = %#v", got)
}
if got := catalog.Members["character_items_zeta"]; got.Family != "character_items" || got.CharacterID != "zeta" || got.Source == "" {
if got := catalog.Members["character_items_zeta"]; got.Family != "character_items" || got.CharacterID != "zeta" {
t.Fatalf("zeta origin = %#v", got)
}

View File

@@ -277,12 +277,12 @@ type ArtifactFamilyCatalog struct {
Members map[string]ArtifactFamilyMemberOrigin
}
// ArtifactFamilyOrigin records one declared family and its source ownership.
// ArtifactFamilyOrigin records one declared family and its contributing sources.
type ArtifactFamilyOrigin struct {
Members []string
MemberDependencies []string
Publish *ScriptoriumArtifactFamilyPublishConfig
Source string
Sources []string
}
// ArtifactFamilyMemberOrigin identifies the canonical party member that
@@ -290,7 +290,6 @@ type ArtifactFamilyOrigin struct {
type ArtifactFamilyMemberOrigin struct {
Family string
CharacterID string
Source string
Dependencies []string
Inputs map[string]string
}

View File

@@ -5,7 +5,6 @@ import (
"encoding/hex"
"fmt"
"path/filepath"
"sort"
"strings"
)
@@ -44,31 +43,6 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
if cfg == nil || cfg.resolution == nil {
return fmt.Errorf("pipeline resolution metadata is required")
}
declared := make(map[string][]string, len(cfg.resolution.ownership))
for _, ownership := range cfg.resolution.ownership {
declared[ownership.path] = append([]string(nil), ownership.sources...)
}
records, err := projectionValueRecords(projectEffectivePipeline(cfg, true))
if err != nil {
return err
}
ownership := make([]pipelineFieldOwnership, 0, len(records))
for _, record := range records {
sources := declared[record.Path]
if len(sources) == 0 {
sources = []string{pipelineDefaultOwnershipSource}
}
ownership = append(ownership, pipelineFieldOwnership{
path: record.Path, sources: append([]string(nil), sources...),
})
}
for path, sources := range declared {
if strings.HasPrefix(path, "scriptorium.artifact_families.") {
ownership = append(ownership, pipelineFieldOwnership{path: path, sources: append([]string(nil), sources...)})
}
}
sort.Slice(ownership, func(i, j int) bool { return ownership[i].path < ownership[j].path })
cfg.resolution.ownership = ownership
return recomputePipelineEffectiveDigest(cfg)
}

View File

@@ -8,6 +8,7 @@ import (
"strconv"
"strings"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gopkg.in/yaml.v3"
)
@@ -80,7 +81,10 @@ func EffectivePipelineSources(cfg *PipelineConfig, party ResolvedParty) ([]Effec
for _, value := range values {
paths = append(paths, value.Path)
}
owners := effectivePipelineOwners(cfg)
owners, err := effectivePipelineOwners(cfg)
if err != nil {
return nil, err
}
families := ArtifactFamilies(cfg)
publishCount := 0
if cfg.Publish != nil {
@@ -219,13 +223,23 @@ func effectiveYAMLNode(value any) (*yaml.Node, error) {
}
}
func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
func effectivePipelineOwners(cfg *PipelineConfig) (map[string][]string, error) {
owners := make(map[string][]string)
if cfg == nil || cfg.resolution == nil {
return owners
return owners, nil
}
for _, ownership := range cfg.resolution.ownership {
owners[ownership.path] = append([]string(nil), ownership.sources...)
paths, err := pipelineMetadataPaths(cfg)
if err != nil {
return nil, err
}
pathsByID := pipelineSourcePaths(cfg)
for _, value := range paths {
info, ok := cfg.resolution.configkitMetadata.Lookup(configkit.Keys(value.segments...))
sources := []string{pipelineDefaultOwnershipSource}
if ok {
sources = ownershipSources(info.Origin, pathsByID)
}
owners[value.record.Path] = sources
}
if cfg.Scriptorium != nil {
for key, artifact := range cfg.Scriptorium.Artifacts {
@@ -236,7 +250,7 @@ func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
}
}
}
return owners
return owners, nil
}
func artifactDependencyOwnershipPaths(artifactKey string, artifact ScriptoriumArtifactConfig) []string {
@@ -308,8 +322,8 @@ func generatedArtifactSources(path, familyKey string, families ArtifactFamilyCat
if len(sources) != 0 {
return sources
}
if family, ok := families.Families[familyKey]; ok && family.Source != "" {
return []string{family.Source}
if family, ok := families.Families[familyKey]; ok && len(family.Sources) != 0 {
return append([]string(nil), family.Sources...)
}
return []string{pipelineDefaultOwnershipSource}
}
@@ -369,8 +383,8 @@ func appendGeneratedPublishSources(records []EffectivePipelineSourceRecord, path
familyPrefix := "scriptorium.artifact_families." + familyKey + ".publish."
sources := sourcesForPipelinePaths(owners, []string{familyPrefix + "enabled", familyPrefix + "required", familyPrefix + "dest_pattern"})
if len(sources) == 0 {
if family, ok := families.Families[familyKey]; ok && family.Source != "" {
sources = []string{family.Source}
if family, ok := families.Families[familyKey]; ok && len(family.Sources) != 0 {
sources = append([]string(nil), family.Sources...)
} else {
sources = []string{pipelineDefaultOwnershipSource}
}

View File

@@ -20,7 +20,6 @@ type pipelineResolutionMetadata struct {
logicalNotariusConfig string
logicalNotariusWorking string
logicalNotariusCaptured bool
ownership []pipelineFieldOwnership
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
artifactFamiliesExpanded bool
configkitMetadata configkit.Metadata
@@ -33,11 +32,6 @@ type pipelineOverlaySelection struct {
overlayPath string
}
type pipelineFieldOwnership struct {
path string
sources []string
}
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
overlay := selectedPipelineOverlayName(opts)
result, err := loadPipelineConfigKitResult(path, overlay)
@@ -45,11 +39,7 @@ func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*Pipeline
return nil, describePipelineLoadError(err)
}
cfg := result.Config
metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts)
if err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
cfg.resolution = metadata
cfg.resolution = pipelineMetadataFromConfigKit(result, opts)
return &cfg, nil
}
@@ -257,19 +247,16 @@ func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Valida
}
}
func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) {
func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) *pipelineResolutionMetadata {
sources := result.Sources()
metadata := result.Metadata()
resolved := &pipelineResolutionMetadata{
configkitMetadata: metadata, configkitSources: sources,
configkitMetadata: result.Metadata(), configkitSources: sources,
}
pathsByID := make(map[string]string, len(sources))
for index, source := range sources {
path := source.Path
if path == "" {
path = source.Label
}
pathsByID[source.ID] = path
resolved.sources = append(resolved.sources, path)
if index == 0 {
resolved.rootPath = path
@@ -280,19 +267,7 @@ func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result
resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path}
}
}
detailed, err := pipelineMetadataPaths(cfg)
if err != nil {
return nil, err
}
for _, value := range detailed {
info, ok := metadata.Lookup(configkit.Keys(value.segments...))
sourceNames := []string{pipelineDefaultOwnershipSource}
if ok {
sourceNames = ownershipSources(info.Origin, pathsByID)
}
resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames})
}
return resolved, nil
return resolved
}
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
@@ -300,14 +275,21 @@ func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error
if err != nil {
return nil, err
}
if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 {
if cfg.Scriptorium == nil {
return effective, nil
}
families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}})
families := cfg.Scriptorium.ArtifactFamilies
if cfg.resolution != nil && len(cfg.resolution.artifactFamilies) != 0 {
families = cfg.resolution.artifactFamilies
}
if len(families) == 0 {
return effective, nil
}
detailedFamilies, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": families}})
if err != nil {
return nil, err
}
return append(effective, families...), nil
return append(effective, detailedFamilies...), nil
}
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
@@ -319,7 +301,7 @@ func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []st
}
seen := map[string]struct{}{}
var sources []string
for _, location := range origin.Locations {
add := func(location configkit.Location) {
value := pathsByID[location.SourceID]
if value == "" {
value = location.SourceID
@@ -329,12 +311,35 @@ func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []st
sources = append(sources, value)
}
}
for _, location := range origin.Locations {
add(location)
}
for _, inheritance := range origin.Inheritance {
for _, location := range inheritance.Locations {
add(location)
}
}
if len(sources) == 0 {
return []string{pipelineDefaultOwnershipSource}
}
return sources
}
func pipelineSourcePaths(cfg *PipelineConfig) map[string]string {
paths := make(map[string]string)
if cfg == nil || cfg.resolution == nil {
return paths
}
for _, source := range cfg.resolution.configkitSources {
path := source.Path
if path == "" {
path = source.Label
}
paths[source.ID] = path
}
return paths
}
func intPtr(value int) *int { return &value }
func float64Ptr(value float64) *float64 { return &value }
func boolPtr(value bool) *bool { return &value }