Preserve composite ConfigKit provenance
This commit is contained in:
@@ -20,7 +20,7 @@ type EffectiveArtifactRecord struct {
|
||||
Dependencies []string
|
||||
FamilyKey string
|
||||
CharacterID string
|
||||
FamilySource string
|
||||
FamilySources []string
|
||||
PartySource string
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ func EffectiveArtifactCatalog(cfg *PipelineConfig, party ResolvedParty) ([]Effec
|
||||
record.DefinitionSources = artifactFamilyDefinitionSources(cfg, member.Family)
|
||||
record.FamilyKey = member.Family
|
||||
record.CharacterID = member.CharacterID
|
||||
record.FamilySource = member.Source
|
||||
record.FamilySources = append([]string(nil), cfg.familyCatalog.Families[member.Family].Sources...)
|
||||
record.PartySource = party.Source.Path
|
||||
} else {
|
||||
record.DefinitionSources = artifactDefinitionSources(cfg, key)
|
||||
@@ -67,60 +67,52 @@ func EffectiveArtifactCatalog(cfg *PipelineConfig, party ResolvedParty) ([]Effec
|
||||
func artifactDefinitionSources(cfg *PipelineConfig, key string) []string {
|
||||
return declarationDefinitionSources(cfg,
|
||||
configkit.Keys("scriptorium", "artifacts", key),
|
||||
"scriptorium.artifacts."+key,
|
||||
)
|
||||
}
|
||||
|
||||
func artifactFamilyDefinitionSources(cfg *PipelineConfig, key string) []string {
|
||||
return declarationDefinitionSources(cfg,
|
||||
configkit.Keys("scriptorium", "artifact_families", key),
|
||||
"scriptorium.artifact_families."+key,
|
||||
)
|
||||
}
|
||||
|
||||
func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Path, prefix string) []string {
|
||||
func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Path) []string {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return nil
|
||||
}
|
||||
pathsByID := make(map[string]string, len(cfg.resolution.configkitSources))
|
||||
for _, source := range cfg.resolution.configkitSources {
|
||||
path := source.Path
|
||||
if path == "" {
|
||||
path = source.Label
|
||||
}
|
||||
pathsByID[source.ID] = path
|
||||
}
|
||||
|
||||
pathsByID := pipelineSourcePaths(cfg)
|
||||
seen := map[string]struct{}{}
|
||||
for _, change := range cfg.resolution.configkitMetadata.History(declaration) {
|
||||
if change.Origin.Layer != configkit.LayerBase && change.Origin.Layer != configkit.LayerOverlay {
|
||||
continue
|
||||
}
|
||||
for _, location := range change.Origin.Locations {
|
||||
source := pathsByID[location.SourceID]
|
||||
if source == "" {
|
||||
source = location.SourceID
|
||||
seen[location.SourceID] = struct{}{}
|
||||
}
|
||||
for _, inheritance := range change.Origin.Inheritance {
|
||||
for _, location := range inheritance.Locations {
|
||||
seen[location.SourceID] = struct{}{}
|
||||
}
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
if ownership.path != prefix && !strings.HasPrefix(ownership.path, prefix+".") {
|
||||
continue
|
||||
if info, ok := cfg.resolution.configkitMetadata.Lookup(declaration); ok {
|
||||
for _, location := range info.Origin.Locations {
|
||||
seen[location.SourceID] = struct{}{}
|
||||
}
|
||||
for _, source := range ownership.sources {
|
||||
if source != pipelineDefaultOwnershipSource && source != "resolution" {
|
||||
seen[source] = struct{}{}
|
||||
for _, inheritance := range info.Origin.Inheritance {
|
||||
for _, location := range inheritance.Locations {
|
||||
seen[location.SourceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 && cfg.resolution.rootPath != "" {
|
||||
seen[cfg.resolution.rootPath] = struct{}{}
|
||||
}
|
||||
result := make([]string, 0, len(seen))
|
||||
for source := range seen {
|
||||
for sourceID := range seen {
|
||||
source := pathsByID[sourceID]
|
||||
if source == "" {
|
||||
source = sourceID
|
||||
}
|
||||
result = append(result, source)
|
||||
}
|
||||
sort.Strings(result)
|
||||
|
||||
@@ -83,7 +83,7 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
|
||||
Members: make([]string, 0, len(characters)),
|
||||
MemberDependencies: append([]string(nil), family.MemberDependencies...),
|
||||
Publish: cloneArtifactFamilyPublish(family.Publish),
|
||||
Source: artifactFamilySource(cfg, familyKey),
|
||||
Sources: artifactFamilyDefinitionSources(cfg, familyKey),
|
||||
}
|
||||
for _, character := range characters {
|
||||
key := familyKey + "_" + character.ID
|
||||
@@ -116,7 +116,7 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
|
||||
}
|
||||
outputOwners[outputPath] = key
|
||||
familyOrigin.Members = append(familyOrigin.Members, key)
|
||||
catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Source: familyOrigin.Source, Dependencies: append([]string(nil), dependencies...), Inputs: inputSourceMap(inputs)}
|
||||
catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Dependencies: append([]string(nil), dependencies...), Inputs: inputSourceMap(inputs)}
|
||||
}
|
||||
catalog.Families[familyKey] = familyOrigin
|
||||
}
|
||||
@@ -351,31 +351,6 @@ func sortedFamilyKeys(families map[string]ScriptoriumArtifactFamilyConfig) []str
|
||||
return keys
|
||||
}
|
||||
|
||||
func artifactFamilySource(cfg *PipelineConfig, family string) string {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return ""
|
||||
}
|
||||
prefix := "scriptorium.artifact_families." + family + "."
|
||||
var fallback string
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
if !strings.HasPrefix(ownership.path, prefix) || len(ownership.sources) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, source := range ownership.sources {
|
||||
if source != pipelineDefaultOwnershipSource {
|
||||
return source
|
||||
}
|
||||
if fallback == "" {
|
||||
fallback = source
|
||||
}
|
||||
}
|
||||
}
|
||||
if fallback != "" {
|
||||
return fallback
|
||||
}
|
||||
return pipelineDefaultOwnershipSource
|
||||
}
|
||||
|
||||
func cloneArtifactDefinitions(in map[string]ScriptoriumArtifactConfig) map[string]ScriptoriumArtifactConfig {
|
||||
if len(in) == 0 {
|
||||
return map[string]ScriptoriumArtifactConfig{}
|
||||
@@ -447,6 +422,7 @@ func cloneArtifactFamilyPublish(in *ScriptoriumArtifactFamilyPublishConfig) *Scr
|
||||
func cloneArtifactFamilyCatalog(in ArtifactFamilyCatalog) ArtifactFamilyCatalog {
|
||||
out := ArtifactFamilyCatalog{Families: make(map[string]ArtifactFamilyOrigin, len(in.Families)), Members: make(map[string]ArtifactFamilyMemberOrigin, len(in.Members))}
|
||||
for key, family := range in.Families {
|
||||
family.Sources = append([]string(nil), family.Sources...)
|
||||
family.Members = append([]string(nil), family.Members...)
|
||||
family.MemberDependencies = append([]string(nil), family.MemberDependencies...)
|
||||
family.Publish = cloneArtifactFamilyPublish(family.Publish)
|
||||
|
||||
@@ -90,13 +90,13 @@ characters:
|
||||
t.Fatalf("zeta class summary = %#v", got)
|
||||
}
|
||||
catalog := ArtifactFamilies(cfg.Pipeline)
|
||||
if got, want := catalog.Families["character_meta"].Source, absolutePath(t, pipelinePath); got != want {
|
||||
t.Fatalf("family source = %q, want %q", got, want)
|
||||
if got, want := catalog.Families["character_meta"].Sources, []string{absolutePath(t, pipelinePath)}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("family sources = %#v, want %#v", got, want)
|
||||
}
|
||||
if got := catalog.Families["character_meta"].Members; !reflect.DeepEqual(got, []string{"character_meta_alpha", "character_meta_zeta"}) {
|
||||
t.Fatalf("meta members = %#v", got)
|
||||
}
|
||||
if got := catalog.Members["character_items_zeta"]; got.Family != "character_items" || got.CharacterID != "zeta" || got.Source == "" {
|
||||
if got := catalog.Members["character_items_zeta"]; got.Family != "character_items" || got.CharacterID != "zeta" {
|
||||
t.Fatalf("zeta origin = %#v", got)
|
||||
}
|
||||
|
||||
|
||||
@@ -277,12 +277,12 @@ type ArtifactFamilyCatalog struct {
|
||||
Members map[string]ArtifactFamilyMemberOrigin
|
||||
}
|
||||
|
||||
// ArtifactFamilyOrigin records one declared family and its source ownership.
|
||||
// ArtifactFamilyOrigin records one declared family and its contributing sources.
|
||||
type ArtifactFamilyOrigin struct {
|
||||
Members []string
|
||||
MemberDependencies []string
|
||||
Publish *ScriptoriumArtifactFamilyPublishConfig
|
||||
Source string
|
||||
Sources []string
|
||||
}
|
||||
|
||||
// ArtifactFamilyMemberOrigin identifies the canonical party member that
|
||||
@@ -290,7 +290,6 @@ type ArtifactFamilyOrigin struct {
|
||||
type ArtifactFamilyMemberOrigin struct {
|
||||
Family string
|
||||
CharacterID string
|
||||
Source string
|
||||
Dependencies []string
|
||||
Inputs map[string]string
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -44,31 +43,6 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return fmt.Errorf("pipeline resolution metadata is required")
|
||||
}
|
||||
declared := make(map[string][]string, len(cfg.resolution.ownership))
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
declared[ownership.path] = append([]string(nil), ownership.sources...)
|
||||
}
|
||||
records, err := projectionValueRecords(projectEffectivePipeline(cfg, true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ownership := make([]pipelineFieldOwnership, 0, len(records))
|
||||
for _, record := range records {
|
||||
sources := declared[record.Path]
|
||||
if len(sources) == 0 {
|
||||
sources = []string{pipelineDefaultOwnershipSource}
|
||||
}
|
||||
ownership = append(ownership, pipelineFieldOwnership{
|
||||
path: record.Path, sources: append([]string(nil), sources...),
|
||||
})
|
||||
}
|
||||
for path, sources := range declared {
|
||||
if strings.HasPrefix(path, "scriptorium.artifact_families.") {
|
||||
ownership = append(ownership, pipelineFieldOwnership{path: path, sources: append([]string(nil), sources...)})
|
||||
}
|
||||
}
|
||||
sort.Slice(ownership, func(i, j int) bool { return ownership[i].path < ownership[j].path })
|
||||
cfg.resolution.ownership = ownership
|
||||
return recomputePipelineEffectiveDigest(cfg)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -80,7 +81,10 @@ func EffectivePipelineSources(cfg *PipelineConfig, party ResolvedParty) ([]Effec
|
||||
for _, value := range values {
|
||||
paths = append(paths, value.Path)
|
||||
}
|
||||
owners := effectivePipelineOwners(cfg)
|
||||
owners, err := effectivePipelineOwners(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
families := ArtifactFamilies(cfg)
|
||||
publishCount := 0
|
||||
if cfg.Publish != nil {
|
||||
@@ -219,13 +223,23 @@ func effectiveYAMLNode(value any) (*yaml.Node, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
|
||||
func effectivePipelineOwners(cfg *PipelineConfig) (map[string][]string, error) {
|
||||
owners := make(map[string][]string)
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return owners
|
||||
return owners, nil
|
||||
}
|
||||
for _, ownership := range cfg.resolution.ownership {
|
||||
owners[ownership.path] = append([]string(nil), ownership.sources...)
|
||||
paths, err := pipelineMetadataPaths(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pathsByID := pipelineSourcePaths(cfg)
|
||||
for _, value := range paths {
|
||||
info, ok := cfg.resolution.configkitMetadata.Lookup(configkit.Keys(value.segments...))
|
||||
sources := []string{pipelineDefaultOwnershipSource}
|
||||
if ok {
|
||||
sources = ownershipSources(info.Origin, pathsByID)
|
||||
}
|
||||
owners[value.record.Path] = sources
|
||||
}
|
||||
if cfg.Scriptorium != nil {
|
||||
for key, artifact := range cfg.Scriptorium.Artifacts {
|
||||
@@ -236,7 +250,7 @@ func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
|
||||
}
|
||||
}
|
||||
}
|
||||
return owners
|
||||
return owners, nil
|
||||
}
|
||||
|
||||
func artifactDependencyOwnershipPaths(artifactKey string, artifact ScriptoriumArtifactConfig) []string {
|
||||
@@ -308,8 +322,8 @@ func generatedArtifactSources(path, familyKey string, families ArtifactFamilyCat
|
||||
if len(sources) != 0 {
|
||||
return sources
|
||||
}
|
||||
if family, ok := families.Families[familyKey]; ok && family.Source != "" {
|
||||
return []string{family.Source}
|
||||
if family, ok := families.Families[familyKey]; ok && len(family.Sources) != 0 {
|
||||
return append([]string(nil), family.Sources...)
|
||||
}
|
||||
return []string{pipelineDefaultOwnershipSource}
|
||||
}
|
||||
@@ -369,8 +383,8 @@ func appendGeneratedPublishSources(records []EffectivePipelineSourceRecord, path
|
||||
familyPrefix := "scriptorium.artifact_families." + familyKey + ".publish."
|
||||
sources := sourcesForPipelinePaths(owners, []string{familyPrefix + "enabled", familyPrefix + "required", familyPrefix + "dest_pattern"})
|
||||
if len(sources) == 0 {
|
||||
if family, ok := families.Families[familyKey]; ok && family.Source != "" {
|
||||
sources = []string{family.Source}
|
||||
if family, ok := families.Families[familyKey]; ok && len(family.Sources) != 0 {
|
||||
sources = append([]string(nil), family.Sources...)
|
||||
} else {
|
||||
sources = []string{pipelineDefaultOwnershipSource}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ type pipelineResolutionMetadata struct {
|
||||
logicalNotariusConfig string
|
||||
logicalNotariusWorking string
|
||||
logicalNotariusCaptured bool
|
||||
ownership []pipelineFieldOwnership
|
||||
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
||||
artifactFamiliesExpanded bool
|
||||
configkitMetadata configkit.Metadata
|
||||
@@ -33,11 +32,6 @@ type pipelineOverlaySelection struct {
|
||||
overlayPath string
|
||||
}
|
||||
|
||||
type pipelineFieldOwnership struct {
|
||||
path string
|
||||
sources []string
|
||||
}
|
||||
|
||||
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
|
||||
overlay := selectedPipelineOverlayName(opts)
|
||||
result, err := loadPipelineConfigKitResult(path, overlay)
|
||||
@@ -45,11 +39,7 @@ func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*Pipeline
|
||||
return nil, describePipelineLoadError(err)
|
||||
}
|
||||
cfg := result.Config
|
||||
metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
||||
}
|
||||
cfg.resolution = metadata
|
||||
cfg.resolution = pipelineMetadataFromConfigKit(result, opts)
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
@@ -257,19 +247,16 @@ func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Valida
|
||||
}
|
||||
}
|
||||
|
||||
func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) {
|
||||
func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) *pipelineResolutionMetadata {
|
||||
sources := result.Sources()
|
||||
metadata := result.Metadata()
|
||||
resolved := &pipelineResolutionMetadata{
|
||||
configkitMetadata: metadata, configkitSources: sources,
|
||||
configkitMetadata: result.Metadata(), configkitSources: sources,
|
||||
}
|
||||
pathsByID := make(map[string]string, len(sources))
|
||||
for index, source := range sources {
|
||||
path := source.Path
|
||||
if path == "" {
|
||||
path = source.Label
|
||||
}
|
||||
pathsByID[source.ID] = path
|
||||
resolved.sources = append(resolved.sources, path)
|
||||
if index == 0 {
|
||||
resolved.rootPath = path
|
||||
@@ -280,19 +267,7 @@ func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result
|
||||
resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path}
|
||||
}
|
||||
}
|
||||
detailed, err := pipelineMetadataPaths(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, value := range detailed {
|
||||
info, ok := metadata.Lookup(configkit.Keys(value.segments...))
|
||||
sourceNames := []string{pipelineDefaultOwnershipSource}
|
||||
if ok {
|
||||
sourceNames = ownershipSources(info.Origin, pathsByID)
|
||||
}
|
||||
resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames})
|
||||
}
|
||||
return resolved, nil
|
||||
return resolved
|
||||
}
|
||||
|
||||
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
|
||||
@@ -300,14 +275,21 @@ func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 {
|
||||
if cfg.Scriptorium == nil {
|
||||
return effective, nil
|
||||
}
|
||||
families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}})
|
||||
families := cfg.Scriptorium.ArtifactFamilies
|
||||
if cfg.resolution != nil && len(cfg.resolution.artifactFamilies) != 0 {
|
||||
families = cfg.resolution.artifactFamilies
|
||||
}
|
||||
if len(families) == 0 {
|
||||
return effective, nil
|
||||
}
|
||||
detailedFamilies, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": families}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(effective, families...), nil
|
||||
return append(effective, detailedFamilies...), nil
|
||||
}
|
||||
|
||||
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
|
||||
@@ -319,7 +301,7 @@ func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []st
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var sources []string
|
||||
for _, location := range origin.Locations {
|
||||
add := func(location configkit.Location) {
|
||||
value := pathsByID[location.SourceID]
|
||||
if value == "" {
|
||||
value = location.SourceID
|
||||
@@ -329,12 +311,35 @@ func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []st
|
||||
sources = append(sources, value)
|
||||
}
|
||||
}
|
||||
for _, location := range origin.Locations {
|
||||
add(location)
|
||||
}
|
||||
for _, inheritance := range origin.Inheritance {
|
||||
for _, location := range inheritance.Locations {
|
||||
add(location)
|
||||
}
|
||||
}
|
||||
if len(sources) == 0 {
|
||||
return []string{pipelineDefaultOwnershipSource}
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func pipelineSourcePaths(cfg *PipelineConfig) map[string]string {
|
||||
paths := make(map[string]string)
|
||||
if cfg == nil || cfg.resolution == nil {
|
||||
return paths
|
||||
}
|
||||
for _, source := range cfg.resolution.configkitSources {
|
||||
path := source.Path
|
||||
if path == "" {
|
||||
path = source.Label
|
||||
}
|
||||
paths[source.ID] = path
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func intPtr(value int) *int { return &value }
|
||||
func float64Ptr(value float64) *float64 { return &value }
|
||||
func boolPtr(value bool) *bool { return &value }
|
||||
|
||||
Reference in New Issue
Block a user