Preserve composite ConfigKit provenance
This commit is contained in:
@@ -127,9 +127,11 @@ artifact key. Its tab-separated header is `key`, `source`, `definitions`,
|
|||||||
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
|
`enabled`, `output`, `dependencies`, `family`, `character`, `family_source`, and
|
||||||
`party_source`. Definitions and effective dependencies are sorted comma-separated
|
`party_source`. Definitions and effective dependencies are sorted comma-separated
|
||||||
lists, and control characters in fields are escaped. Family-member rows identify
|
lists, and control characters in fields are escaped. Family-member rows identify
|
||||||
their family, character, defining family file, and party file. An empty catalog
|
their family, character, contributing family source files, and party file.
|
||||||
prints only the header. Ordinary artifacts can be inspected without a campaign;
|
Family source paths are comma-separated when ConfigKit reports a composite
|
||||||
party-driven families retain the campaign requirement. See
|
origin. An empty catalog prints only the header. Ordinary artifacts can be
|
||||||
|
inspected without a campaign; party-driven families retain the campaign
|
||||||
|
requirement. See
|
||||||
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
|
[Scriptorium Artifact Entries](./config.md#scriptorium-artifact-entries) for the
|
||||||
configuration layout.
|
configuration layout.
|
||||||
|
|
||||||
|
|||||||
@@ -67,7 +67,8 @@ not promised to be a reloadable authoring document. `config sources` prints a st
|
|||||||
source projection instead of effective values: root/import/overlay/default
|
source projection instead of effective values: root/import/overlay/default
|
||||||
ownership plus campaign/party and generated-family records. Canonical derived
|
ownership plus campaign/party and generated-family records. Canonical derived
|
||||||
players trace to the party. `config artifacts` lists the concrete artifact map,
|
players trace to the party. `config artifacts` lists the concrete artifact map,
|
||||||
effective dependencies, and defining sources. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
effective dependencies, and definition contributor paths, which may identify
|
||||||
|
several files for one family member. The [CLI reference](cli.md#config-validate-config-show-config-sources-config-artifacts-and-config-diff)
|
||||||
owns command syntax and output conventions.
|
owns command syntax and output conventions.
|
||||||
|
|
||||||
`config diff` compares normalized field values rather than YAML text or source
|
`config diff` compares normalized field values rather than YAML text or source
|
||||||
|
|||||||
@@ -74,7 +74,10 @@ commands do not marshal runtime objects directly.
|
|||||||
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
`EffectivePipelineSources` and `EffectiveCampaignSources` provide the separate
|
||||||
safe provenance projection for `config sources`. Pipeline ownership begins with
|
safe provenance projection for `config sources`. Pipeline ownership begins with
|
||||||
the complete logical field paths projected from the schema and classifies
|
the complete logical field paths projected from the schema and classifies
|
||||||
each contributor as root, import, overlay, or centralized default. The
|
each contributor as root, import, overlay, or centralized default. It reads
|
||||||
|
ConfigKit's effective origins and inheritance locations when building each
|
||||||
|
projection, so inherited values retain their contributors and controlling
|
||||||
|
directives without a second cached leaf-ownership table. The
|
||||||
projection replaces generated concrete member paths with paired family and
|
projection replaces generated concrete member paths with paired family and
|
||||||
canonical-party records, and does the same for generated publish rules.
|
canonical-party records, and does the same for generated publish rules.
|
||||||
Campaign records identify campaign-owned fields and party inputs; canonical
|
Campaign records identify campaign-owned fields and party inputs; canonical
|
||||||
@@ -84,8 +87,9 @@ records with selection metadata and never reparses configuration files.
|
|||||||
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
|
`EffectiveArtifactCatalog` projects the same finalized concrete artifact map for
|
||||||
`config artifacts`. It derives source identity from each map key, carries sorted
|
`config artifacts`. It derives source identity from each map key, carries sorted
|
||||||
effective dependencies, aggregates ConfigKit definition contributors, and adds
|
effective dependencies, aggregates ConfigKit definition contributors, and adds
|
||||||
family/character/party provenance for generated members. It does not inspect
|
family/character/party provenance for generated members. Family source evidence
|
||||||
files named after artifacts or infer another runtime graph.
|
can name multiple contributing files. It does not inspect files named after
|
||||||
|
artifacts or infer another runtime graph.
|
||||||
|
|
||||||
`config diff` uses a thin paired loader that performs two independent ConfigKit
|
`config diff` uses a thin paired loader that performs two independent ConfigKit
|
||||||
loads. When campaign
|
loads. When campaign
|
||||||
@@ -152,10 +156,8 @@ rediscovering additional conflicts.
|
|||||||
|
|
||||||
The loaded pipeline retains private runtime metadata for the absolute root
|
The loaded pipeline retains private runtime metadata for the absolute root
|
||||||
path, ordered imports, selected overlay name, source, and path, original
|
path, ordered imports, selected overlay name, source, and path, original
|
||||||
ConfigKit metadata and source snapshots, effective digest,
|
ConfigKit metadata and source snapshots, and effective digest. Leaf source
|
||||||
and leaf ownership. Base
|
records are projected from that retained native metadata when requested. This
|
||||||
leaves retain their root/import owners, replaced leaves belong to the selected
|
|
||||||
overlay, and centrally supplied values use the synthetic `default` owner. This
|
|
||||||
metadata does not participate in YAML decoding or alter the public
|
metadata does not participate in YAML decoding or alter the public
|
||||||
configuration model.
|
configuration model.
|
||||||
|
|
||||||
@@ -179,8 +181,8 @@ for later runtime expansion.
|
|||||||
## Test Surfaces
|
## Test Surfaces
|
||||||
|
|
||||||
`pipeline_configkit_test.go` exercises real imports and fixed-path overlays,
|
`pipeline_configkit_test.go` exercises real imports and fixed-path overlays,
|
||||||
base selection, unselected-overlay isolation, source attribution, and Narratio's
|
base selection, unselected-overlay isolation, source attribution, cross-file
|
||||||
unique artifact/family base-declaration rule. Existing focused configuration
|
inheritance, and effective composition. Existing focused configuration
|
||||||
tests protect defaults, explicit-value semantics, strict decoding, validation,
|
tests protect defaults, explicit-value semantics, strict decoding, validation,
|
||||||
metadata, and root-relative path behavior. Application configuration-loader tests protect the
|
metadata, and root-relative path behavior. Application configuration-loader tests protect the
|
||||||
single-read boundary by changing the pipeline file after its initial load and
|
single-read boundary by changing the pipeline file after its initial load and
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ Complete when valid cross-file composition/inheritance succeeds and invalid
|
|||||||
effective configuration still fails before execution, without authored-source
|
effective configuration still fails before execution, without authored-source
|
||||||
acceptance gates. Keep native metadata available for the next stage.
|
acceptance gates. Keep native metadata available for the next stage.
|
||||||
|
|
||||||
## Stage 3: Simplify provenance inspection around native APIs
|
## Stage 3: Simplify provenance inspection around native APIs [x]
|
||||||
|
|
||||||
Purpose: preserve useful operator inspection without a parallel provenance
|
Purpose: preserve useful operator inspection without a parallel provenance
|
||||||
system or a unique-defining-file contract for composite entries.
|
system or a unique-defining-file contract for composite entries.
|
||||||
|
|||||||
@@ -109,6 +109,96 @@ func TestConfigArtifactsReportsOverlayFamilyContributors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfigInspectionReportsInheritedCompositeSources(t *testing.T) {
|
||||||
|
pipelinePath, campaignPath := writeInspectionFamilyConfig(t)
|
||||||
|
dir := filepath.Dir(pipelinePath)
|
||||||
|
parentPath := filepath.Join(dir, "families", "parent.yml")
|
||||||
|
childPath := filepath.Join(dir, "families", "child.yml")
|
||||||
|
overlayPath := filepath.Join(dir, "overlays", "testing.yml")
|
||||||
|
mustWriteTestFile(t, pipelinePath, `configkit:
|
||||||
|
imports:
|
||||||
|
- path: families/parent.yml
|
||||||
|
- path: families/child.yml
|
||||||
|
workspace:
|
||||||
|
root: `+filepath.Join(dir, "workspace")+`
|
||||||
|
campaigns:
|
||||||
|
root: `+filepath.Join(dir, "campaigns")+`
|
||||||
|
whisperx:
|
||||||
|
transcribe_url: https://transcription.example.com/transcribe
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, parentPath, `scriptorium:
|
||||||
|
artifact_families:
|
||||||
|
character_meta:
|
||||||
|
enabled: false
|
||||||
|
for_each: party.characters
|
||||||
|
prompt_id: dnd.character_meta
|
||||||
|
profile_id: production
|
||||||
|
output_path_pattern: artifacts/characters/{character_id}/meta.md
|
||||||
|
artifacts:
|
||||||
|
notes:
|
||||||
|
enabled: false
|
||||||
|
prompt_id: dnd.notes
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, childPath, `scriptorium:
|
||||||
|
artifact_families:
|
||||||
|
character_note:
|
||||||
|
extends: character_meta
|
||||||
|
profile_id: testing
|
||||||
|
output_path_pattern: artifacts/characters/{character_id}/note.md
|
||||||
|
artifacts:
|
||||||
|
notes:
|
||||||
|
output_path: artifacts/notes.md
|
||||||
|
`)
|
||||||
|
mustWriteTestFile(t, overlayPath, `scriptorium:
|
||||||
|
artifact_families:
|
||||||
|
character_note:
|
||||||
|
prompt_id: dnd.character_note.testing
|
||||||
|
artifacts:
|
||||||
|
notes:
|
||||||
|
output_path: artifacts/notes-testing.md
|
||||||
|
`)
|
||||||
|
args := []string{"--config", pipelinePath, "--campaign-file", campaignPath, "--overlay", "testing"}
|
||||||
|
|
||||||
|
var catalog bytes.Buffer
|
||||||
|
if err := ConfigArtifacts(context.Background(), args, &catalog); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
records := catalogRecordsByKey(t, catalog.String())
|
||||||
|
notes := records["notes"]
|
||||||
|
if notes[4] != "artifacts/notes-testing.md" {
|
||||||
|
t.Fatalf("composed artifact output = %#v", notes)
|
||||||
|
}
|
||||||
|
for _, contributor := range []string{parentPath, childPath, overlayPath} {
|
||||||
|
if !strings.Contains(notes[2], contributor) {
|
||||||
|
t.Errorf("composed artifact definitions %q omit %q", notes[2], contributor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
note := records["character_note_arannis"]
|
||||||
|
if note[4] != "artifacts/characters/arannis/note.md" || note[6] != "character_note" || note[7] != "arannis" {
|
||||||
|
t.Fatalf("inherited family catalog identity and output = %#v", note)
|
||||||
|
}
|
||||||
|
for _, contributor := range []string{parentPath, childPath, overlayPath} {
|
||||||
|
if !strings.Contains(note[2], contributor) || !strings.Contains(note[8], contributor) {
|
||||||
|
t.Errorf("family provenance = definitions %q, family sources %q; want %q", note[2], note[8], contributor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var sources bytes.Buffer
|
||||||
|
if err := ConfigSources(context.Background(), args, &sources); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"scriptorium.artifacts.notes.output_path\toverlay\t" + overlayPath,
|
||||||
|
"scriptorium.artifacts.character_note_arannis.output_path\tfamily\t" + childPath,
|
||||||
|
"scriptorium.artifacts.character_note_arannis.prompt_id\tfamily\t" + overlayPath,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(sources.String(), want) {
|
||||||
|
t.Errorf("effective source output missing %q:\n%s", want, sources.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConfigArtifactsReportsBaseAndOverlayContributors(t *testing.T) {
|
func TestConfigArtifactsReportsBaseAndOverlayContributors(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
pipelinePath := filepath.Join(dir, "pipeline.yml")
|
||||||
|
|||||||
@@ -178,7 +178,7 @@ func configArtifactsWithDependencies(_ context.Context, args []string, out io.Wr
|
|||||||
strings.Join(record.Dependencies, ","),
|
strings.Join(record.Dependencies, ","),
|
||||||
record.FamilyKey,
|
record.FamilyKey,
|
||||||
record.CharacterID,
|
record.CharacterID,
|
||||||
config.NormalizedConfigurationPath(record.FamilySource),
|
strings.Join(normalizedCatalogPaths(record.FamilySources), ","),
|
||||||
config.NormalizedConfigurationPath(record.PartySource),
|
config.NormalizedConfigurationPath(record.PartySource),
|
||||||
}
|
}
|
||||||
for index := range fields {
|
for index := range fields {
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ type EffectiveArtifactRecord struct {
|
|||||||
Dependencies []string
|
Dependencies []string
|
||||||
FamilyKey string
|
FamilyKey string
|
||||||
CharacterID string
|
CharacterID string
|
||||||
FamilySource string
|
FamilySources []string
|
||||||
PartySource string
|
PartySource string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,7 +53,7 @@ func EffectiveArtifactCatalog(cfg *PipelineConfig, party ResolvedParty) ([]Effec
|
|||||||
record.DefinitionSources = artifactFamilyDefinitionSources(cfg, member.Family)
|
record.DefinitionSources = artifactFamilyDefinitionSources(cfg, member.Family)
|
||||||
record.FamilyKey = member.Family
|
record.FamilyKey = member.Family
|
||||||
record.CharacterID = member.CharacterID
|
record.CharacterID = member.CharacterID
|
||||||
record.FamilySource = member.Source
|
record.FamilySources = append([]string(nil), cfg.familyCatalog.Families[member.Family].Sources...)
|
||||||
record.PartySource = party.Source.Path
|
record.PartySource = party.Source.Path
|
||||||
} else {
|
} else {
|
||||||
record.DefinitionSources = artifactDefinitionSources(cfg, key)
|
record.DefinitionSources = artifactDefinitionSources(cfg, key)
|
||||||
@@ -67,60 +67,52 @@ func EffectiveArtifactCatalog(cfg *PipelineConfig, party ResolvedParty) ([]Effec
|
|||||||
func artifactDefinitionSources(cfg *PipelineConfig, key string) []string {
|
func artifactDefinitionSources(cfg *PipelineConfig, key string) []string {
|
||||||
return declarationDefinitionSources(cfg,
|
return declarationDefinitionSources(cfg,
|
||||||
configkit.Keys("scriptorium", "artifacts", key),
|
configkit.Keys("scriptorium", "artifacts", key),
|
||||||
"scriptorium.artifacts."+key,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func artifactFamilyDefinitionSources(cfg *PipelineConfig, key string) []string {
|
func artifactFamilyDefinitionSources(cfg *PipelineConfig, key string) []string {
|
||||||
return declarationDefinitionSources(cfg,
|
return declarationDefinitionSources(cfg,
|
||||||
configkit.Keys("scriptorium", "artifact_families", key),
|
configkit.Keys("scriptorium", "artifact_families", key),
|
||||||
"scriptorium.artifact_families."+key,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Path, prefix string) []string {
|
func declarationDefinitionSources(cfg *PipelineConfig, declaration configkit.Path) []string {
|
||||||
if cfg == nil || cfg.resolution == nil {
|
if cfg == nil || cfg.resolution == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
pathsByID := make(map[string]string, len(cfg.resolution.configkitSources))
|
pathsByID := pipelineSourcePaths(cfg)
|
||||||
for _, source := range cfg.resolution.configkitSources {
|
|
||||||
path := source.Path
|
|
||||||
if path == "" {
|
|
||||||
path = source.Label
|
|
||||||
}
|
|
||||||
pathsByID[source.ID] = path
|
|
||||||
}
|
|
||||||
|
|
||||||
seen := map[string]struct{}{}
|
seen := map[string]struct{}{}
|
||||||
for _, change := range cfg.resolution.configkitMetadata.History(declaration) {
|
for _, change := range cfg.resolution.configkitMetadata.History(declaration) {
|
||||||
if change.Origin.Layer != configkit.LayerBase && change.Origin.Layer != configkit.LayerOverlay {
|
if change.Origin.Layer != configkit.LayerBase && change.Origin.Layer != configkit.LayerOverlay {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, location := range change.Origin.Locations {
|
for _, location := range change.Origin.Locations {
|
||||||
source := pathsByID[location.SourceID]
|
seen[location.SourceID] = struct{}{}
|
||||||
if source == "" {
|
}
|
||||||
source = location.SourceID
|
for _, inheritance := range change.Origin.Inheritance {
|
||||||
|
for _, location := range inheritance.Locations {
|
||||||
|
seen[location.SourceID] = struct{}{}
|
||||||
}
|
}
|
||||||
seen[source] = struct{}{}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(seen) == 0 {
|
if len(seen) == 0 {
|
||||||
for _, ownership := range cfg.resolution.ownership {
|
if info, ok := cfg.resolution.configkitMetadata.Lookup(declaration); ok {
|
||||||
if ownership.path != prefix && !strings.HasPrefix(ownership.path, prefix+".") {
|
for _, location := range info.Origin.Locations {
|
||||||
continue
|
seen[location.SourceID] = struct{}{}
|
||||||
}
|
}
|
||||||
for _, source := range ownership.sources {
|
for _, inheritance := range info.Origin.Inheritance {
|
||||||
if source != pipelineDefaultOwnershipSource && source != "resolution" {
|
for _, location := range inheritance.Locations {
|
||||||
seen[source] = struct{}{}
|
seen[location.SourceID] = struct{}{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(seen) == 0 && cfg.resolution.rootPath != "" {
|
|
||||||
seen[cfg.resolution.rootPath] = struct{}{}
|
|
||||||
}
|
|
||||||
result := make([]string, 0, len(seen))
|
result := make([]string, 0, len(seen))
|
||||||
for source := range seen {
|
for sourceID := range seen {
|
||||||
|
source := pathsByID[sourceID]
|
||||||
|
if source == "" {
|
||||||
|
source = sourceID
|
||||||
|
}
|
||||||
result = append(result, source)
|
result = append(result, source)
|
||||||
}
|
}
|
||||||
sort.Strings(result)
|
sort.Strings(result)
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
|
|||||||
Members: make([]string, 0, len(characters)),
|
Members: make([]string, 0, len(characters)),
|
||||||
MemberDependencies: append([]string(nil), family.MemberDependencies...),
|
MemberDependencies: append([]string(nil), family.MemberDependencies...),
|
||||||
Publish: cloneArtifactFamilyPublish(family.Publish),
|
Publish: cloneArtifactFamilyPublish(family.Publish),
|
||||||
Source: artifactFamilySource(cfg, familyKey),
|
Sources: artifactFamilyDefinitionSources(cfg, familyKey),
|
||||||
}
|
}
|
||||||
for _, character := range characters {
|
for _, character := range characters {
|
||||||
key := familyKey + "_" + character.ID
|
key := familyKey + "_" + character.ID
|
||||||
@@ -116,7 +116,7 @@ func expandPipelineArtifactFamilies(cfg *PipelineConfig, party ResolvedParty) er
|
|||||||
}
|
}
|
||||||
outputOwners[outputPath] = key
|
outputOwners[outputPath] = key
|
||||||
familyOrigin.Members = append(familyOrigin.Members, key)
|
familyOrigin.Members = append(familyOrigin.Members, key)
|
||||||
catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Source: familyOrigin.Source, Dependencies: append([]string(nil), dependencies...), Inputs: inputSourceMap(inputs)}
|
catalog.Members[key] = ArtifactFamilyMemberOrigin{Family: familyKey, CharacterID: character.ID, Dependencies: append([]string(nil), dependencies...), Inputs: inputSourceMap(inputs)}
|
||||||
}
|
}
|
||||||
catalog.Families[familyKey] = familyOrigin
|
catalog.Families[familyKey] = familyOrigin
|
||||||
}
|
}
|
||||||
@@ -351,31 +351,6 @@ func sortedFamilyKeys(families map[string]ScriptoriumArtifactFamilyConfig) []str
|
|||||||
return keys
|
return keys
|
||||||
}
|
}
|
||||||
|
|
||||||
func artifactFamilySource(cfg *PipelineConfig, family string) string {
|
|
||||||
if cfg == nil || cfg.resolution == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
prefix := "scriptorium.artifact_families." + family + "."
|
|
||||||
var fallback string
|
|
||||||
for _, ownership := range cfg.resolution.ownership {
|
|
||||||
if !strings.HasPrefix(ownership.path, prefix) || len(ownership.sources) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, source := range ownership.sources {
|
|
||||||
if source != pipelineDefaultOwnershipSource {
|
|
||||||
return source
|
|
||||||
}
|
|
||||||
if fallback == "" {
|
|
||||||
fallback = source
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if fallback != "" {
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
return pipelineDefaultOwnershipSource
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneArtifactDefinitions(in map[string]ScriptoriumArtifactConfig) map[string]ScriptoriumArtifactConfig {
|
func cloneArtifactDefinitions(in map[string]ScriptoriumArtifactConfig) map[string]ScriptoriumArtifactConfig {
|
||||||
if len(in) == 0 {
|
if len(in) == 0 {
|
||||||
return map[string]ScriptoriumArtifactConfig{}
|
return map[string]ScriptoriumArtifactConfig{}
|
||||||
@@ -447,6 +422,7 @@ func cloneArtifactFamilyPublish(in *ScriptoriumArtifactFamilyPublishConfig) *Scr
|
|||||||
func cloneArtifactFamilyCatalog(in ArtifactFamilyCatalog) ArtifactFamilyCatalog {
|
func cloneArtifactFamilyCatalog(in ArtifactFamilyCatalog) ArtifactFamilyCatalog {
|
||||||
out := ArtifactFamilyCatalog{Families: make(map[string]ArtifactFamilyOrigin, len(in.Families)), Members: make(map[string]ArtifactFamilyMemberOrigin, len(in.Members))}
|
out := ArtifactFamilyCatalog{Families: make(map[string]ArtifactFamilyOrigin, len(in.Families)), Members: make(map[string]ArtifactFamilyMemberOrigin, len(in.Members))}
|
||||||
for key, family := range in.Families {
|
for key, family := range in.Families {
|
||||||
|
family.Sources = append([]string(nil), family.Sources...)
|
||||||
family.Members = append([]string(nil), family.Members...)
|
family.Members = append([]string(nil), family.Members...)
|
||||||
family.MemberDependencies = append([]string(nil), family.MemberDependencies...)
|
family.MemberDependencies = append([]string(nil), family.MemberDependencies...)
|
||||||
family.Publish = cloneArtifactFamilyPublish(family.Publish)
|
family.Publish = cloneArtifactFamilyPublish(family.Publish)
|
||||||
|
|||||||
@@ -90,13 +90,13 @@ characters:
|
|||||||
t.Fatalf("zeta class summary = %#v", got)
|
t.Fatalf("zeta class summary = %#v", got)
|
||||||
}
|
}
|
||||||
catalog := ArtifactFamilies(cfg.Pipeline)
|
catalog := ArtifactFamilies(cfg.Pipeline)
|
||||||
if got, want := catalog.Families["character_meta"].Source, absolutePath(t, pipelinePath); got != want {
|
if got, want := catalog.Families["character_meta"].Sources, []string{absolutePath(t, pipelinePath)}; !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("family source = %q, want %q", got, want)
|
t.Fatalf("family sources = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
if got := catalog.Families["character_meta"].Members; !reflect.DeepEqual(got, []string{"character_meta_alpha", "character_meta_zeta"}) {
|
if got := catalog.Families["character_meta"].Members; !reflect.DeepEqual(got, []string{"character_meta_alpha", "character_meta_zeta"}) {
|
||||||
t.Fatalf("meta members = %#v", got)
|
t.Fatalf("meta members = %#v", got)
|
||||||
}
|
}
|
||||||
if got := catalog.Members["character_items_zeta"]; got.Family != "character_items" || got.CharacterID != "zeta" || got.Source == "" {
|
if got := catalog.Members["character_items_zeta"]; got.Family != "character_items" || got.CharacterID != "zeta" {
|
||||||
t.Fatalf("zeta origin = %#v", got)
|
t.Fatalf("zeta origin = %#v", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -277,12 +277,12 @@ type ArtifactFamilyCatalog struct {
|
|||||||
Members map[string]ArtifactFamilyMemberOrigin
|
Members map[string]ArtifactFamilyMemberOrigin
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactFamilyOrigin records one declared family and its source ownership.
|
// ArtifactFamilyOrigin records one declared family and its contributing sources.
|
||||||
type ArtifactFamilyOrigin struct {
|
type ArtifactFamilyOrigin struct {
|
||||||
Members []string
|
Members []string
|
||||||
MemberDependencies []string
|
MemberDependencies []string
|
||||||
Publish *ScriptoriumArtifactFamilyPublishConfig
|
Publish *ScriptoriumArtifactFamilyPublishConfig
|
||||||
Source string
|
Sources []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArtifactFamilyMemberOrigin identifies the canonical party member that
|
// ArtifactFamilyMemberOrigin identifies the canonical party member that
|
||||||
@@ -290,7 +290,6 @@ type ArtifactFamilyOrigin struct {
|
|||||||
type ArtifactFamilyMemberOrigin struct {
|
type ArtifactFamilyMemberOrigin struct {
|
||||||
Family string
|
Family string
|
||||||
CharacterID string
|
CharacterID string
|
||||||
Source string
|
|
||||||
Dependencies []string
|
Dependencies []string
|
||||||
Inputs map[string]string
|
Inputs map[string]string
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -44,31 +43,6 @@ func finalizePipelineResolution(cfg *PipelineConfig) error {
|
|||||||
if cfg == nil || cfg.resolution == nil {
|
if cfg == nil || cfg.resolution == nil {
|
||||||
return fmt.Errorf("pipeline resolution metadata is required")
|
return fmt.Errorf("pipeline resolution metadata is required")
|
||||||
}
|
}
|
||||||
declared := make(map[string][]string, len(cfg.resolution.ownership))
|
|
||||||
for _, ownership := range cfg.resolution.ownership {
|
|
||||||
declared[ownership.path] = append([]string(nil), ownership.sources...)
|
|
||||||
}
|
|
||||||
records, err := projectionValueRecords(projectEffectivePipeline(cfg, true))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ownership := make([]pipelineFieldOwnership, 0, len(records))
|
|
||||||
for _, record := range records {
|
|
||||||
sources := declared[record.Path]
|
|
||||||
if len(sources) == 0 {
|
|
||||||
sources = []string{pipelineDefaultOwnershipSource}
|
|
||||||
}
|
|
||||||
ownership = append(ownership, pipelineFieldOwnership{
|
|
||||||
path: record.Path, sources: append([]string(nil), sources...),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for path, sources := range declared {
|
|
||||||
if strings.HasPrefix(path, "scriptorium.artifact_families.") {
|
|
||||||
ownership = append(ownership, pipelineFieldOwnership{path: path, sources: append([]string(nil), sources...)})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Slice(ownership, func(i, j int) bool { return ownership[i].path < ownership[j].path })
|
|
||||||
cfg.resolution.ownership = ownership
|
|
||||||
return recomputePipelineEffectiveDigest(cfg)
|
return recomputePipelineEffectiveDigest(cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -80,7 +81,10 @@ func EffectivePipelineSources(cfg *PipelineConfig, party ResolvedParty) ([]Effec
|
|||||||
for _, value := range values {
|
for _, value := range values {
|
||||||
paths = append(paths, value.Path)
|
paths = append(paths, value.Path)
|
||||||
}
|
}
|
||||||
owners := effectivePipelineOwners(cfg)
|
owners, err := effectivePipelineOwners(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
families := ArtifactFamilies(cfg)
|
families := ArtifactFamilies(cfg)
|
||||||
publishCount := 0
|
publishCount := 0
|
||||||
if cfg.Publish != nil {
|
if cfg.Publish != nil {
|
||||||
@@ -219,13 +223,23 @@ func effectiveYAMLNode(value any) (*yaml.Node, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
|
func effectivePipelineOwners(cfg *PipelineConfig) (map[string][]string, error) {
|
||||||
owners := make(map[string][]string)
|
owners := make(map[string][]string)
|
||||||
if cfg == nil || cfg.resolution == nil {
|
if cfg == nil || cfg.resolution == nil {
|
||||||
return owners
|
return owners, nil
|
||||||
}
|
}
|
||||||
for _, ownership := range cfg.resolution.ownership {
|
paths, err := pipelineMetadataPaths(cfg)
|
||||||
owners[ownership.path] = append([]string(nil), ownership.sources...)
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
pathsByID := pipelineSourcePaths(cfg)
|
||||||
|
for _, value := range paths {
|
||||||
|
info, ok := cfg.resolution.configkitMetadata.Lookup(configkit.Keys(value.segments...))
|
||||||
|
sources := []string{pipelineDefaultOwnershipSource}
|
||||||
|
if ok {
|
||||||
|
sources = ownershipSources(info.Origin, pathsByID)
|
||||||
|
}
|
||||||
|
owners[value.record.Path] = sources
|
||||||
}
|
}
|
||||||
if cfg.Scriptorium != nil {
|
if cfg.Scriptorium != nil {
|
||||||
for key, artifact := range cfg.Scriptorium.Artifacts {
|
for key, artifact := range cfg.Scriptorium.Artifacts {
|
||||||
@@ -236,7 +250,7 @@ func effectivePipelineOwners(cfg *PipelineConfig) map[string][]string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return owners
|
return owners, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func artifactDependencyOwnershipPaths(artifactKey string, artifact ScriptoriumArtifactConfig) []string {
|
func artifactDependencyOwnershipPaths(artifactKey string, artifact ScriptoriumArtifactConfig) []string {
|
||||||
@@ -308,8 +322,8 @@ func generatedArtifactSources(path, familyKey string, families ArtifactFamilyCat
|
|||||||
if len(sources) != 0 {
|
if len(sources) != 0 {
|
||||||
return sources
|
return sources
|
||||||
}
|
}
|
||||||
if family, ok := families.Families[familyKey]; ok && family.Source != "" {
|
if family, ok := families.Families[familyKey]; ok && len(family.Sources) != 0 {
|
||||||
return []string{family.Source}
|
return append([]string(nil), family.Sources...)
|
||||||
}
|
}
|
||||||
return []string{pipelineDefaultOwnershipSource}
|
return []string{pipelineDefaultOwnershipSource}
|
||||||
}
|
}
|
||||||
@@ -369,8 +383,8 @@ func appendGeneratedPublishSources(records []EffectivePipelineSourceRecord, path
|
|||||||
familyPrefix := "scriptorium.artifact_families." + familyKey + ".publish."
|
familyPrefix := "scriptorium.artifact_families." + familyKey + ".publish."
|
||||||
sources := sourcesForPipelinePaths(owners, []string{familyPrefix + "enabled", familyPrefix + "required", familyPrefix + "dest_pattern"})
|
sources := sourcesForPipelinePaths(owners, []string{familyPrefix + "enabled", familyPrefix + "required", familyPrefix + "dest_pattern"})
|
||||||
if len(sources) == 0 {
|
if len(sources) == 0 {
|
||||||
if family, ok := families.Families[familyKey]; ok && family.Source != "" {
|
if family, ok := families.Families[familyKey]; ok && len(family.Sources) != 0 {
|
||||||
sources = []string{family.Source}
|
sources = append([]string(nil), family.Sources...)
|
||||||
} else {
|
} else {
|
||||||
sources = []string{pipelineDefaultOwnershipSource}
|
sources = []string{pipelineDefaultOwnershipSource}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ type pipelineResolutionMetadata struct {
|
|||||||
logicalNotariusConfig string
|
logicalNotariusConfig string
|
||||||
logicalNotariusWorking string
|
logicalNotariusWorking string
|
||||||
logicalNotariusCaptured bool
|
logicalNotariusCaptured bool
|
||||||
ownership []pipelineFieldOwnership
|
|
||||||
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
artifactFamilies map[string]ScriptoriumArtifactFamilyConfig
|
||||||
artifactFamiliesExpanded bool
|
artifactFamiliesExpanded bool
|
||||||
configkitMetadata configkit.Metadata
|
configkitMetadata configkit.Metadata
|
||||||
@@ -33,11 +32,6 @@ type pipelineOverlaySelection struct {
|
|||||||
overlayPath string
|
overlayPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
type pipelineFieldOwnership struct {
|
|
||||||
path string
|
|
||||||
sources []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
|
func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*PipelineConfig, error) {
|
||||||
overlay := selectedPipelineOverlayName(opts)
|
overlay := selectedPipelineOverlayName(opts)
|
||||||
result, err := loadPipelineConfigKitResult(path, overlay)
|
result, err := loadPipelineConfigKitResult(path, overlay)
|
||||||
@@ -45,11 +39,7 @@ func loadPipelineWithConfigKit(path string, opts PipelineLoadOptions) (*Pipeline
|
|||||||
return nil, describePipelineLoadError(err)
|
return nil, describePipelineLoadError(err)
|
||||||
}
|
}
|
||||||
cfg := result.Config
|
cfg := result.Config
|
||||||
metadata, err := pipelineMetadataFromConfigKit(&cfg, result, opts)
|
cfg.resolution = pipelineMetadataFromConfigKit(result, opts)
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("load pipeline config: %w", err)
|
|
||||||
}
|
|
||||||
cfg.resolution = metadata
|
|
||||||
return &cfg, nil
|
return &cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -257,19 +247,16 @@ func validatePipelineConfigKit(cfg *PipelineConfig, validation *configkit.Valida
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) (*pipelineResolutionMetadata, error) {
|
func pipelineMetadataFromConfigKit(result *configkit.Result[PipelineConfig], opts PipelineLoadOptions) *pipelineResolutionMetadata {
|
||||||
sources := result.Sources()
|
sources := result.Sources()
|
||||||
metadata := result.Metadata()
|
|
||||||
resolved := &pipelineResolutionMetadata{
|
resolved := &pipelineResolutionMetadata{
|
||||||
configkitMetadata: metadata, configkitSources: sources,
|
configkitMetadata: result.Metadata(), configkitSources: sources,
|
||||||
}
|
}
|
||||||
pathsByID := make(map[string]string, len(sources))
|
|
||||||
for index, source := range sources {
|
for index, source := range sources {
|
||||||
path := source.Path
|
path := source.Path
|
||||||
if path == "" {
|
if path == "" {
|
||||||
path = source.Label
|
path = source.Label
|
||||||
}
|
}
|
||||||
pathsByID[source.ID] = path
|
|
||||||
resolved.sources = append(resolved.sources, path)
|
resolved.sources = append(resolved.sources, path)
|
||||||
if index == 0 {
|
if index == 0 {
|
||||||
resolved.rootPath = path
|
resolved.rootPath = path
|
||||||
@@ -280,19 +267,7 @@ func pipelineMetadataFromConfigKit(cfg *PipelineConfig, result *configkit.Result
|
|||||||
resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path}
|
resolved.selectedOverlay = &pipelineOverlaySelection{name: selectedPipelineOverlayName(opts), source: "explicit", overlayPath: path}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
detailed, err := pipelineMetadataPaths(cfg)
|
return resolved
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, value := range detailed {
|
|
||||||
info, ok := metadata.Lookup(configkit.Keys(value.segments...))
|
|
||||||
sourceNames := []string{pipelineDefaultOwnershipSource}
|
|
||||||
if ok {
|
|
||||||
sourceNames = ownershipSources(info.Origin, pathsByID)
|
|
||||||
}
|
|
||||||
resolved.ownership = append(resolved.ownership, pipelineFieldOwnership{path: value.record.Path, sources: sourceNames})
|
|
||||||
}
|
|
||||||
return resolved, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
|
func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error) {
|
||||||
@@ -300,14 +275,21 @@ func pipelineMetadataPaths(cfg *PipelineConfig) ([]projectedPipelineValue, error
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if cfg.Scriptorium == nil || len(cfg.Scriptorium.ArtifactFamilies) == 0 {
|
if cfg.Scriptorium == nil {
|
||||||
return effective, nil
|
return effective, nil
|
||||||
}
|
}
|
||||||
families, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": cfg.Scriptorium.ArtifactFamilies}})
|
families := cfg.Scriptorium.ArtifactFamilies
|
||||||
|
if cfg.resolution != nil && len(cfg.resolution.artifactFamilies) != 0 {
|
||||||
|
families = cfg.resolution.artifactFamilies
|
||||||
|
}
|
||||||
|
if len(families) == 0 {
|
||||||
|
return effective, nil
|
||||||
|
}
|
||||||
|
detailedFamilies, err := projectionDetailedRecords(map[string]any{"scriptorium": map[string]any{"artifact_families": families}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return append(effective, families...), nil
|
return append(effective, detailedFamilies...), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
|
func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []string {
|
||||||
@@ -319,7 +301,7 @@ func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []st
|
|||||||
}
|
}
|
||||||
seen := map[string]struct{}{}
|
seen := map[string]struct{}{}
|
||||||
var sources []string
|
var sources []string
|
||||||
for _, location := range origin.Locations {
|
add := func(location configkit.Location) {
|
||||||
value := pathsByID[location.SourceID]
|
value := pathsByID[location.SourceID]
|
||||||
if value == "" {
|
if value == "" {
|
||||||
value = location.SourceID
|
value = location.SourceID
|
||||||
@@ -329,12 +311,35 @@ func ownershipSources(origin configkit.Origin, pathsByID map[string]string) []st
|
|||||||
sources = append(sources, value)
|
sources = append(sources, value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, location := range origin.Locations {
|
||||||
|
add(location)
|
||||||
|
}
|
||||||
|
for _, inheritance := range origin.Inheritance {
|
||||||
|
for _, location := range inheritance.Locations {
|
||||||
|
add(location)
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(sources) == 0 {
|
if len(sources) == 0 {
|
||||||
return []string{pipelineDefaultOwnershipSource}
|
return []string{pipelineDefaultOwnershipSource}
|
||||||
}
|
}
|
||||||
return sources
|
return sources
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func pipelineSourcePaths(cfg *PipelineConfig) map[string]string {
|
||||||
|
paths := make(map[string]string)
|
||||||
|
if cfg == nil || cfg.resolution == nil {
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
for _, source := range cfg.resolution.configkitSources {
|
||||||
|
path := source.Path
|
||||||
|
if path == "" {
|
||||||
|
path = source.Label
|
||||||
|
}
|
||||||
|
paths[source.ID] = path
|
||||||
|
}
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
|
||||||
func intPtr(value int) *int { return &value }
|
func intPtr(value int) *int { return &value }
|
||||||
func float64Ptr(value float64) *float64 { return &value }
|
func float64Ptr(value float64) *float64 { return &value }
|
||||||
func boolPtr(value bool) *bool { return &value }
|
func boolPtr(value bool) *bool { return &value }
|
||||||
|
|||||||
Reference in New Issue
Block a user