notarius/internal/framework/pipeline/profile.go

package pipeline

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"sort"
	"strings"

	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)

const (
	DefaultChunkModule     = "generic"
	DefaultMergeModule     = "appendorder"
	DefaultNormalizeModule = "noop"
	DefaultOutputModule    = "json"
	DefaultLLMProfile      = "default"
)

type ModuleBinding struct {
	Module     string            `json:"module"`
	LLMProfile string            `json:"llm_profile,omitempty"`
	Retries    int               `json:"retries,omitempty"`
	Options    map[string]any    `json:"options,omitempty"`
	References map[string]string `json:"references,omitempty"`
	Validators ValidatorOverride `json:"validators,omitempty"`
}

type ValidatorOverride struct {
	Set        bool            `json:"set,omitempty"`
	Validators []ModuleBinding `json:"validators,omitempty"`
}

func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
	type moduleBindingJSON struct {
		Module     string            `json:"module"`
		LLMProfile string            `json:"llm_profile,omitempty"`
		Retries    int               `json:"retries,omitempty"`
		Options    map[string]any    `json:"options,omitempty"`
		References map[string]string `json:"references,omitempty"`
		Validators *[]ModuleBinding  `json:"validators,omitempty"`
	}
	out := moduleBindingJSON{
		Module:     binding.Module,
		LLMProfile: binding.LLMProfile,
		Retries:    binding.Retries,
		Options:    binding.Options,
		References: binding.References,
	}
	if binding.Validators.Set {
		validators := cloneModuleBindings(binding.Validators.Validators)
		out.Validators = &validators
	}
	return json.Marshal(out)
}

type ArtifactLaneProfile struct {
	Extract    ModuleBinding     `json:"extract"`
	Merge      ModuleBinding     `json:"merge,omitempty"`
	Normalize  ModuleBinding     `json:"normalize,omitempty"`
	Validators []ModuleBinding   `json:"validators,omitempty"`
	References map[string]string `json:"references,omitempty"`
}

type PipelineProfile struct {
	ID         string                         `json:"id"`
	Input      ModuleBinding                  `json:"input"`
	Chunk      ModuleBinding                  `json:"chunk,omitempty"`
	Artifacts  map[string]ArtifactLaneProfile `json:"artifacts"`
	Output     ModuleBinding                  `json:"output,omitempty"`
	References map[string]string              `json:"references,omitempty"`
}

type ResolveOptions struct {
	Only               []string
	ReferenceOverrides []ReferenceBinding
	ReferenceUnbinds   []ReferenceUnbind
}

type ReferenceBinding struct {
	Stage         ModuleStage `json:"stage,omitempty"`
	LaneID        string      `json:"lane_id,omitempty"`
	SlotName      string      `json:"slot_name"`
	Source        string      `json:"source"`
	BindingSource string      `json:"binding_source,omitempty"`
}

type ReferenceUnbind struct {
	Stage    ModuleStage `json:"stage,omitempty"`
	LaneID   string      `json:"lane_id,omitempty"`
	SlotName string      `json:"slot_name"`
}

type ResolvedReferenceTarget struct {
	Stage        ModuleStage            `json:"stage"`
	LaneID       string                 `json:"lane_id,omitempty"`
	Module       string                 `json:"module"`
	Bindings     []ReferenceBinding     `json:"bindings,omitempty"`
	ReferenceSet contracts.ReferenceSet `json:"-"`
}

type ResolvedArtifactLane struct {
	ID                  string
	Extract             ModuleBinding
	Merge               ModuleBinding
	Normalize           ModuleBinding
	Validators          []ModuleBinding
	ExtractReferences   ResolvedReferenceTarget `json:"extract_references"`
	MergeReferences     ResolvedReferenceTarget `json:"merge_references"`
	NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}

type ResolvedValidatorChain struct {
	Stage      ModuleStage         `json:"stage"`
	LaneID     string              `json:"lane_id,omitempty"`
	ModuleKey  string              `json:"module_key"`
	Validators []ResolvedValidator `json:"validators"`
}

type ResolvedValidator struct {
	Binding        ModuleBinding            `json:"binding"`
	ExecutionClass contracts.ExecutionClass `json:"execution_class"`
}

type ResolvedPipeline struct {
	ID              string
	Digest          string
	Input           ModuleBinding
	Chunk           ModuleBinding
	ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
	ArtifactLanes   []ResolvedArtifactLane
	ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
	Output          ModuleBinding
}

type ModuleCatalog struct {
	Inputs          *InputAdapterRegistry
	Chunkers        *ChunkerRegistry
	Extractors      *ExtractorRegistry
	Mergers         *MergerRegistry
	Normalizers     *NormalizerRegistry
	Validators      *ValidatorRegistry
	ValidatorChains *ValidatorChainRegistry
	Outputs         *OutputEncoderRegistry
}

func Binding(module string) ModuleBinding {
	return ModuleBinding{Module: strings.TrimSpace(module)}
}

func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog ModuleCatalog) (ResolvedPipeline, error) {
	pipelineID := strings.TrimSpace(profile.ID)
	if pipelineID == "" {
		return ResolvedPipeline{}, fmt.Errorf("pipeline id must not be empty")
	}

	if len(profile.Artifacts) == 0 {
		return ResolvedPipeline{}, fmt.Errorf("pipeline %q must declare at least one artifact lane", pipelineID)
	}

	input := resolveBinding(profile.Input, "")
	if input.Module == "" {
		return ResolvedPipeline{}, fmt.Errorf("pipeline %q input module must not be empty", pipelineID)
	}
	inputModuleSpec, err := inputSpec(catalog, input.Module)
	if err != nil {
		return ResolvedPipeline{}, moduleLookupError(pipelineID, "", StageInput, input.Module, err)
	}

	capabilities := newCapabilitySet()
	if missing, ok := capabilities.missing(inputModuleSpec.Requires); ok {
		return ResolvedPipeline{}, capabilityError(pipelineID, "", StageInput, input.Module, missing)
	}
	capabilities.add(inputModuleSpec.Provides...)

	chunk := resolveBinding(profile.Chunk, DefaultChunkModule)
	chunkSpec, err := chunkerSpec(catalog, chunk.Module)
	if err != nil {
		return ResolvedPipeline{}, moduleLookupError(pipelineID, "", StageChunk, chunk.Module, err)
	}
	if missing, ok := capabilities.missing(chunkSpec.Requires); ok {
		return ResolvedPipeline{}, capabilityError(pipelineID, "", StageChunk, chunk.Module, missing)
	}
	capabilities.add(chunkSpec.Provides...)

	lanesByID, selectedLaneIDs, err := selectedArtifactLanes(pipelineID, profile.Artifacts, options)
	if err != nil {
		return ResolvedPipeline{}, err
	}
	if len(selectedLaneIDs) == 0 {
		return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
	}
	if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, lanesByID, catalog); err != nil {
		return ResolvedPipeline{}, err
	}

	chunkReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
		PipelineID:         pipelineID,
		Stage:              StageChunk,
		Module:             chunk.Module,
		Slots:              chunkSpec.ReferenceSlots,
		PipelineReferences: profile.References,
		LocalReferences:    chunk.References,
		Options:            options,
	})
	if err != nil {
		return ResolvedPipeline{}, err
	}
	resolved := ResolvedPipeline{
		ID:              pipelineID,
		Input:           input,
		Chunk:           chunk,
		ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
		Output:          resolveBinding(profile.Output, DefaultOutputModule),
	}
	chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
	if err != nil {
		return ResolvedPipeline{}, err
	}
	resolved.ValidatorChains = append(resolved.ValidatorChains, chunkValidatorChain)
	outputCapabilities := capabilities.clone()

	for _, laneID := range selectedLaneIDs {
		laneProfile := lanesByID[laneID]
		lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
		if err != nil {
			return ResolvedPipeline{}, err
		}
		resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
		resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
		outputCapabilities.addSet(laneCapabilities)
	}

	outputSpec, err := outputSpec(catalog, resolved.Output.Module)
	if err != nil {
		return ResolvedPipeline{}, moduleLookupError(pipelineID, "", StageOutput, resolved.Output.Module, err)
	}
	if missing, ok := outputCapabilities.missing(outputSpec.Requires); ok {
		return ResolvedPipeline{}, capabilityError(pipelineID, "", StageOutput, resolved.Output.Module, missing)
	}

	digest, err := resolvedPipelineDigest(resolved)
	if err != nil {
		return ResolvedPipeline{}, fmt.Errorf("pipeline %q digest: %w", pipelineID, err)
	}
	resolved.Digest = digest
	return resolved, nil
}

func resolveArtifactLane(
	pipelineID string,
	laneID string,
	profile ArtifactLaneProfile,
	pipelineReferences map[string]string,
	options ResolveOptions,
	inherited capabilitySet,
	catalog ModuleCatalog,
) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
	lane := ResolvedArtifactLane{
		ID:         laneID,
		Extract:    resolveBinding(profile.Extract, ""),
		Merge:      resolveBinding(profile.Merge, DefaultMergeModule),
		Normalize:  resolveBinding(profile.Normalize, DefaultNormalizeModule),
		Validators: resolveBindings(profile.Validators, ""),
	}
	if lane.Extract.Module == "" {
		return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
	}

	capabilities := inherited.clone()

	extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
	}
	if missing, ok := capabilities.missing(extractSpec.Requires); ok {
		return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
	}
	extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
	references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
		PipelineID:         pipelineID,
		LaneID:             laneID,
		Stage:              StageExtract,
		Module:             lane.Extract.Module,
		Slots:              extractSpec.ReferenceSlots,
		PipelineReferences: pipelineReferences,
		LocalReferences:    extractReferences,
		Options:            options,
	})
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, err
	}
	lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
	capabilities.add(extractSpec.Provides...)

	mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
	}
	if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
		return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
	}
	mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
		PipelineID:         pipelineID,
		LaneID:             laneID,
		Stage:              StageMerge,
		Module:             lane.Merge.Module,
		Slots:              mergeSpec.ReferenceSlots,
		PipelineReferences: pipelineReferences,
		LocalReferences:    lane.Merge.References,
		Options:            options,
	})
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, err
	}
	lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
	capabilities.add(mergeSpec.Provides...)

	normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
	}
	if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
		return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
	}
	normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
		PipelineID:         pipelineID,
		LaneID:             laneID,
		Stage:              StageNormalize,
		Module:             lane.Normalize.Module,
		Slots:              normalizeSpec.ReferenceSlots,
		PipelineReferences: pipelineReferences,
		LocalReferences:    lane.Normalize.References,
		Options:            options,
	})
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, err
	}
	lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
	capabilities.add(normalizeSpec.Provides...)

	if len(lane.Validators) > 0 {
		return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
	}

	extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, err
	}
	mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, err
	}
	normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
	if err != nil {
		return ResolvedArtifactLane{}, nil, nil, err
	}
	validatorChains := []ResolvedValidatorChain{extractValidatorChain, mergeValidatorChain, normalizeValidatorChain}

	return lane, validatorChains, capabilities, nil
}

func configuredValidatorsError(pipelineID string, laneID string) error {
	return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
}

func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
	chain := ResolvedValidatorChain{
		Stage:     stage,
		LaneID:    strings.TrimSpace(laneID),
		ModuleKey: strings.TrimSpace(module),
	}
	if chain.ModuleKey == "" {
		return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain %q module key must not be empty", pipelineID, stage)
	}
	switch stage {
	case StageChunk, StageExtract, StageMerge, StageNormalize:
	default:
		return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
	}

	var bindings []ModuleBinding
	if override.Set {
		bindings = cloneModuleBindings(override.Validators)
	} else if catalog.ValidatorChains != nil {
		bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
	}
	if len(bindings) == 0 {
		return chain, nil
	}
	if catalog.Validators == nil {
		return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator registry must not be nil for %s validator chain on module %q", pipelineID, stage, chain.ModuleKey)
	}
	chain.Validators = make([]ResolvedValidator, 0, len(bindings))
	for _, validator := range bindings {
		spec, ok := catalog.Validators.Spec(validator.Module)
		if !ok {
			return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
		}
		if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
			return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
		}
		chain.Validators = append(chain.Validators, ResolvedValidator{
			Binding:        cloneModuleBinding(validator),
			ExecutionClass: spec.ExecutionClass,
		})
	}
	return chain, nil
}

func cloneResolvedValidatorChains(chains []ResolvedValidatorChain) []ResolvedValidatorChain {
	if len(chains) == 0 {
		return nil
	}
	out := make([]ResolvedValidatorChain, len(chains))
	for i, chain := range chains {
		out[i] = ResolvedValidatorChain{
			Stage:      chain.Stage,
			LaneID:     strings.TrimSpace(chain.LaneID),
			ModuleKey:  strings.TrimSpace(chain.ModuleKey),
			Validators: cloneResolvedValidators(chain.Validators),
		}
	}
	return out
}

func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator {
	if len(validators) == 0 {
		return nil
	}
	out := make([]ResolvedValidator, len(validators))
	for i, validator := range validators {
		out[i] = ResolvedValidator{
			Binding:        cloneModuleBinding(validator.Binding),
			ExecutionClass: validator.ExecutionClass,
		}
	}
	return out
}

func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
	return ResolvedReferenceTarget{
		Stage:    stage,
		LaneID:   strings.TrimSpace(laneID),
		Module:   strings.TrimSpace(module),
		Bindings: append([]ReferenceBinding(nil), bindings...),
	}
}

func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
	if len(base) == 0 && len(override) == 0 {
		return nil
	}
	out := make(map[string]string, len(base)+len(override))
	for key, value := range base {
		out[key] = value
	}
	for key, value := range override {
		out[key] = value
	}
	return out
}

func validatePipelineReferenceDefaults(
	pipelineID string,
	pipelineReferences map[string]string,
	chunkSpec ModuleSpec,
	lanesByID map[string]ArtifactLaneProfile,
	catalog ModuleCatalog,
) error {
	normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
	if err != nil {
		return err
	}
	if len(normalizedPipelineReferences) == 0 {
		return nil
	}

	declaredByAnyTarget := make(map[string]struct{}, len(normalizedPipelineReferences))
	for _, slot := range chunkSpec.ReferenceSlots {
		declaredByAnyTarget[slot.Name] = struct{}{}
	}
	for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
		laneProfile := lanesByID[laneID]
		extract := resolveBinding(laneProfile.Extract, "")
		if extract.Module == "" {
			return fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
		}
		extractSpec, err := extractorSpec(catalog, extract.Module)
		if err != nil {
			return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
		}
		for _, slot := range extractSpec.ReferenceSlots {
			declaredByAnyTarget[slot.Name] = struct{}{}
		}

		merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
		mergeSpec, err := mergerSpec(catalog, merge.Module)
		if err != nil {
			return moduleLookupError(pipelineID, laneID, StageMerge, merge.Module, err)
		}
		for _, slot := range mergeSpec.ReferenceSlots {
			declaredByAnyTarget[slot.Name] = struct{}{}
		}

		normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
		normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
		if err != nil {
			return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
		}
		for _, slot := range normalizeSpec.ReferenceSlots {
			declaredByAnyTarget[slot.Name] = struct{}{}
		}
	}

	for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
		if _, ok := declaredByAnyTarget[slotName]; !ok {
			return fmt.Errorf("pipeline %q reference slot %q is not declared by any eligible reference target", pipelineID, slotName)
		}
	}
	return nil
}

type referenceResolutionTarget struct {
	PipelineID         string
	LaneID             string
	Stage              ModuleStage
	Module             string
	Slots              []contracts.ReferenceSlot
	PipelineReferences map[string]string
	LocalReferences    map[string]string
	Options            ResolveOptions
}

func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]ReferenceBinding, error) {
	slotByName := make(map[string]contracts.ReferenceSlot, len(target.Slots))
	for _, slot := range target.Slots {
		slotByName[slot.Name] = slot
	}

	bindings := make(map[string]ReferenceBinding)
	addBinding := func(slotName, source, bindingSource string) error {
		slotName = strings.TrimSpace(slotName)
		source = strings.TrimSpace(source)
		if slotName == "" {
			return fmt.Errorf("%s reference slot name must not be empty", referenceTargetErrorContext(target))
		}
		if source == "" {
			return fmt.Errorf("%s reference slot %q source must not be empty", referenceTargetErrorContext(target), slotName)
		}
		if _, ok := slotByName[slotName]; !ok {
			return fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetErrorContext(target), slotName, target.Stage, target.Module)
		}
		bindings[slotName] = ReferenceBinding{
			LaneID:        target.LaneID,
			SlotName:      slotName,
			Source:        source,
			BindingSource: bindingSource,
		}
		return nil
	}

	normalizedPipelineReferences, err := normalizedReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
	if err != nil {
		return nil, err
	}
	for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
		if _, ok := slotByName[slotName]; !ok {
			continue
		}
		if err := addBinding(slotName, normalizedPipelineReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
			return nil, err
		}
	}

	normalizedLocalReferences, err := normalizedReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
	if err != nil {
		return nil, err
	}
	for _, slotName := range sortedStringMapKeys(normalizedLocalReferences) {
		if err := addBinding(slotName, normalizedLocalReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
			return nil, err
		}
	}

	for _, override := range target.Options.ReferenceOverrides {
		match, err := referenceOverrideMatchesTarget(target, override)
		if err != nil {
			return nil, err
		}
		if !match {
			continue
		}
		source := override.BindingSource
		if strings.TrimSpace(source) == "" {
			source = contracts.ReferenceBindingSourceCLI
		}
		if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
			return nil, err
		}
	}

	for _, unbind := range target.Options.ReferenceUnbinds {
		match, err := referenceUnbindMatchesTarget(target, unbind)
		if err != nil {
			return nil, err
		}
		if !match {
			continue
		}
		slotName := strings.TrimSpace(unbind.SlotName)
		if slotName == "" {
			return nil, fmt.Errorf("%s reference unbind slot name must not be empty", referenceTargetErrorContext(target))
		}
		if _, ok := slotByName[slotName]; !ok {
			return nil, fmt.Errorf("%s reference slot %q is not declared", referenceTargetErrorContext(target), slotName)
		}
		delete(bindings, slotName)
	}

	for _, slot := range target.Slots {
		if slot.Required {
			if _, ok := bindings[slot.Name]; !ok {
				return nil, fmt.Errorf("%s required reference slot %q is not bound", referenceTargetErrorContext(target), slot.Name)
			}
		}
	}

	return sortedReferenceBindings(bindings), nil
}

func referenceOverrideMatchesTarget(target referenceResolutionTarget, override ReferenceBinding) (bool, error) {
	stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "override", override.Stage, override.LaneID)
	if err != nil {
		return false, err
	}
	return stage == target.Stage && laneID == target.LaneID, nil
}

func referenceUnbindMatchesTarget(target referenceResolutionTarget, unbind ReferenceUnbind) (bool, error) {
	stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "unbind", unbind.Stage, unbind.LaneID)
	if err != nil {
		return false, err
	}
	return stage == target.Stage && laneID == target.LaneID, nil
}

func normalizeReferenceOptionTarget(pipelineID string, operation string, stage ModuleStage, laneID string) (ModuleStage, string, error) {
	stage = ModuleStage(strings.TrimSpace(string(stage)))
	if stage == "" {
		stage = StageExtract
	}
	laneID = strings.TrimSpace(laneID)

	switch stage {
	case StageChunk:
		if laneID != "" {
			return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation)
		}
	case StageExtract, StageMerge, StageNormalize:
		if laneID == "" {
			return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation)
		}
	default:
		return "", "", fmt.Errorf("pipeline %q reference %s stage %q is not supported", pipelineID, operation, stage)
	}
	return stage, laneID, nil
}

func sortedReferenceBindings(bindings map[string]ReferenceBinding) []ReferenceBinding {
	keys := sortedReferenceBindingKeys(bindings)
	resolved := make([]ReferenceBinding, 0, len(keys))
	for _, slotName := range keys {
		resolved = append(resolved, bindings[slotName])
	}
	return resolved
}

func referenceTargetErrorContext(target referenceResolutionTarget) string {
	if target.LaneID != "" {
		return fmt.Sprintf("pipeline %q lane %q %s module %q", target.PipelineID, target.LaneID, target.Stage, target.Module)
	}
	return fmt.Sprintf("pipeline %q %s module %q", target.PipelineID, target.Stage, target.Module)
}

func referenceTargetSlotLabel(target referenceResolutionTarget) string {
	if target.LaneID != "" {
		return fmt.Sprintf("pipeline %q lane %q %s reference slot", target.PipelineID, target.LaneID, target.Stage)
	}
	return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
}

func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
	if len(values) == 0 {
		return nil, nil
	}
	out := make(map[string]string, len(values))
	for rawSlotName, rawSource := range values {
		slotName := strings.TrimSpace(rawSlotName)
		if slotName == "" {
			return nil, fmt.Errorf("%s must not be empty", keyName)
		}
		if _, ok := out[slotName]; ok {
			return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
		}
		source := strings.TrimSpace(rawSource)
		if source == "" {
			return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
		}
		out[slotName] = source
	}
	return out, nil
}

func sortedArtifactLaneProfileKeys(values map[string]ArtifactLaneProfile) []string {
	if len(values) == 0 {
		return nil
	}
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func sortedStringMapKeys(values map[string]string) []string {
	if len(values) == 0 {
		return nil
	}
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
	if len(values) == 0 {
		return nil
	}
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}

func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
	module := strings.TrimSpace(binding.Module)
	if module == "" {
		module = defaultModule
	}
	llmProfile := strings.TrimSpace(binding.LLMProfile)
	return ModuleBinding{
		Module:     module,
		LLMProfile: llmProfile,
		Retries:    binding.Retries,
		Options:    cloneOptions(binding.Options),
		References: normalizeReferenceMap(binding.References),
		Validators: cloneValidatorOverride(binding.Validators),
	}
}

func resolveBindings(bindings []ModuleBinding, defaultModule string) []ModuleBinding {
	if len(bindings) == 0 {
		return nil
	}

	resolved := make([]ModuleBinding, 0, len(bindings))
	for _, binding := range bindings {
		resolvedBinding := resolveBinding(binding, defaultModule)
		resolved = append(resolved, resolvedBinding)
	}
	return resolved
}

func cloneOptions(options map[string]any) map[string]any {
	if len(options) == 0 {
		return nil
	}

	copied := make(map[string]any, len(options))
	for key, value := range options {
		copied[key] = value
	}
	return copied
}

func normalizeReferenceMap(values map[string]string) map[string]string {
	if len(values) == 0 {
		return nil
	}
	out := make(map[string]string, len(values))
	keys := make([]string, 0, len(values))
	rawByNormalized := make(map[string]string, len(values))
	for rawKey := range values {
		key := strings.TrimSpace(rawKey)
		rawByNormalized[key] = rawKey
		keys = append(keys, key)
	}
	sort.Strings(keys)
	for _, key := range keys {
		out[key] = strings.TrimSpace(values[rawByNormalized[key]])
	}
	return out
}

func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
	lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
	for rawLaneID, lane := range artifacts {
		laneID := strings.TrimSpace(rawLaneID)
		if laneID == "" {
			return nil, nil, fmt.Errorf("pipeline %q artifact lane id must not be empty", pipelineID)
		}
		if _, ok := lanesByID[laneID]; ok {
			return nil, nil, fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", pipelineID, laneID)
		}
		lanesByID[laneID] = lane
	}

	if len(options.Only) == 0 {
		keys := make([]string, 0, len(lanesByID))
		for laneID := range lanesByID {
			keys = append(keys, laneID)
		}
		sort.Strings(keys)
		return lanesByID, keys, nil
	}

	selected := make(map[string]struct{}, len(options.Only))
	for _, rawLaneID := range options.Only {
		laneID := strings.TrimSpace(rawLaneID)
		if laneID == "" {
			return nil, nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", pipelineID)
		}
		if _, ok := lanesByID[laneID]; !ok {
			return nil, nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", pipelineID, laneID)
		}
		selected[laneID] = struct{}{}
	}

	keys := make([]string, 0, len(selected))
	for laneID := range selected {
		keys = append(keys, laneID)
	}
	sort.Strings(keys)
	return lanesByID, keys, nil
}

func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
	withoutDigest := struct {
		ID              string
		Input           ModuleBinding
		Chunk           ModuleBinding
		ChunkReferences ResolvedReferenceTarget
		ArtifactLanes   []ResolvedArtifactLane
		Output          ModuleBinding
	}{
		ID:              resolved.ID,
		Input:           resolved.Input,
		Chunk:           resolved.Chunk,
		ChunkReferences: resolved.ChunkReferences,
		ArtifactLanes:   resolved.ArtifactLanes,
		Output:          resolved.Output,
	}
	encoded, err := json.Marshal(withoutDigest)
	if err != nil {
		return "", err
	}

	sum := sha256.Sum256(encoded)
	return "sha256:" + hex.EncodeToString(sum[:]), nil
}

func inputSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
	return registrySpec(catalog.Inputs, key)
}

func chunkerSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
	return registrySpec(catalog.Chunkers, key)
}

func extractorSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
	return registrySpec(catalog.Extractors, key)
}

func mergerSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
	return registrySpec(catalog.Mergers, key)
}

func normalizerSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
	return registrySpec(catalog.Normalizers, key)
}

func outputSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
	return registrySpec(catalog.Outputs, key)
}

type specRegistry interface {
	Spec(key string) (ModuleSpec, bool)
}

func registrySpec(registry specRegistry, key string) (ModuleSpec, error) {
	if registry == nil {
		return ModuleSpec{}, fmt.Errorf("module %q is not registered", key)
	}
	spec, ok := registry.Spec(key)
	if !ok {
		return ModuleSpec{}, fmt.Errorf("module %q is not registered", key)
	}
	return spec, nil
}

func moduleLookupError(pipelineID, laneID string, stage ModuleStage, module string, err error) error {
	if laneID != "" {
		return fmt.Errorf("pipeline %q lane %q %s module %q: %w", pipelineID, laneID, stage, module, err)
	}
	return fmt.Errorf("pipeline %q %s module %q: %w", pipelineID, stage, module, err)
}

func capabilityError(pipelineID, laneID string, stage ModuleStage, module, capability string) error {
	if laneID != "" {
		return fmt.Errorf("pipeline %q lane %q %s module %q requires missing capability %q", pipelineID, laneID, stage, module, capability)
	}
	return fmt.Errorf("pipeline %q %s module %q requires missing capability %q", pipelineID, stage, module, capability)
}

type capabilitySet map[string]struct{}

func newCapabilitySet() capabilitySet {
	return make(capabilitySet)
}

func (set capabilitySet) clone() capabilitySet {
	copied := make(capabilitySet, len(set))
	for capability := range set {
		copied[capability] = struct{}{}
	}
	return copied
}

func (set capabilitySet) add(values ...string) {
	for _, value := range values {
		set[value] = struct{}{}
	}
}

func (set capabilitySet) addSet(other capabilitySet) {
	for value := range other {
		set[value] = struct{}{}
	}
}

func (set capabilitySet) missing(required []string) (string, bool) {
	for _, capability := range required {
		if _, ok := set[capability]; !ok {
			return capability, true
		}
	}
	return "", false
}