notarius/internal/core/config/pipeline_document.go

package config

import (
	"sort"
	"strings"

	configkit "gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)

// documentLoadState keeps the spelling supplied by the user while resolver
// operations move map entries to their canonical keys. It is intentionally
// load-local: source-location handling consumes it during a later resolver
// pass and it never becomes runtime configuration state.
type documentLoadState struct {
	originalPaths       map[string]configkit.Path
	mainDirectory       string
	invocationDirectory string
}

func newDocumentLoadState(invocationDirectory string) *documentLoadState {
	return &documentLoadState{originalPaths: make(map[string]configkit.Path), invocationDirectory: cleanInvocationDirectory(invocationDirectory)}
}

func (state *documentLoadState) rememberPath(normalized, original configkit.Path) {
	if state != nil {
		state.originalPaths[normalized.String()] = original
	}
}

// originalPath follows recorded map-key moves back to the spelling present in
// the source. Resolver code that needs a source coordinate must use this
// rather than reconstructing a path from a normalized identifier.
func (state *documentLoadState) originalPath(path configkit.Path) configkit.Path {
	if state == nil || len(state.originalPaths) == 0 {
		return path
	}
	value := path.String()
	for {
		changed := false
		segments := strings.Split(strings.TrimPrefix(value, "/"), "/")
		for length := 1; length <= len(segments); length++ {
			prefix := "/" + strings.Join(segments[:length], "/")
			original, ok := state.originalPaths[prefix]
			if !ok {
				continue
			}
			value = original.String() + strings.TrimPrefix(value, prefix)
			changed = true
			break
		}
		if !changed {
			break
		}
	}
	if original, err := configkit.ParsePath(value); err == nil {
		return original
	}
	return path
}

func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
	if document == nil {
		return
	}
	validateDocumentPipelinePresence(document, resolution)
	normalizer := documentNormalizer{Resolution: resolution}

	pipelinesPath := configkit.Keys("pipelines")
	for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
		profile := document.Pipelines[id]
		normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state)
		document.Pipelines[id] = profile
	}
	document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state)
}

func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) {
	pipelinesPath := configkit.Keys("pipelines")
	for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
		profile := document.Pipelines[id]
		path := pipelinesPath.Key(id)
		validatePipelineDocumentPresence(profile, path, resolution)
	}
}

func validatePipelineDocumentPresence(profile PipelineDocument, path configkit.Path, resolution *configkit.Resolution) {
	rejectDocumentNull(resolution, path.Key("llm_profile"), profile.LLMProfile == nil)
	rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), profile.StructuredOutputRepairAttempts == nil)
	rejectDocumentNull(resolution, path.Key("validation_policy"), profile.ValidationPolicy == nil)
	validateDocumentPolicyPresence(profile.ValidationPolicy, path.Key("validation_policy"), resolution)

	stepsPath := path.Key("steps")
	if resolution.IsExplicit(stepsPath) && profile.Steps == nil {
		resolution.Error(stepsPath, "must not be null")
	}
	if resolution.IsExplicit(stepsPath) && len(profile.Steps) == 0 {
		resolution.Error(stepsPath, "must contain at least one ordered step")
	}
	if resolution.IsExplicit(stepsPath) && resolution.IsExplicit(path.Key("artifacts")) {
		resolution.Error(stepsPath, "must not be declared with artifacts", path.Key("artifacts"))
	}

	validateBindingDocumentPresence(profile.Input, path.Key("input"), resolution)
	validateOptionalBindingDocumentPresence(profile.Chunk, path.Key("chunk"), resolution)
	validateOptionalBindingDocumentPresence(profile.Output, path.Key("output"), resolution)
	validateReferenceDocumentMapPresence(profile.References, path.Key("references"), resolution)
	for _, laneID := range sortedLaneDocumentKeys(profile.Artifacts) {
		validateArtifactLaneDocumentPresence(profile.Artifacts[laneID], path.Key("artifacts").Key(laneID), resolution)
	}
	for index, step := range profile.Steps {
		stepPath := stepsPath.Index(index)
		validateReferenceDocumentMapPresence(step.References, stepPath.Key("references"), resolution)
		for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
			validateArtifactLaneDocumentPresence(step.Artifacts[laneID], stepPath.Key("artifacts").Key(laneID), resolution)
		}
	}
}

func validateArtifactLaneDocumentPresence(lane ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution) {
	if resolution.IsExplicit(path.Key("references")) {
		resolution.Error(path.Key("references"), "artifact-lane references are not supported; use binding references")
	}
	validateBindingDocumentPresence(lane.Extract, path.Key("extract"), resolution)
	validateOptionalBindingDocumentPresence(lane.Merge, path.Key("merge"), resolution)
	validateOptionalBindingDocumentPresence(lane.Normalize, path.Key("normalize"), resolution)
}

func validateOptionalBindingDocumentPresence(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution) {
	if binding == nil {
		if resolution.IsExplicit(path) {
			resolution.Error(path, "must not be null when configured")
		}
		return
	}
	validateBindingDocumentPresence(*binding, path, resolution)
}

func validateBindingDocumentPresence(binding BindingDocument, path configkit.Path, resolution *configkit.Resolution) {
	rejectDocumentNull(resolution, path.Key("llm_profile"), binding.LLMProfile == nil)
	rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), binding.StructuredOutputRepairAttempts == nil)
	rejectDocumentNull(resolution, path.Key("validation_policy"), binding.ValidationPolicy == nil)
	validateDocumentPolicyPresence(binding.ValidationPolicy, path.Key("validation_policy"), resolution)
	if binding.Validators == nil && resolution.IsExplicit(path.Key("validators")) {
		resolution.Error(path.Key("validators"), "must not be null")
	}
	validateReferenceDocumentMapPresence(binding.References, path.Key("references"), resolution)
	for index, validator := range binding.Validators {
		validatorPath := path.Key("validators").Index(index)
		rejectDocumentNull(resolution, validatorPath.Key("llm_profile"), validator.LLMProfile == nil)
		rejectDocumentNull(resolution, validatorPath.Key("structured_output_repair_attempts"), validator.StructuredOutputRepairAttempts == nil)
	}
}

func validateReferenceDocumentMapPresence(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution) {
	for _, name := range sortedReferenceDocumentKeys(references) {
		source := references[name]
		sourcePath := path.Key(name)
		rejectDocumentNull(resolution, sourcePath.Key("path"), source.Path == nil)
		rejectDocumentNull(resolution, sourcePath.Key("artifact"), source.Artifact == nil)
	}
}

func rejectDocumentNull(resolution *configkit.Resolution, path configkit.Path, isNil bool) {
	if isNil && resolution.IsExplicit(path) {
		resolution.Error(path, "must not be null")
	}
}

func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path configkit.Path, resolution *configkit.Resolution) {
	if policy == nil {
		return
	}
	rejectDocumentNull(resolution, path.Key("producer_structural_failure"), policy.ProducerStructuralFailure == nil)
	rejectDocumentNull(resolution, path.Key("semantic_rejection"), policy.SemanticRejection == nil)
	rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
}

func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
	normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
	normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
	if profile.Chunk != nil {
		normalizeDocumentBinding(profile.Chunk, path.Key("chunk"), resolution, state)
	}
	if profile.Output != nil {
		normalizeDocumentBinding(profile.Output, path.Key("output"), resolution, state)
	}
	profile.References = normalizeDocumentReferences(profile.References, path.Key("references"), resolution, state)
	for _, laneID := range sortedLaneDocumentKeys(profile.Artifacts) {
		lane := profile.Artifacts[laneID]
		normalizeArtifactLaneDocument(&lane, path.Key("artifacts").Key(laneID), resolution, state)
		profile.Artifacts[laneID] = lane
	}
	profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
	resolution = resolution.inSequence(path.Key("steps"))
	for index := range profile.Steps {
		step := &profile.Steps[index]
		stepPath := path.Key("steps").Index(index)
		normalizeDocumentValue(resolution, stepPath.Key("id"), &step.ID, strings.TrimSpace(step.ID), "trimmed step identifier")
		step.References = normalizeDocumentReferences(step.References, stepPath.Key("references"), resolution, state)
		for _, laneID := range sortedLaneDocumentKeys(step.Artifacts) {
			lane := step.Artifacts[laneID]
			normalizeArtifactLaneDocument(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
			step.Artifacts[laneID] = lane
		}
		step.Artifacts = normalizeDocumentMap(step.Artifacts, stepPath.Key("artifacts"), "artifact lane identifier", resolution, state)
	}
}

func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
	normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state)
	if lane.Merge != nil {
		normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state)
	}
	if lane.Normalize != nil {
		normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
	}
	resolution = resolution.inSequence(path.Key("validators"))
	for index := range lane.Validators {
		normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
	}
}

func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
	normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
	normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
	binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
	resolution = resolution.inSequence(path.Key("validators"))
	for index := range binding.Validators {
		normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
	}
}

func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) {
	normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier")
	normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile")
}

func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument {
	for _, name := range sortedReferenceDocumentKeys(references) {
		source := references[name]
		sourcePath := path.Key(name)
		normalizePipelineDocumentString(resolution, sourcePath.Key("path"), &source.Path, "trimmed reference path")
		if source.Artifact != nil {
			normalizeDocumentValue(resolution, sourcePath.Key("artifact").Key("step"), &source.Artifact.Step, strings.TrimSpace(source.Artifact.Step), "trimmed artifact step selector")
			normalizeDocumentValue(resolution, sourcePath.Key("artifact").Key("lane"), &source.Artifact.Lane, strings.TrimSpace(source.Artifact.Lane), "trimmed artifact lane selector")
		}
		references[name] = source
	}
	return normalizeDocumentMap(references, path, "reference name", resolution, state)
}

func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) {
	if target == nil || !resolution.IsExplicit(path) {
		return
	}
	if *target != value {
		resolution.Record(path, configkit.ChangeNormalization, reason)
		*target = value
	}
}

func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) {
	if target == nil || *target == nil || !resolution.IsExplicit(path) {
		return
	}
	value := strings.TrimSpace(**target)
	if **target != value {
		resolution.Record(path, configkit.ChangeNormalization, reason)
		**target = value
	}
}

func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T {
	if len(values) == 0 {
		return values
	}
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	canonical := make(map[string]string, len(values))
	valid := true
	for _, key := range keys {
		trimmed := strings.TrimSpace(key)
		if trimmed == "" {
			resolution.Error(path.Key(key), kind+" must not be empty")
			valid = false
			continue
		}
		if first, exists := canonical[trimmed]; exists {
			resolution.Error(path.Key(key), kind+" collides after whitespace normalization", path.Key(first))
			valid = false
			continue
		}
		canonical[trimmed] = key
	}
	if !valid {
		return values
	}
	result := make(map[string]T, len(values))
	changed := false
	dependencies := make([]configkit.Path, 0, len(keys))
	for _, key := range keys {
		trimmed := strings.TrimSpace(key)
		result[trimmed] = values[key]
		dependencies = append(dependencies, path.Key(key))
		if trimmed != key {
			changed = true
			state.rememberPath(path.Key(trimmed), path.Key(key))
		}
	}
	if changed {
		resolution.Record(path, configkit.ChangeNormalization, "normalized "+kind+" keys", dependencies...)
		values = result
	}
	return values
}

func validateDocumentPipelines(document *Document, validation *configkit.Validation) {
	if document == nil {
		return
	}
	pipelinesPath := configkit.Keys("pipelines")
	for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
		profile := document.Pipelines[id]
		path := pipelinesPath.Key(id)
		validatePipelineDocument(profile, path, validation)
	}
}

func validatePipelineDocument(profile PipelineDocument, path configkit.Path, validation *configkit.Validation) {
	validateSelectedDocumentProfile(profile.LLMProfile, path.Key("llm_profile"), validation)
	validateDocumentRepairAttempts(profile.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
	validateDocumentPolicy(profile.ValidationPolicy, path.Key("validation_policy"), validation)
	validateDocumentBinding(profile.Input, path.Key("input"), "input", false, false, validation)
	if profile.Chunk != nil {
		validateDocumentBinding(*profile.Chunk, path.Key("chunk"), "chunk", true, true, validation)
	}
	if profile.Output != nil {
		validateDocumentBinding(*profile.Output, path.Key("output"), "output", false, false, validation)
	}
	validateDocumentReferences(profile.References, path.Key("references"), false, validation)

	seenLanes := make(map[string]configkit.Path)
	if profile.Steps != nil {
		seenSteps := make(map[string]configkit.Path)
		for index, step := range profile.Steps {
			stepPath := path.Key("steps").Index(index)
			if strings.TrimSpace(step.ID) == "" {
				validation.Error(stepPath.Key("id"), "must not be empty")
			} else if first, exists := seenSteps[step.ID]; exists {
				validation.Error(stepPath.Key("id"), "must be unique", first)
			} else {
				seenSteps[step.ID] = stepPath.Key("id")
			}
			validateDocumentReferences(step.References, stepPath.Key("references"), true, validation)
			validateDocumentLanes(step.Artifacts, stepPath.Key("artifacts"), seenLanes, validation)
		}
		return
	}
	validateDocumentLanes(profile.Artifacts, path.Key("artifacts"), seenLanes, validation)
}

func validateDocumentLanes(lanes map[string]ArtifactLaneDocument, path configkit.Path, seen map[string]configkit.Path, validation *configkit.Validation) {
	for _, id := range sortedLaneDocumentKeys(lanes) {
		lanePath := path.Key(id)
		if first, exists := seen[id]; exists {
			validation.Error(lanePath, "artifact lane identifier must be unique across steps", first)
		} else {
			seen[id] = lanePath
		}
		lane := lanes[id]
		validateDocumentBinding(lane.Extract, lanePath.Key("extract"), "extract", true, true, validation)
		if lane.Merge != nil {
			validateDocumentBinding(*lane.Merge, lanePath.Key("merge"), "merge", true, true, validation)
		}
		if lane.Normalize != nil {
			validateDocumentBinding(*lane.Normalize, lanePath.Key("normalize"), "normalize", true, true, validation)
		}
		if len(lane.Validators) > 0 {
			validation.Error(lanePath.Key("validators"), "artifact-lane validators are not supported; attach validators to a binding")
		}
	}
}

func validateDocumentBinding(binding BindingDocument, path configkit.Path, slot string, referencesAllowed, validatorsAllowed bool, validation *configkit.Validation) {
	if strings.TrimSpace(binding.Module) == "" {
		validation.Error(path.Key("module"), "must not be empty")
	}
	validateSelectedDocumentProfile(binding.LLMProfile, path.Key("llm_profile"), validation)
	validateDocumentRepairAttempts(binding.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
	if binding.Retries < 0 {
		validation.Error(path.Key("retries"), "must be greater than or equal to zero")
	}
	if binding.ValidationPolicy != nil {
		if slot != "chunk" && slot != "extract" && slot != "merge" && slot != "normalize" {
			validation.Error(path.Key("validation_policy"), "is not supported for this binding")
		}
		validateDocumentPolicy(binding.ValidationPolicy, path.Key("validation_policy"), validation)
	}
	if len(binding.References) > 0 && !referencesAllowed {
		validation.Error(path.Key("references"), "are not supported for this binding")
	}
	if referencesAllowed {
		validateDocumentReferences(binding.References, path.Key("references"), true, validation)
	}
	if binding.Validators != nil && !validatorsAllowed {
		validation.Error(path.Key("validators"), "are not supported for this binding")
	}
	for index, validator := range binding.Validators {
		validateValidatorDocument(validator, path.Key("validators").Index(index), validation)
	}
}

func validateValidatorDocument(validator ValidatorDocument, path configkit.Path, validation *configkit.Validation) {
	if strings.TrimSpace(validator.Module) == "" {
		validation.Error(path.Key("module"), "must not be empty")
	}
	validateSelectedDocumentProfile(validator.LLMProfile, path.Key("llm_profile"), validation)
	validateDocumentRepairAttempts(validator.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
	if validator.Retries < 0 {
		validation.Error(path.Key("retries"), "must be greater than or equal to zero")
	}
}

func validateDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, generatedAllowed bool, validation *configkit.Validation) {
	for _, name := range sortedReferenceDocumentKeys(references) {
		source := references[name]
		sourcePath := path.Key(name)
		hasPath := source.Path != nil
		hasArtifact := source.Artifact != nil
		if hasPath == hasArtifact {
			validation.Error(sourcePath, "must select exactly one reference source")
			continue
		}
		if hasPath {
			if strings.TrimSpace(*source.Path) == "" {
				validation.Error(sourcePath.Key("path"), "must not be empty")
			}
			continue
		}
		if !generatedAllowed {
			validation.Error(sourcePath.Key("artifact"), "is not supported here; use an external path")
			continue
		}
		if strings.TrimSpace(source.Artifact.Step) == "" {
			validation.Error(sourcePath.Key("artifact").Key("step"), "must not be empty")
		}
		if strings.TrimSpace(source.Artifact.Lane) == "" {
			validation.Error(sourcePath.Key("artifact").Key("lane"), "must not be empty")
		}
	}
}

func validateSelectedDocumentProfile(value *string, path configkit.Path, validation *configkit.Validation) {
	if value != nil && strings.TrimSpace(*value) == "" {
		validation.Error(path, "must not be empty when set")
	}
}

func validateDocumentRepairAttempts(value *int, path configkit.Path, validation *configkit.Validation) {
	if value != nil && (*value < 0 || *value > 3) {
		validation.Error(path, "must be between zero and three")
	}
}

func validateDocumentPolicy(policy *ValidationPolicyDocument, path configkit.Path, validation *configkit.Validation) {
	if policy == nil {
		return
	}
	if policy.ProducerStructuralFailure != nil {
		switch *policy.ProducerStructuralFailure {
		case pipeline.ProducerStructuralFailureFailRun, pipeline.ProducerStructuralFailureRejectOutput:
		default:
			validation.Error(path.Key("producer_structural_failure"), "must be fail_run or reject_output")
		}
	}
	if policy.SemanticRejection != nil {
		switch *policy.SemanticRejection {
		case pipeline.SemanticRejectionFailRun, pipeline.SemanticRejectionRejectOutput:
		default:
			validation.Error(path.Key("semantic_rejection"), "must be fail_run or reject_output")
		}
	}
	if policy.ValidatorFailure != nil {
		switch *policy.ValidatorFailure {
		case pipeline.ValidatorFailureWarnContinue, pipeline.ValidatorFailureFailRun:
		default:
			validation.Error(path.Key("validator_failure"), "must be warn_continue or fail_run")
		}
	}
}

func sortedPipelineDocumentKeys(values map[string]PipelineDocument) []string {
	return sortedDocumentKeys(values)
}
func sortedLaneDocumentKeys(values map[string]ArtifactLaneDocument) []string {
	return sortedDocumentKeys(values)
}
func sortedReferenceDocumentKeys(values map[string]ReferenceDocument) []string {
	return sortedDocumentKeys(values)
}

func sortedDocumentKeys[T any](values map[string]T) []string {
	keys := make([]string, 0, len(values))
	for key := range values {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}