ConfigKit/internal/engine/imports.go

package engine

import (
	"bytes"
	"errors"
	"io/fs"
	"strconv"
	"strings"

	"gitea.maximumdirect.net/eric/ConfigKit/internal/document"
	yamladapter "gitea.maximumdirect.net/eric/ConfigKit/internal/yaml"
)

const (
	maxImportDepth   = 64
	maxImportSources = 1024
	maxGlobEntries   = 1_000_000
)

type importLimits struct {
	depth   int
	sources int
	bytes   int64
	nodes   int
	entries int
}

func defaultImportLimits() importLimits {
	return importLimits{
		depth: maxImportDepth, sources: maxImportSources,
		bytes: maxTotalBytes, nodes: maxDocumentNodes, entries: maxGlobEntries,
	}
}

type importRoute struct {
	canonical string
	locations []Location
}

type importAccounting struct {
	sources int
	bytes   int64
	nodes   int
	entries int
}

type importTraversal struct {
	access       *filesystemAccess
	accounting   *importAccounting
	limits       importLimits
	rule         *VersionRule
	versionState *sourceVersionState

	scope   acquisitionScope
	sources []SourceInfo
	active  map[resourceIdentity]int
	visited map[resourceIdentity]importRoute
	stack   []importRoute
}

func loadImportedDocuments(source Source, context *FilesystemContext, main parsedSource, rule *VersionRule, limits importLimits) (*composedDocument, []SourceInfo, []Diagnostic) {
	return loadImportedDocumentsTracked(source, context, main, Path{}, rule, limits, &importAccounting{}, nil, nil)
}

func loadImportedDocumentsTracked(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken) (*composedDocument, []SourceInfo, []Diagnostic) {
	return loadImportedDocumentsTrackedWithVersion(source, context, main, destination, rule, limits, accounting, visited, hostToken, nil)
}

func loadImportedDocumentsTrackedWithVersion(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
	var processing *sourceVersionProcessingResult
	if versionState != nil && versionState.mode != sourceVersionModeDefault {
		processing = versionState.mainProcessing
	}
	return loadImportedDocumentsTrackedWithProcessing(source, context, main, destination, rule, limits, accounting, visited, hostToken, versionState, processing)
}

func loadImportedDocumentsTrackedWithProcessing(source Source, context *FilesystemContext, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, hostToken *namespaceToken, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) {
	if nativeContext(source, context) {
		access, basePath, mainPath, diagnostic := nativeFilesystemAccess(source, context)
		if diagnostic != nil {
			return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
		}
		return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
	}
	rootPath, basePath, mainPath, diagnostic := importFilesystem(source, context, main)
	if diagnostic != nil {
		return nil, sourceInfos(main.source), []Diagnostic{*diagnostic}
	}
	access, err := openHostAccess(rootPath)
	if err != nil {
		diagnostic := globFilesystemDiagnostic(err, "configuration root could not be opened", Location{})
		diagnostic.Primary = nil
		return nil, sourceInfos(main.source), []Diagnostic{diagnostic}
	}
	if hostToken != nil {
		access.token = hostToken
	}
	return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
}

func loadImportedWithAccess(access *filesystemAccess, basePath, mainPath string, main parsedSource, rule *VersionRule, limits importLimits, accounting *importAccounting) (*composedDocument, []SourceInfo, []Diagnostic) {
	return loadImportedWithAccessTracked(access, basePath, mainPath, main, Path{}, rule, limits, accounting, nil)
}

func loadImportedWithAccessTracked(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute) (*composedDocument, []SourceInfo, []Diagnostic) {
	return loadImportedWithAccessTrackedWithVersion(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, nil)
}

func loadImportedWithAccessTrackedWithVersion(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState) (*composedDocument, []SourceInfo, []Diagnostic) {
	var processing *sourceVersionProcessingResult
	if versionState != nil && versionState.mode != sourceVersionModeDefault {
		processing = versionState.mainProcessing
	}
	return loadImportedWithAccessTrackedAndProcessing(access, basePath, mainPath, main, destination, rule, limits, accounting, visited, versionState, processing)
}

func loadImportedWithAccessTrackedAndProcessing(access *filesystemAccess, basePath, mainPath string, main parsedSource, destination Path, rule *VersionRule, limits importLimits, accounting *importAccounting, visited map[resourceIdentity]importRoute, versionState *sourceVersionState, processing *sourceVersionProcessingResult) (*composedDocument, []SourceInfo, []Diagnostic) {
	if !access.contains(basePath) || (mainPath != "" && !access.contains(mainPath)) {
		_ = access.close()
		return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "selected source and import base must stay inside the configuration root", nil)}
	}
	for _, directory := range []string{access.rootPath, basePath} {
		info, err := access.stat(directory)
		if err != nil || !info.IsDir() {
			_ = access.close()
			return nil, sourceInfos(main.source), []Diagnostic{importDiagnostic(CodeImportInvalid, "configuration root and import base must be directories", nil)}
		}
	}
	mainInfo := cloneSourceInfo(*main.source)
	mainInfo.Destination = destination.clone()
	if mainPath != "" {
		mainInfo.Path = mainPath
	}
	mainKey := access.identity(mainPath)
	if mainPath == "" {
		mainKey = access.syntheticIdentity()
	}
	if previous, repeated := visited[mainKey]; repeated {
		_ = access.close()
		return nil, sourceInfos(main.source), []Diagnostic{importRepeatedDiagnostic(Location{SourceID: main.source.ID}, previous, importRoute{})}
	}
	accounting.sources++
	accounting.bytes += main.bytes
	accounting.nodes += countInputNodes(main.document)
	state := importTraversal{
		access: access, accounting: accounting, limits: limits, rule: rule, versionState: versionState,
		scope:   acquisitionScope{destination: destination.clone()},
		sources: []SourceInfo{mainInfo},
		active:  make(map[resourceIdentity]int), visited: visited,
	}
	if state.visited == nil {
		state.visited = make(map[resourceIdentity]importRoute)
	}
	if diagnostic := state.checkInitialLimits(); diagnostic != nil {
		_ = access.close()
		return nil, cloneSourceInfos(state.sources), []Diagnostic{*diagnostic}
	}

	mainIdentity := mainPath
	if mainIdentity == "" {
		mainIdentity = "<selected-source>"
	}
	mainRoute := importRoute{canonical: mainIdentity}
	state.active[mainKey] = 0
	state.visited[mainKey] = mainRoute
	state.stack = append(state.stack, mainRoute)
	var body *document.Node
	var declarations []importDeclaration
	var diagnostics []Diagnostic
	if processing != nil {
		body = processing.payload
		declarations = processing.imports
	} else {
		body, declarations, diagnostics = splitImportDirectives(main.document)
	}
	if len(diagnostics) == 0 {
		_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
		placementNodes := destinationWrapperNodeCount(destination)
		if err != nil {
			diagnostics = []Diagnostic{sourceDiagnostic(CodeSourceLimitExceeded, main.source.ID, "configuration exceeds the source depth or node limit after destination placement")}
		} else if accounting.nodes+placementNodes > limits.nodes {
			diagnostics = []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)}
		} else {
			accounting.nodes += placementNodes
			state.scope.contributions = append(state.scope.contributions, acquiredContribution{
				source: cloneSourceInfo(mainInfo), payload: body, document: placed,
				destination: destination.clone(), processing: processing,
			})
			diagnostics = state.walkDeclarations(basePath, declarations, 0, destination)
		}
	}

	closeErr := access.close()
	if len(diagnostics) != 0 {
		return nil, cloneSourceInfos(state.sources), diagnostics
	}
	if closeErr != nil {
		return nil, cloneSourceInfos(state.sources), []Diagnostic{importDiagnostic(CodeImportRead, "configuration root could not be closed", nil)}
	}
	documents := make([]*document.Node, 0, len(state.scope.contributions))
	for _, contribution := range state.scope.contributions {
		documents = append(documents, contribution.document)
	}
	composed, diagnostics := composeDocuments(documents...)
	if composed != nil {
		composed.acquisition = []acquisitionScope{cloneAcquisitionScope(state.scope)}
		composed.inputBytes = accounting.bytes
		composed.inputNodes = accounting.nodes
	}
	return composed, cloneSourceInfos(state.sources), diagnostics
}

func (s *importTraversal) checkInitialLimits() *Diagnostic {
	if s.accounting.sources > s.limits.sources {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", nil)
		return &value
	}
	if s.accounting.bytes > s.limits.bytes {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate byte limit", nil)
		return &value
	}
	if s.accounting.nodes > s.limits.nodes {
		value := importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", nil)
		return &value
	}
	return nil
}

func (s *importTraversal) walkDeclarations(basePath string, declarations []importDeclaration, depth int, parentDestination Path) []Diagnostic {
	for _, declaration := range declarations {
		childDestination := appendDestination(parentDestination, declaration.destination)
		if declaration.glob != "" {
			matches, diagnostics := s.expandImportGlob(basePath, declaration)
			if len(diagnostics) != 0 {
				return diagnostics
			}
			if len(matches) == 0 {
				if declaration.optional {
					continue
				}
				location := declaration.location
				return []Diagnostic{importDiagnostic(CodeImportMissing, "required import glob matched no files", &location)}
			}
			for _, match := range matches {
				matched := declaration
				matched.path = match
				matched.glob = ""
				if diagnostics := s.visitPath(basePath, matched, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 {
					return diagnostics
				}
			}
			continue
		}
		if diagnostics := s.visitPath(basePath, declaration, childDestination, declaration.destination, depth+1); len(diagnostics) != 0 {
			return diagnostics
		}
	}
	return nil
}

func (s *importTraversal) visitPath(basePath string, declaration importDeclaration, destination, relativeDestination Path, depth int) []Diagnostic {
	location := declaration.location
	if depth > s.limits.depth {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the active import-depth limit", &location)}
	}
	if !validImportPath(declaration.path) {
		return []Diagnostic{importDiagnostic(CodeImportInvalid, "import path must be a nonempty slash-relative path without a network scheme", &location)}
	}

	candidate := s.access.join(basePath, declaration.path)
	if !s.access.contains(candidate) {
		return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target is outside the configuration root", &location)}
	}
	canonical, err := s.access.canonical(candidate)
	if err != nil {
		if declaration.optional && errors.Is(err, fs.ErrNotExist) {
			return nil
		}
		code := CodeImportInvalid
		message := "import target could not be canonicalized"
		if errors.Is(err, fs.ErrNotExist) {
			code, message = CodeImportMissing, "required import target does not exist"
		} else if errors.Is(err, fs.ErrPermission) {
			message = "import target cannot be accessed"
		}
		return []Diagnostic{importDiagnostic(code, message, &location)}
	}
	if !s.access.contains(canonical) {
		return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target resolves outside the configuration root", &location)}
	}

	currentRoute := s.currentRoute(location)
	identity := s.access.identity(canonical)
	if activeIndex, ok := s.active[identity]; ok {
		cycle := append([]importRoute(nil), s.stack[activeIndex:]...)
		return []Diagnostic{importCycleDiagnostic(canonical, location, cycle)}
	}
	if previous, ok := s.visited[identity]; ok {
		return []Diagnostic{importRepeatedDiagnostic(location, previous, currentRoute)}
	}
	if s.accounting.sources+1 > s.limits.sources {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source-count limit", &location)}
	}

	if _, err := s.access.relative(canonical); err != nil {
		return []Diagnostic{importDiagnostic(CodeImportOutsideRoot, "import target could not be addressed inside the configuration root", &location)}
	}
	sourceID := "source-" + strconv.Itoa(s.accounting.sources+1)
	kind := SourceKindFile
	label := declaration.path
	if s.access.native != nil {
		kind = SourceKindFilesystem
		label = s.access.label + "/" + canonical
	}
	info := SourceInfo{ID: sourceID, Label: label, Path: canonical, Destination: destination.clone(), Layer: LayerBase, Kind: kind, ImportedFrom: &location}
	if kind == SourceKindFilesystem {
		info.namespace = s.access.token
	}
	s.sources = append(s.sources, info)
	data, skipped, diagnostic := s.readImport(canonical, sourceID, declaration.optional)
	if diagnostic != nil {
		return []Diagnostic{*diagnostic}
	}
	if skipped {
		s.sources = s.sources[:len(s.sources)-1]
		return nil
	}
	s.accounting.sources++

	root, issues := yamladapter.Parse(bytes.NewReader(data), sourceID, yamladapter.Limits{
		Depth: maxDocumentDepth, Nodes: maxDocumentNodes,
	})
	if len(issues) != 0 {
		return parserDiagnostics(sourceID, issues)
	}
	nodeCount := countInputNodes(root)
	if s.accounting.nodes+nodeCount > s.limits.nodes {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
	}
	s.accounting.nodes += nodeCount

	var body *document.Node
	var declarations []importDeclaration
	var diagnostics []Diagnostic
	var processing *sourceVersionProcessingResult
	if s.versionState != nil && s.versionState.mode != sourceVersionModeDefault {
		processed, failures := s.versionState.process(root, sourceVersionContribution, destination, sourceID)
		if len(failures) != 0 {
			return failures
		}
		processing = &processed
		body, declarations = processed.payload, processed.imports
	} else {
		body, declarations, diagnostics = splitImportDirectives(root)
		if len(diagnostics) != 0 {
			return diagnostics
		}
	}
	_, placed, err := placeDocument(body, destination, body.Location, maxDocumentDepth, maxDocumentNodes)
	if err != nil {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the source depth or node limit after destination placement", &location)}
	}
	placementNodes := destinationWrapperNodeCount(destination)
	if s.accounting.nodes+placementNodes > s.limits.nodes {
		return []Diagnostic{importDiagnostic(CodeImportLimitExceeded, "configuration exceeds the aggregate node limit", &location)}
	}
	s.accounting.nodes += placementNodes
	protectionRule := s.rule
	if s.versionState != nil && s.versionState.effectiveRule != nil {
		protectionRule = s.versionState.effectiveRule
	}
	if versionNode := fragmentVersionClaim(placed, protectionRule); versionNode != nil {
		versionLocation := publicLocation(versionNode.Location)
		return []Diagnostic{{
			Code: CodeImportVersion, Severity: SeverityError, Path: protectionRule.Path.clone(),
			Message: "imported fragments cannot contribute to the schema-version path", Primary: &versionLocation,
		}}
	}

	route := importRoute{canonical: canonical, locations: currentRoute.locations}
	s.scope.contributions = append(s.scope.contributions, acquiredContribution{
		source: cloneSourceInfo(info), payload: body, document: placed,
		destination: destination.clone(), relativeDestination: relativeDestination.clone(), processing: processing,
	})
	s.active[identity] = len(s.stack)
	s.visited[identity] = route
	s.stack = append(s.stack, route)
	diagnostics = s.walkDeclarations(s.access.dir(canonical), declarations, depth, destination)
	delete(s.active, identity)
	s.stack = s.stack[:len(s.stack)-1]
	return diagnostics
}

func (s *importTraversal) readImport(name, sourceID string, optional bool) ([]byte, bool, *Diagnostic) {
	remaining := s.limits.bytes - s.accounting.bytes
	limit := min(int64(maxSourceBytes), remaining)
	if limit < 0 {
		limit = 0
	}
	data, failure := readRegularFile(name, s.access.stat, s.access.open, limit)
	if failure != nil {
		if optional && failure.missingTarget {
			return nil, true, nil
		}
		value := sourceReadDiagnostic(failure, sourceID, CodeImportLimitExceeded, "configuration exceeds a source or aggregate byte limit")
		return nil, false, &value
	}
	s.accounting.bytes += int64(len(data))
	return data, false, nil
}

func (s *importTraversal) currentRoute(location Location) importRoute {
	locations := make([]Location, 0, len(s.stack)+1)
	for _, route := range s.stack {
		if len(route.locations) != 0 {
			locations = append(locations, route.locations[len(route.locations)-1])
		}
	}
	locations = append(locations, location)
	return importRoute{locations: locations}
}

func sourceInfos(source *SourceInfo) []SourceInfo {
	if source == nil {
		return nil
	}
	return []SourceInfo{cloneSourceInfo(*source)}
}

func importDiagnostic(code Code, message string, location *Location) Diagnostic {
	return Diagnostic{Code: code, Severity: SeverityError, Message: message, Primary: cloneLocation(location)}
}

func importCycleDiagnostic(target string, location Location, cycle []importRoute) Diagnostic {
	paths := make([]string, 0, len(cycle)+1)
	related := make([]Location, 0, len(cycle))
	for _, route := range cycle {
		paths = append(paths, route.canonical)
		if len(route.locations) != 0 {
			related = append(related, route.locations[len(route.locations)-1])
		}
	}
	paths = append(paths, target)
	return Diagnostic{
		Code: CodeImportCycle, Severity: SeverityError,
		Message: "import cycle detected: " + strings.Join(paths, " -> "),
		Primary: &location, Related: related,
	}
}

func importRepeatedDiagnostic(location Location, previous, current importRoute) Diagnostic {
	related := append([]Location(nil), previous.locations...)
	if len(current.locations) > 1 {
		related = append(related, current.locations[:len(current.locations)-1]...)
	}
	return Diagnostic{
		Code: CodeImportRepeated, Severity: SeverityError,
		Message: "configuration source is included by more than one import route",
		Primary: &location, Related: related,
	}
}