Files
notarius/internal/framework/pipeline/runner.go

596 lines
19 KiB
Go

package pipeline
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"mime"
"path"
"path/filepath"
"strings"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type Registries struct {
Inputs *InputAdapterRegistry
Chunkers *ChunkerRegistry
ArtifactCodecs *ArtifactCodecRegistry
ArtifactEvidence *ArtifactEvidenceRegistry
Extractors *ExtractorRegistry
Mergers *MergerRegistry
Normalizers *NormalizerRegistry
Validators *ValidatorRegistry
ValidatorChains *ValidatorChainRegistry
Outputs *OutputEncoderRegistry
}
type Runner struct{}
// DefaultWorkers is the shared ready-job limit when the caller omits one.
const DefaultWorkers = 64
func New() *Runner {
return &Runner{}
}
type RunInput struct {
Cache contracts.ArtifactCache
// ObserveCache receives scoped transient evidence, including prepared details
// on a hit. Shared observers must synchronize concurrent calls.
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
Debug DebugRecorder
// Workers bounds ready extraction and lane-completion jobs together.
// Zero selects DefaultWorkers; configuration rejects explicit zero.
Workers int
pipeline ResolvedPipeline
// reuseIneligible carries chunk ancestry; reference eligibility stays scoped.
reuseIneligible bool
stepID string
references map[referenceTargetKey]contracts.ReferenceSet
referenceReuseEligibility map[referenceTargetKey]bool
}
type RunOutput struct {
FallbackCount int `json:"fallback_count,omitempty"`
Diagnostics contracts.DiagnosticCollection `json:"diagnostics,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
normalizeOutputs []contracts.SerializedOutput
normalizeReuseEligibility map[generatedOutputKey]bool
diagnosticGroups []contracts.DiagnosticGroup
}
// ArtifactCount counts logical deliverables, not intermediate lane values,
// cache entries, diagnostic records, or debug files.
func (output RunOutput) ArtifactCount() int {
return len(output.OutputFiles)
}
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
if r == nil {
return output, fmt.Errorf("runner must not be nil")
}
if err := ctx.Err(); err != nil {
return output, err
}
if err := validateRunInput(input); err != nil {
return output, err
}
input.Metadata, err = cloneMetadata(input.Metadata)
if err != nil {
return output, fmt.Errorf("clone run metadata: %w", err)
}
input.pipeline = input.Prepared.resolved
sessionID := strings.TrimSpace(input.SessionID)
debugRecorder := input.Debug
if debugRecorder == nil {
debugRecorder = NoopDebugRecorder()
}
debugRecorder = synchronizedDebugRecorder(debugRecorder)
input.Debug = debugRecorder
inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil)
if diagnosticErr != nil {
return failOutput(output), fmt.Errorf("promote input diagnostics: %w", diagnosticErr)
}
appendDiagnosticGroups(&output, inputDiagnostics)
if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
Stage: "run",
StartedAt: startedTime(input.StartedAt),
Payload: map[string]any{
"pipeline_id": input.pipeline.ID,
"pipeline_digest": input.pipeline.Digest,
"run_id": input.RunID,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
}
adapter := input.Prepared.input
var doc *source.SourceDocument
sourceStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
Stage: "source",
ModuleKey: adapter.Key(),
StartedAt: sourceStarted,
Payload: debugSourceInput{
SourceID: input.SourceID,
Path: input.Path,
Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
Options: redactSensitiveMap(input.pipeline.Input.Options),
Metadata: redactSensitiveMap(input.Metadata),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
}
requestMetadata, err := cloneMetadata(input.Metadata)
if err != nil {
return failOutput(output), fmt.Errorf("clone input adapter metadata: %w", err)
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID, Path: input.Path, Raw: input.RawInput,
LLMProfile: input.pipeline.Input.LLMProfile, Metadata: requestMetadata,
})
if ctxErr := ctx.Err(); ctxErr != nil {
return failOutput(output), ctxErr
}
if err != nil {
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
}
if err := source.ValidateDocument(doc); err != nil {
return failOutput(output), fmt.Errorf("validate source document: %w", err)
}
if err := writeDebugTimed(debugRecorder, "source/output.json", debugTimedEnvelope{
Stage: "source",
ModuleKey: adapter.Key(),
StartedAt: sourceStarted,
Payload: map[string]any{
"document": debugSourceDocumentEnvelope(doc),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
}
sourceInput := sourceInputMaterial(input.Path, input.RawInput)
chunker := input.Prepared.chunker
chunkStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
StartedAt: chunkStarted,
Payload: map[string]any{
"cache_mode": input.pipeline.ChunkControls.Cache,
"source": debugSourceDocumentEnvelope(doc),
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
"metadata": redactSensitiveMap(input.Metadata),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
}
chunkResult, err := r.runChunkPlan(ctx, input, doc, sourceInput, sessionID)
if err != nil {
return failOutput(output), err
}
appendDiagnosticGroups(&output, chunkResult.diagnostics)
chunkDebugPayload := map[string]any{
"cache_mode": input.pipeline.ChunkControls.Cache,
"accepted": chunkResult.accepted,
"materialized_chunks": debugSourceChunkEnvelopes(chunkResult.chunks),
}
if chunkResult.plan != nil {
chunkDebugPayload["plan"] = debugChunkPlanEnvelope(*chunkResult.plan)
}
if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
StartedAt: chunkStarted,
Payload: chunkDebugPayload,
}); err != nil {
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
}
var acceptedChunkMap *contracts.SerializedArtifact
if chunkResult.accepted {
if chunkResult.plan == nil {
return failOutput(output), fmt.Errorf("accepted chunk plan is missing")
}
chunkProfile := ""
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
chunkProfile = input.pipeline.Chunk.LLMProfile
}
artifact, buildErr := chunkmap.Serialize(chunkmap.BuildRequest{
Source: doc,
Plan: *chunkResult.plan,
Chunks: chunkResult.chunks,
RequestedChunker: input.pipeline.Chunk.Module,
Producer: chunkmap.Producer{
InputModule: adapter.Key(),
ChunkModule: chunker.Key(),
LLMProfile: chunkProfile,
},
})
if buildErr != nil {
return failOutput(output), fmt.Errorf("serialize accepted chunk map: %w", buildErr)
}
acceptedChunkMap = &artifact
}
if chunkResult.accepted {
input.reuseIneligible = !chunkResult.outcome.ReuseEligible
if chunkResult.outcome.Disposition == contracts.ProducerFallback {
output.FallbackCount++
}
if err := r.runPreparedSteps(ctx, input, doc, sourceInput, sessionID, chunkResult.chunks, &output); err != nil {
return failOutput(output), err
}
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
if err := finalizeDiagnostics(&output); err != nil {
return failOutput(output), fmt.Errorf("aggregate diagnostics: %w", err)
}
encoder := input.Prepared.output
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
outputStarted := time.Now().UTC()
evidenceArtifact, evidenceSummary, err := buildOutputEvidenceContext(input.Prepared, doc, output.normalizeOutputs)
if err != nil {
return failOutput(output), err
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
if evidenceSummary != nil {
if err := writeDebugTimed(debugRecorder, "output/evidence-context.json", debugTimedEnvelope{
Stage: string(StageOutput),
ModuleKey: encoder.Key(),
StartedAt: outputStarted,
Payload: *evidenceSummary,
}); err != nil {
return failOutput(output), fmt.Errorf("write evidence context debug artifact: %w", err)
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
}
outputDebugPayload := map[string]any{
"normalize_outputs": debugSerializedOutputEnvelopes(output.normalizeOutputs),
"options": redactSensitiveMap(input.pipeline.Output.Options),
"metadata": redactSensitiveMap(input.Metadata),
}
if acceptedChunkMap != nil {
outputDebugPayload["chunk_map"] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: *acceptedChunkMap})
}
if err := writeDebugTimed(debugRecorder, "output/input.json", debugTimedEnvelope{
Stage: string(StageOutput),
ModuleKey: encoder.Key(),
StartedAt: outputStarted,
Payload: outputDebugPayload,
}); err != nil {
return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
NormalizeOutputs: cloneSerializedOutputs(output.normalizeOutputs),
ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap),
EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact),
})
if ctxErr := ctx.Err(); ctxErr != nil {
return failOutput(output), ctxErr
}
if err != nil {
return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err)
}
files, err := outputFilesFromResult(encoded)
if err != nil {
return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err)
}
if err := writeDebugTimed(debugRecorder, "output/output.json", debugTimedEnvelope{
Stage: string(StageOutput),
ModuleKey: encoder.Key(),
StartedAt: outputStarted,
Payload: map[string]any{
"files": debugOutputFiles(files),
},
}); err != nil {
return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
output.OutputFiles = files
return output, nil
}
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
for _, step := range input.Prepared.Steps {
stepInput := input
stepInput.stepID = step.ID
stepReferences, err := buildStepReferenceSets(input, step, output.normalizeOutputs)
if err != nil {
return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err)
}
stepInput.references = stepReferences
stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, doc, sourceInput, sessionID, chunks)
mergeLaneOutput(output, laneOutput)
if laneErr != nil {
return fmt.Errorf("execute pipeline step %q: %w", step.ID, laneErr)
}
}
return nil
}
func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
for _, chain := range chains {
if chain.Stage != stage {
continue
}
if chain.ModuleKey != moduleKey {
continue
}
if strings.TrimSpace(chain.LaneID) != strings.TrimSpace(laneID) {
continue
}
return ResolvedValidatorChain{
Stage: chain.Stage,
LaneID: chain.LaneID,
ModuleKey: chain.ModuleKey,
Validators: cloneResolvedValidators(chain.Validators),
}
}
return ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
ModuleKey: strings.TrimSpace(moduleKey),
}
}
func validateRunInput(input RunInput) error {
if input.Workers < 0 {
return fmt.Errorf("workers must be positive when specified")
}
if input.Prepared == nil {
return fmt.Errorf("prepared pipeline must not be nil")
}
if err := validateResolvedPipeline(input.Prepared.resolved); err != nil {
return err
}
return nil
}
func validateResolvedPipeline(pipeline ResolvedPipeline) error {
if pipeline.ID == "" {
return fmt.Errorf("resolved pipeline id must not be empty")
}
if pipeline.Digest == "" {
return fmt.Errorf("resolved pipeline digest must not be empty")
}
if pipeline.Input.Module == "" {
return fmt.Errorf("resolved pipeline input module must not be empty")
}
if pipeline.Chunk.Module == "" {
return fmt.Errorf("resolved pipeline chunk module must not be empty")
}
if pipeline.Output.Module == "" {
return fmt.Errorf("resolved pipeline output module must not be empty")
}
if len(pipeline.Steps) == 0 {
return fmt.Errorf("resolved pipeline artifact lanes must not be empty")
}
for _, lane := range pipeline.AllArtifactLanes() {
if lane.ID == "" {
return fmt.Errorf("resolved pipeline artifact lane id must not be empty")
}
if lane.Extract.Module == "" {
return fmt.Errorf("resolved pipeline lane %q extract module must not be empty", lane.ID)
}
if lane.Merge.Module == "" {
return fmt.Errorf("resolved pipeline lane %q merge module must not be empty", lane.ID)
}
if lane.Normalize.Module == "" {
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
}
if len(lane.Validators) > 0 {
return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID)
}
}
return nil
}
func startedTime(t time.Time) time.Time {
if t.IsZero() {
return time.Now().UTC()
}
return t.UTC()
}
func failOutput(output RunOutput) RunOutput {
// Preserve any diagnostics already accepted by the pipeline when a later
// operation fails. Producers and validators validate each diagnostic before
// it is appended, so finalization here cannot introduce a new failure path.
_ = finalizeDiagnostics(&output)
return output
}
func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
out := make([]contracts.OutputFile, 0, len(result.Files))
for _, file := range result.Files {
if err := validateOutputFileName(file.Name); err != nil {
return nil, err
}
out = append(out, contracts.OutputFile{
Name: file.Name,
ContentType: file.ContentType,
Bytes: append([]byte(nil), file.Bytes...),
})
}
return out, nil
}
func validateOutputFileName(name string) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("output file name must not be empty")
}
if strings.Contains(name, "\\") {
return fmt.Errorf("output file name %q must use slash-separated relative paths", name)
}
if path.IsAbs(name) {
return fmt.Errorf("output file name %q must be relative", name)
}
if strings.Contains(name, "..") {
return fmt.Errorf("output file name %q must not contain ..", name)
}
cleaned := path.Clean(name)
if cleaned == "." || cleaned != name {
return fmt.Errorf("output file name %q must be clean", name)
}
return nil
}
func cloneMetadata(metadata map[string]any) (map[string]any, error) {
return source.CloneMetadata(metadata)
}
func sourceInputMaterial(inputPath string, content []byte) contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial(
"source",
sourceInputMediaType(inputPath),
content,
sourceInputDigest(content),
sourceInputOriginURI(inputPath),
)
}
func chunkInputMaterial(sourceInput contracts.LLMInputMaterial, chunk source.Chunk) contracts.LLMInputMaterial {
return contracts.NewLLMInputMaterial(
"source",
chunk.MediaType,
chunk.Content,
sourceInputDigest(chunk.Content),
sourceInput.OriginURI,
)
}
func sourceInputMediaType(inputPath string) string {
extension := strings.ToLower(filepath.Ext(strings.TrimSpace(inputPath)))
if extension == ".json" {
return "application/json"
}
mediaType := mime.TypeByExtension(extension)
if strings.TrimSpace(mediaType) == "" {
return unknownMediaType
}
return canonicalMediaType(mediaType)
}
func sourceInputDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
func sourceInputOriginURI(inputPath string) string {
if strings.TrimSpace(inputPath) == "" {
return ""
}
return fileURI(inputPath)
}
func cloneSourceChunkPtr(chunk *source.Chunk) (*source.Chunk, error) {
if chunk == nil {
return nil, nil
}
cloned, err := cloneSourceChunk(*chunk)
if err != nil {
return nil, err
}
return &cloned, nil
}
func cloneSourceChunk(chunk source.Chunk) (source.Chunk, error) {
chunk.Content = append([]byte(nil), chunk.Content...)
var err error
chunk.Units, err = cloneSourceUnits(chunk.Units)
if err != nil {
return source.Chunk{}, err
}
chunk.Metadata, err = cloneMetadata(chunk.Metadata)
if err != nil {
return source.Chunk{}, fmt.Errorf("clone chunk metadata: %w", err)
}
chunk.Annotations = source.CloneChunkAnnotations(chunk.Annotations)
chunk.PlanAnnotations = source.CloneChunkAnnotations(chunk.PlanAnnotations)
return chunk, nil
}
func cloneSourceChunks(chunks []source.Chunk) ([]source.Chunk, error) {
if len(chunks) == 0 {
return nil, nil
}
out := make([]source.Chunk, 0, len(chunks))
for _, chunk := range chunks {
cloned, err := cloneSourceChunk(chunk)
if err != nil {
return nil, err
}
out = append(out, cloned)
}
return out, nil
}
func cloneSourceUnits(units []source.SourceUnit) ([]source.SourceUnit, error) {
if len(units) == 0 {
return nil, nil
}
out := make([]source.SourceUnit, 0, len(units))
for _, unit := range units {
cloned, err := cloneSourceUnit(unit)
if err != nil {
return nil, err
}
out = append(out, cloned)
}
return out, nil
}
func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput {
if len(outputs) == 0 {
return nil
}
out := make([]contracts.SerializedOutput, 0, len(outputs))
for _, output := range outputs {
out = append(out, contracts.CloneSerializedOutput(output))
}
return out
}