package pipeline import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "mime" "path" "path/filepath" "strings" "sync" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/promptkit" ) type Registries struct { Inputs *InputAdapterRegistry Chunkers *ChunkerRegistry ArtifactCodecs *ArtifactCodecRegistry ArtifactEvidence *ArtifactEvidenceRegistry Extractors *ExtractorRegistry Mergers *MergerRegistry Normalizers *NormalizerRegistry Validators *ValidatorRegistry ValidatorChains *ValidatorChainRegistry Outputs *OutputEncoderRegistry } type Runner struct{} // DefaultWorkers is the shared ready-job limit when the caller omits one. const DefaultWorkers = 64 func New() *Runner { return &Runner{} } type RunInput struct { Cache contracts.ArtifactCache // ObserveCache receives scoped transient evidence, including prepared details // on a hit. Shared observers must synchronize concurrent calls. ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error Execution contracts.ExecutionOptions // Executions is the invocation's native group. The runner enrolls its // requests; callers may use Snapshot without maintaining copied totals. Executions *promptkit.ExecutionGroup Prepared *PreparedPipeline SourceID string Path string RawInput []byte SessionID string RunID string StartedAt time.Time Metadata map[string]any Diagnostics []contracts.ProducerDiagnostic Debug DebugRecorder // Workers bounds ready extraction and lane-completion jobs together. // Zero selects DefaultWorkers; configuration rejects explicit zero. Workers int pipeline ResolvedPipeline // reuseIneligible carries chunk ancestry; reference eligibility stays scoped. reuseIneligible bool stepID string references map[referenceTargetKey]contracts.ReferenceSet referenceReuseEligibility map[referenceTargetKey]bool // Shared by operation decoders, but built only when a cache hit needs it. cacheSourceIndex func() source.DocumentIndex } type RunOutput struct { FallbackCount int `json:"fallback_count,omitempty"` Diagnostics contracts.DiagnosticCollection `json:"diagnostics,omitempty"` OutputFiles []contracts.OutputFile `json:"-"` normalizeOutputs []contracts.SerializedOutput normalizeReuseEligibility map[generatedOutputKey]bool diagnosticGroups []contracts.DiagnosticGroup } // ArtifactCount counts logical deliverables, not intermediate lane values, // cache entries, diagnostic records, or debug files. func (output RunOutput) ArtifactCount() int { return len(output.OutputFiles) } func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) { if r == nil { return output, fmt.Errorf("runner must not be nil") } if err := ctx.Err(); err != nil { return output, err } if err := validateRunInput(input); err != nil { return output, err } input.Metadata, err = cloneMetadata(input.Metadata) if err != nil { return output, fmt.Errorf("clone run metadata: %w", err) } input.pipeline = input.Prepared.resolved sessionID := strings.TrimSpace(input.SessionID) debugRecorder := input.Debug if debugRecorder == nil { debugRecorder = NoopDebugRecorder() } debugRecorder = synchronizedDebugRecorder(debugRecorder) input.Debug = debugRecorder if input.Executions == nil { input.Executions = &promptkit.ExecutionGroup{} } input.Execution.Observer = promptkit.ComposeObservers(input.Execution.Observer, input.Executions.Observe) defer func() { if debugRecorder.Enabled() { if captureErr := debugRecorder.WriteJSON("executions.json", input.Executions.Snapshot()); captureErr != nil { err = errors.Join(err, &contracts.CaptureError{Cause: captureErr}) output = failOutput(output) } } }() inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil) if diagnosticErr != nil { return failOutput(output), fmt.Errorf("promote input diagnostics: %w", diagnosticErr) } appendDiagnosticGroups(&output, inputDiagnostics) if err := writeDebugTimed(debugRecorder, "pipeline.json", debugTimedEnvelope{ Stage: "run", StartedAt: startedTime(input.StartedAt), Payload: map[string]any{ "pipeline_id": input.pipeline.ID, "pipeline_digest": input.pipeline.Digest, "run_id": input.RunID, }, }); err != nil { return failOutput(output), fmt.Errorf("write debug run artifact: %w", err) } if debugRecorder.Enabled() { if err := debugRecorder.WriteJSON("composition.json", captureComposition(input.pipeline)); err != nil { return failOutput(output), &contracts.CaptureError{Cause: err} } } adapter := input.Prepared.input var doc *source.SourceDocument sourceStarted := time.Now().UTC() if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{ Stage: "source", ModuleKey: adapter.Key(), StartedAt: sourceStarted, Payload: debugSourceInput{ SourceID: input.SourceID, Path: input.Path, Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil), Metadata: input.Metadata, }, }); err != nil { return failOutput(output), fmt.Errorf("write source debug artifact: %w", err) } requestMetadata, err := cloneMetadata(input.Metadata) if err != nil { return failOutput(output), fmt.Errorf("clone input adapter metadata: %w", err) } if err := ctx.Err(); err != nil { return failOutput(output), err } doc, err = adapter.Parse(ctx, contracts.ParseRequest{ SourceID: input.SourceID, Path: input.Path, Raw: input.RawInput, LLMProfile: input.pipeline.Input.LLMProfile, Metadata: requestMetadata, }) if ctxErr := ctx.Err(); ctxErr != nil { return failOutput(output), ctxErr } if err != nil { return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err) } if err := source.ValidateDocument(doc); err != nil { return failOutput(output), fmt.Errorf("validate source document: %w", err) } input.cacheSourceIndex = sync.OnceValue(func() source.DocumentIndex { return source.NewDocumentIndex(doc) }) if err := writeDebugTimed(debugRecorder, "source/output.json", debugTimedEnvelope{ Stage: "source", ModuleKey: adapter.Key(), StartedAt: sourceStarted, Payload: map[string]any{ "document": debugSourceDocumentEnvelope(doc), }, }); err != nil { return failOutput(output), fmt.Errorf("write source debug artifact: %w", err) } sourceInput := sourceInputMaterial(input.Path, input.RawInput) chunker := input.Prepared.chunker chunkStarted := time.Now().UTC() if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{ Stage: string(StageChunk), ModuleKey: chunker.Key(), StartedAt: chunkStarted, Payload: map[string]any{ "cache_mode": input.pipeline.ChunkControls.Cache, "source": debugSourceDocumentEnvelope(doc), "source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil), "metadata": input.Metadata, }, }); err != nil { return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err) } chunkResult, err := r.runChunkPlan(ctx, input, doc, sourceInput, sessionID) if err != nil { return failOutput(output), err } appendDiagnosticGroups(&output, chunkResult.diagnostics) chunkDebugPayload := map[string]any{ "cache_mode": input.pipeline.ChunkControls.Cache, "accepted": chunkResult.accepted, "materialized_chunks": debugSourceChunkEnvelopes(chunkResult.chunks), } if chunkResult.plan != nil { chunkDebugPayload["plan"] = debugChunkPlanEnvelope(*chunkResult.plan) } if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{ Stage: string(StageChunk), ModuleKey: chunker.Key(), StartedAt: chunkStarted, Payload: chunkDebugPayload, }); err != nil { return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err) } var acceptedChunkMap *contracts.SerializedArtifact if chunkResult.accepted { if chunkResult.plan == nil { return failOutput(output), fmt.Errorf("accepted chunk plan is missing") } chunkProfile := "" if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked { chunkProfile = input.pipeline.Chunk.LLMProfile } artifact, buildErr := chunkmap.Serialize(chunkmap.BuildRequest{ Source: doc, Plan: *chunkResult.plan, Chunks: chunkResult.chunks, RequestedChunker: input.pipeline.Chunk.Module, Producer: chunkmap.Producer{ InputModule: adapter.Key(), ChunkModule: chunker.Key(), LLMProfile: chunkProfile, }, }) if buildErr != nil { return failOutput(output), fmt.Errorf("serialize accepted chunk map: %w", buildErr) } acceptedChunkMap = &artifact } if chunkResult.accepted { input.reuseIneligible = !chunkResult.outcome.ReuseEligible if chunkResult.outcome.Disposition == contracts.ProducerFallback { output.FallbackCount++ } if err := r.runPreparedSteps(ctx, input, doc, sourceInput, sessionID, chunkResult.chunks, &output); err != nil { return failOutput(output), err } } if err := ctx.Err(); err != nil { return failOutput(output), err } if err := finalizeDiagnostics(&output); err != nil { return failOutput(output), fmt.Errorf("aggregate diagnostics: %w", err) } encoder := input.Prepared.output if err := ctx.Err(); err != nil { return failOutput(output), err } outputStarted := time.Now().UTC() evidenceArtifact, evidenceSummary, err := buildOutputEvidenceContext(input.Prepared, doc, output.normalizeOutputs) if err != nil { return failOutput(output), err } if err := ctx.Err(); err != nil { return failOutput(output), err } if evidenceSummary != nil { if err := writeDebugTimed(debugRecorder, "output/evidence-context.json", debugTimedEnvelope{ Stage: string(StageOutput), ModuleKey: encoder.Key(), StartedAt: outputStarted, Payload: *evidenceSummary, }); err != nil { return failOutput(output), fmt.Errorf("write evidence context debug artifact: %w", err) } if err := ctx.Err(); err != nil { return failOutput(output), err } } outputDebugPayload := map[string]any{ "normalize_outputs": debugSerializedOutputEnvelopes(output.normalizeOutputs), "metadata": input.Metadata, } if acceptedChunkMap != nil { outputDebugPayload["chunk_map"] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: *acceptedChunkMap}) } if err := writeDebugTimed(debugRecorder, "output/input.json", debugTimedEnvelope{ Stage: string(StageOutput), ModuleKey: encoder.Key(), StartedAt: outputStarted, Payload: outputDebugPayload, }); err != nil { return failOutput(output), fmt.Errorf("write output debug artifact: %w", err) } if err := ctx.Err(); err != nil { return failOutput(output), err } encoded, err := encoder.Encode(ctx, contracts.OutputRequest{ NormalizeOutputs: cloneSerializedOutputs(output.normalizeOutputs), ChunkMap: contracts.CloneSerializedArtifactPointer(acceptedChunkMap), EvidenceContext: contracts.CloneSerializedArtifactPointer(evidenceArtifact), }) if ctxErr := ctx.Err(); ctxErr != nil { return failOutput(output), ctxErr } if err != nil { return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err) } files, err := outputFilesFromResult(encoded) if err != nil { return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err) } if err := writeDebugTimed(debugRecorder, "output/output.json", debugTimedEnvelope{ Stage: string(StageOutput), ModuleKey: encoder.Key(), StartedAt: outputStarted, Payload: map[string]any{ "files": debugOutputFiles(files), }, }); err != nil { return failOutput(output), fmt.Errorf("write output debug artifact: %w", err) } if err := ctx.Err(); err != nil { return failOutput(output), err } output.OutputFiles = files return output, nil } func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error { for _, step := range input.Prepared.Steps { stepInput := input stepInput.stepID = step.ID stepReferences, err := buildStepReferenceSets(input, step, output.normalizeOutputs) if err != nil { return fmt.Errorf("prepare generated references for pipeline step %q: %w", step.ID, err) } stepInput.references = stepReferences stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility) laneOutput, laneErr := r.runLanes(ctx, stepInput, step, doc, sourceInput, sessionID, chunks) mergeLaneOutput(output, laneOutput) if laneErr != nil { return fmt.Errorf("execute pipeline step %q: %w", step.ID, laneErr) } } return nil } func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain { for _, chain := range chains { if chain.Stage != stage { continue } if chain.ModuleKey != moduleKey { continue } if strings.TrimSpace(chain.LaneID) != strings.TrimSpace(laneID) { continue } return ResolvedValidatorChain{ Stage: chain.Stage, LaneID: chain.LaneID, ModuleKey: chain.ModuleKey, Validators: cloneResolvedValidators(chain.Validators), } } return ResolvedValidatorChain{ Stage: stage, LaneID: strings.TrimSpace(laneID), ModuleKey: strings.TrimSpace(moduleKey), } } func validateRunInput(input RunInput) error { if input.Workers < 0 { return fmt.Errorf("workers must be positive when specified") } if input.Prepared == nil { return fmt.Errorf("prepared pipeline must not be nil") } if err := validateResolvedPipeline(input.Prepared.resolved); err != nil { return err } return nil } func validateResolvedPipeline(pipeline ResolvedPipeline) error { if pipeline.ID == "" { return fmt.Errorf("resolved pipeline id must not be empty") } if pipeline.Digest == "" { return fmt.Errorf("resolved pipeline digest must not be empty") } if pipeline.Input.Module == "" { return fmt.Errorf("resolved pipeline input module must not be empty") } if pipeline.Chunk.Module == "" { return fmt.Errorf("resolved pipeline chunk module must not be empty") } if pipeline.Output.Module == "" { return fmt.Errorf("resolved pipeline output module must not be empty") } if len(pipeline.Steps) == 0 { return fmt.Errorf("resolved pipeline artifact lanes must not be empty") } for _, lane := range pipeline.AllArtifactLanes() { if lane.ID == "" { return fmt.Errorf("resolved pipeline artifact lane id must not be empty") } if lane.Extract.Module == "" { return fmt.Errorf("resolved pipeline lane %q extract module must not be empty", lane.ID) } if lane.Merge.Module == "" { return fmt.Errorf("resolved pipeline lane %q merge module must not be empty", lane.ID) } if lane.Normalize.Module == "" { return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID) } if len(lane.Validators) > 0 { return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID) } } return nil } func startedTime(t time.Time) time.Time { if t.IsZero() { return time.Now().UTC() } return t.UTC() } func failOutput(output RunOutput) RunOutput { // Preserve any diagnostics already accepted by the pipeline when a later // operation fails. Producers and validators validate each diagnostic before // it is appended, so finalization here cannot introduce a new failure path. _ = finalizeDiagnostics(&output) return output } func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) { out := make([]contracts.OutputFile, 0, len(result.Files)) for _, file := range result.Files { if err := validateOutputFileName(file.Name); err != nil { return nil, err } out = append(out, contracts.OutputFile{ Name: file.Name, ContentType: file.ContentType, Bytes: append([]byte(nil), file.Bytes...), }) } return out, nil } func validateOutputFileName(name string) error { if strings.TrimSpace(name) == "" { return fmt.Errorf("output file name must not be empty") } if strings.Contains(name, "\\") { return fmt.Errorf("output file name %q must use slash-separated relative paths", name) } if path.IsAbs(name) { return fmt.Errorf("output file name %q must be relative", name) } if strings.Contains(name, "..") { return fmt.Errorf("output file name %q must not contain ..", name) } cleaned := path.Clean(name) if cleaned == "." || cleaned != name { return fmt.Errorf("output file name %q must be clean", name) } return nil } func cloneMetadata(metadata map[string]any) (map[string]any, error) { return source.CloneMetadata(metadata) } func sourceInputMaterial(inputPath string, content []byte) contracts.LLMInputMaterial { return contracts.NewLLMInputMaterial( "source", sourceInputMediaType(inputPath), content, sourceInputDigest(content), sourceInputOriginURI(inputPath), ) } func chunkInputMaterial(sourceInput contracts.LLMInputMaterial, chunk source.Chunk) contracts.LLMInputMaterial { return contracts.NewLLMInputMaterial( "source", chunk.MediaType, chunk.Content, sourceInputDigest(chunk.Content), sourceInput.OriginURI, ) } func sourceInputMediaType(inputPath string) string { extension := strings.ToLower(filepath.Ext(strings.TrimSpace(inputPath))) if extension == ".json" { return "application/json" } mediaType := mime.TypeByExtension(extension) if strings.TrimSpace(mediaType) == "" { return unknownMediaType } return canonicalMediaType(mediaType) } func sourceInputDigest(content []byte) string { sum := sha256.Sum256(content) return "sha256:" + hex.EncodeToString(sum[:]) } func sourceInputOriginURI(inputPath string) string { if strings.TrimSpace(inputPath) == "" { return "" } return fileURI(inputPath) } func cloneSourceChunkPtr(chunk *source.Chunk) (*source.Chunk, error) { if chunk == nil { return nil, nil } cloned, err := cloneSourceChunk(*chunk) if err != nil { return nil, err } return &cloned, nil } func cloneSourceChunk(chunk source.Chunk) (source.Chunk, error) { chunk.Content = append([]byte(nil), chunk.Content...) var err error chunk.Units, err = cloneSourceUnits(chunk.Units) if err != nil { return source.Chunk{}, err } chunk.Metadata, err = cloneMetadata(chunk.Metadata) if err != nil { return source.Chunk{}, fmt.Errorf("clone chunk metadata: %w", err) } chunk.Annotations = source.CloneChunkAnnotations(chunk.Annotations) chunk.PlanAnnotations = source.CloneChunkAnnotations(chunk.PlanAnnotations) return chunk, nil } func cloneSourceChunks(chunks []source.Chunk) ([]source.Chunk, error) { if len(chunks) == 0 { return nil, nil } out := make([]source.Chunk, 0, len(chunks)) for _, chunk := range chunks { cloned, err := cloneSourceChunk(chunk) if err != nil { return nil, err } out = append(out, cloned) } return out, nil } func cloneSourceUnits(units []source.SourceUnit) ([]source.SourceUnit, error) { if len(units) == 0 { return nil, nil } out := make([]source.SourceUnit, 0, len(units)) for _, unit := range units { cloned, err := cloneSourceUnit(unit) if err != nil { return nil, err } out = append(out, cloned) } return out, nil } func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput { if len(outputs) == 0 { return nil } out := make([]contracts.SerializedOutput, 0, len(outputs)) for _, output := range outputs { out = append(out, contracts.CloneSerializedOutput(output)) } return out }