package pipeline import ( "context" "errors" "fmt" "path" "sort" "strings" "sync" "time" "go.maximumdirect.net/promptkit" "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) type laneExtractState struct { fallbackCount int index int prepared preparedLaneExecutor reuseEligible bool values []erasedExtractArtifact diagnostics []contracts.DiagnosticGroup results map[int]extractJobResult remaining int failed bool } type finalizedExtractResults struct { accepted []erasedExtractArtifact diagnostics []contracts.DiagnosticGroup reuseEligible bool } type extractJob struct { lane *laneExtractState chunk source.Chunk } type extractJobResult struct { laneIndex int chunkIndex int value erasedExtractArtifact diagnostics []contracts.DiagnosticGroup outcome contracts.ProducerOutcome err error } type laneCompletion struct { index int output RunOutput err error } type orderedRunError struct { stage int lane int chunk int err error } func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) { output := RunOutput{} states, err := initializeLaneStates(input, step, chunks) if err != nil { return output, err } completedOutputs, runErrors := r.runLaneEngine(parent, laneEngineConfig{ input: input, doc: doc, sourceInput: sourceInput, sessionID: sessionID, chunks: chunks, states: states, }) mergeCompletedLanes(&output, completedOutputs) if err := selectRunError(parent, runErrors); err != nil { return output, err } return output, nil } func initializeLaneStates(input RunInput, step PreparedPipelineStep, chunks []source.Chunk) ([]*laneExtractState, error) { states := make([]*laneExtractState, len(step.lanes)) for i, prepared := range step.lanes { if prepared.typed == nil { return nil, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID) } states[i] = &laneExtractState{index: i, prepared: prepared, reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, prepared.resolved.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))} } return states, nil } type laneEngineConfig struct { input RunInput doc *source.SourceDocument sourceInput contracts.LLMInputMaterial sessionID string chunks []source.Chunk states []*laneExtractState } // laneJob is ready to execute: a nil chunk selects the lane's merge/normalize // continuation, otherwise it selects one extraction. Prerequisites are resolved // by the coordinator before dispatch, never by an occupied worker. type laneJob struct { lane *laneExtractState chunk *source.Chunk } type laneJobResult struct { extract *extractJobResult completion *laneCompletion } type laneEngine struct { runner *Runner laneEngineConfig ctx context.Context cancel context.CancelFunc workerCount int jobs chan laneJob results chan laneJobResult workers sync.WaitGroup pending []*laneExtractState completedOutputs []RunOutput runErrors []orderedRunError nextExtract int active int } func (r *Runner) runLaneEngine(parent context.Context, config laneEngineConfig) ([]RunOutput, []orderedRunError) { engine := newLaneEngine(r, parent, config) return engine.run() } func newLaneEngine(r *Runner, parent context.Context, config laneEngineConfig) *laneEngine { workerCount := config.input.Workers if workerCount == 0 { workerCount = DefaultWorkers } ctx, cancel := context.WithCancel(parent) return &laneEngine{ runner: r, laneEngineConfig: config, ctx: ctx, cancel: cancel, workerCount: workerCount, jobs: make(chan laneJob), results: make(chan laneJobResult, workerCount), completedOutputs: make([]RunOutput, len(config.states)), } } func (e *laneEngine) run() ([]RunOutput, []orderedRunError) { defer e.cancel() e.startWorkers() e.collect() close(e.jobs) e.workers.Wait() return e.completedOutputs, e.runErrors } func (e *laneEngine) startWorkers() { for i := 0; i < e.workerCount; i++ { e.workers.Add(1) go func() { defer e.workers.Done() for job := range e.jobs { if job.chunk != nil { result := e.runner.runExtractJob(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, extractJob{lane: job.lane, chunk: *job.chunk}) e.results <- laneJobResult{extract: &result} continue } completion := laneCompletion{index: job.lane.index} if err := e.ctx.Err(); err != nil { completion.err = err } else { completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane) } e.results <- laneJobResult{completion: &completion} } }() } } // nextJob gives ready continuations priority, then preserves chunk-first, // lane-second extraction order. Dispatch remains lazy rather than creating a // queue or goroutine for every extraction. func (e *laneEngine) nextJob() (laneJob, bool) { if len(e.pending) > 0 { return laneJob{lane: e.pending[0]}, true } if e.nextExtract < len(e.chunks)*len(e.states) { state := e.states[e.nextExtract%len(e.states)] return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true } return laneJob{}, false } // collect alone owns dispatch and result accounting. Cancellation disables new // dispatch, but every dispatched job still returns one result before workers exit. func (e *laneEngine) collect() { for { if e.ctx.Err() != nil { if e.active == 0 { return } e.receiveResult(<-e.results) continue } job, ready := e.nextJob() if !ready && e.active == 0 { return } var jobs chan<- laneJob if ready && e.active < e.workerCount { jobs = e.jobs } select { case <-e.ctx.Done(): case result := <-e.results: e.receiveResult(result) case jobs <- job: e.active++ if job.chunk == nil { e.pending = e.pending[1:] } else { e.nextExtract++ } } } } func (e *laneEngine) receiveResult(result laneJobResult) { e.active-- if result.extract != nil { e.handleExtractResult(*result.extract) } else { e.handleCompletion(*result.completion) } } func (e *laneEngine) handleExtractResult(result extractJobResult) { state := e.states[result.laneIndex] state.remaining-- if result.err != nil { state.failed = true e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err}) e.cancel() } else { state.results[result.chunkIndex] = result } if state.remaining != 0 || state.failed || e.ctx.Err() != nil { return } finalizeLaneExtract(state) e.pending = append(e.pending, state) } func (e *laneEngine) handleCompletion(completion laneCompletion) { e.completedOutputs[completion.index] = completion.output if completion.err != nil { e.runErrors = append(e.runErrors, classifyLaneError(completion.index, len(e.chunks), completion.err)) e.cancel() } } func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) { for i := range completedOutputs { mergeLaneOutput(output, completedOutputs[i]) } } func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) (result extractJobResult) { state := job.lane lane, typed := state.prepared.resolved, state.prepared.typed result = extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index} if err := ctx.Err(); err != nil { result.err = err return result } chunk, err := cloneSourceChunk(job.chunk) if err != nil { result.err = err return result } metadata, err := cloneMetadata(input.Metadata) if err != nil { result.err = err return result } references := operationReferenceSet(input, lane.ExtractReferences) execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk) var capturedArtifact contracts.SerializedArtifact defer func() { result.err = captureFailure(result.err, captureOperationResult(execution, capturedArtifact, result.outcome, result.err)) }() target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution, chunk: &chunk, ref: chunk.Ref} if err := captureOperationInputs(execution, target, lane.Extract, nil); err != nil { result.err = err return result } var assessment contracts.CandidateAssessment assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators) operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences), Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) { var err error assessment, err = assess(ctx, value) return assessment, err }, } var cacheDiagnostics []contracts.ProducerDiagnostic operation.Cache = artifactProducerCache(input, typed, lane.ExtractExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, doc, &chunk, &cacheDiagnostics) extracted, err := typed.extract(ctx, typed.extractor, contracts.TypedExtractionRequest[any]{Operation: operation, Source: doc, Chunk: &chunk, SourceInput: target.sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Extract.LLMProfile, Metadata: metadata}) if err != nil { result.err = fmt.Errorf("extract lane %q chunk %q: %w", lane.ID, chunk.ID, err) return result } value, outcome, err := completeProducerOperation(ctx, lane.ExtractExecutionClass, operation, extracted.Value, extracted.Outcome, extracted.Fallback) if err != nil { result.err = fmt.Errorf("accept extract lane %q chunk %q: %w", lane.ID, chunk.ID, err) return result } stored, err := serializeArtifact(typed.codec, value, false) if err != nil { result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err) return result } capturedArtifact, result.outcome = stored, outcome result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk) if err != nil { result.err = err return result } result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value, artifact: stored} result.outcome = outcome return result } func finalizeLaneExtract(state *laneExtractState) { indexes := make([]int, 0, len(state.results)) for index := range state.results { indexes = append(indexes, index) } sort.Ints(indexes) for _, index := range indexes { result := state.results[index] state.values = append(state.values, result.value) state.diagnostics = append(state.diagnostics, result.diagnostics...) if !result.outcome.ReuseEligible { state.reuseEligible = false } if result.outcome.Disposition == contracts.ProducerFallback { state.fallbackCount++ } } } func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) { lane := state.prepared.resolved local := RunOutput{FallbackCount: state.fallbackCount} results := finalizedExtractResults{ accepted: state.values, diagnostics: state.diagnostics, reuseEligible: state.reuseEligible, } local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...) if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "metadata": input.Metadata}}); err != nil { return local, &laneRunError{stage: StageExtract, err: err} } if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil { return local, &laneRunError{stage: StageExtract, err: err} } if len(results.accepted) == 0 { return local, nil } err := r.continueTypedLane(ctx, input, doc, sourceInput, sessionID, state.prepared, results, &local) return local, err } func debugExtractArtifacts(values []erasedExtractArtifact) []any { outputs := make([]any, 0, len(values)) for _, value := range values { outputs = append(outputs, map[string]any{"chunk_id": value.ChunkID, "chunk_index": value.ChunkIndex, "chunk_ref": value.ChunkRef, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{LaneID: value.LaneID, SourceID: value.SourceID, Artifact: value.artifact})}) } return outputs } func classifyLaneError(lane, sentinel int, err error) orderedRunError { stage := 1 var laneErr *laneRunError if errors.As(err, &laneErr) { switch laneErr.stage { case StageExtract: stage = 0 case StageNormalize: stage = 2 } } else if strings.Contains(strings.ToLower(err.Error()), "normalize") { stage = 2 } return orderedRunError{stage: stage, lane: lane, chunk: sentinel, err: err} } func selectRunError(parent context.Context, values []orderedRunError) error { if err := parent.Err(); err != nil { return err } if len(values) == 0 { return nil } hasReal := false for _, value := range values { if !isSiblingCancellation(value.err) { hasReal = true break } } var best *orderedRunError for _, value := range values { if hasReal && isSiblingCancellation(value.err) { continue } if best == nil || value.stage < best.stage || value.stage == best.stage && (value.lane < best.lane || value.lane == best.lane && value.chunk < best.chunk) { best = &value } } return best.err } // The runner cancels siblings, but does not give them a separate deadline. // Deadline-shaped provider/native failures remain real when the caller is active. func isSiblingCancellation(err error) bool { return errors.Is(err, context.Canceled) && !errors.Is(err, promptkit.ErrOperationTimeout) } func mergeLaneOutput(dst *RunOutput, src RunOutput) { if dst == nil { return } dst.FallbackCount += src.FallbackCount dst.normalizeOutputs = append(dst.normalizeOutputs, cloneSerializedOutputs(src.normalizeOutputs)...) appendDiagnosticGroups(dst, src.diagnosticGroups) if len(src.normalizeReuseEligibility) > 0 { if dst.normalizeReuseEligibility == nil { dst.normalizeReuseEligibility = make(map[generatedOutputKey]bool, len(src.normalizeReuseEligibility)) } for key, eligible := range src.normalizeReuseEligibility { dst.normalizeReuseEligibility[key] = eligible } } }