2026-07-17 08:37:52 +00:00
|
|
|
package pipeline
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"errors"
|
|
|
|
|
"fmt"
|
|
|
|
|
"path"
|
|
|
|
|
"sort"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"time"
|
|
|
|
|
|
2026-08-09 00:41:28 +00:00
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
2026-07-17 08:37:52 +00:00
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type laneExtractState struct {
|
2026-11-21 23:48:51 +00:00
|
|
|
fallbackCount int
|
|
|
|
|
index int
|
|
|
|
|
prepared preparedLaneExecutor
|
|
|
|
|
reuseEligible bool
|
|
|
|
|
values []erasedExtractArtifact
|
|
|
|
|
diagnostics []contracts.DiagnosticGroup
|
|
|
|
|
results map[int]extractJobResult
|
|
|
|
|
remaining int
|
|
|
|
|
failed bool
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
|
|
|
|
|
2026-07-17 14:05:21 +00:00
|
|
|
type finalizedExtractResults struct {
|
2026-11-21 23:48:51 +00:00
|
|
|
accepted []erasedExtractArtifact
|
|
|
|
|
diagnostics []contracts.DiagnosticGroup
|
|
|
|
|
reuseEligible bool
|
2026-07-17 14:05:21 +00:00
|
|
|
}
|
|
|
|
|
|
2026-07-17 08:37:52 +00:00
|
|
|
type extractJob struct {
|
|
|
|
|
lane *laneExtractState
|
|
|
|
|
chunk source.Chunk
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type extractJobResult struct {
|
2026-11-21 23:48:51 +00:00
|
|
|
laneIndex int
|
|
|
|
|
chunkIndex int
|
|
|
|
|
value erasedExtractArtifact
|
|
|
|
|
diagnostics []contracts.DiagnosticGroup
|
|
|
|
|
outcome contracts.ProducerOutcome
|
|
|
|
|
err error
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type laneCompletion struct {
|
|
|
|
|
index int
|
|
|
|
|
output RunOutput
|
|
|
|
|
err error
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type orderedRunError struct {
|
|
|
|
|
stage int
|
|
|
|
|
lane int
|
|
|
|
|
chunk int
|
|
|
|
|
err error
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:24:45 +00:00
|
|
|
func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
|
2026-11-22 02:34:40 +00:00
|
|
|
output := RunOutput{}
|
|
|
|
|
states, err := initializeLaneStates(input, step, chunks)
|
2026-07-22 02:03:27 +00:00
|
|
|
if err != nil {
|
|
|
|
|
return output, err
|
|
|
|
|
}
|
2026-07-22 14:24:03 +00:00
|
|
|
completedOutputs, runErrors := r.runLaneEngine(parent, laneEngineConfig{
|
2026-11-22 02:24:45 +00:00
|
|
|
input: input, doc: doc,
|
2026-07-22 14:24:03 +00:00
|
|
|
sourceInput: sourceInput, sessionID: sessionID, chunks: chunks, states: states,
|
|
|
|
|
})
|
2026-11-22 02:34:40 +00:00
|
|
|
mergeCompletedLanes(&output, completedOutputs)
|
2026-07-22 02:03:27 +00:00
|
|
|
if err := selectRunError(parent, runErrors); err != nil {
|
|
|
|
|
return output, err
|
|
|
|
|
}
|
|
|
|
|
return output, nil
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:34:40 +00:00
|
|
|
func initializeLaneStates(input RunInput, step PreparedPipelineStep, chunks []source.Chunk) ([]*laneExtractState, error) {
|
2026-07-21 21:19:00 +00:00
|
|
|
states := make([]*laneExtractState, len(step.lanes))
|
|
|
|
|
for i, prepared := range step.lanes {
|
2026-07-17 08:37:52 +00:00
|
|
|
if prepared.typed == nil {
|
2026-11-22 02:24:45 +00:00
|
|
|
return nil, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
states[i] = &laneExtractState{index: i, prepared: prepared, reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, prepared.resolved.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
2026-07-22 02:03:27 +00:00
|
|
|
return states, nil
|
|
|
|
|
}
|
2026-07-17 08:37:52 +00:00
|
|
|
|
2026-07-22 14:24:03 +00:00
|
|
|
type laneEngineConfig struct {
|
|
|
|
|
input RunInput
|
|
|
|
|
doc *source.SourceDocument
|
|
|
|
|
sourceInput contracts.LLMInputMaterial
|
|
|
|
|
sessionID string
|
|
|
|
|
chunks []source.Chunk
|
|
|
|
|
states []*laneExtractState
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 01:45:31 +00:00
|
|
|
// laneJob is ready to execute: a nil chunk selects the lane's merge/normalize
|
|
|
|
|
// continuation, otherwise it selects one extraction. Prerequisites are resolved
|
|
|
|
|
// by the coordinator before dispatch, never by an occupied worker.
|
|
|
|
|
type laneJob struct {
|
|
|
|
|
lane *laneExtractState
|
|
|
|
|
chunk *source.Chunk
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type laneJobResult struct {
|
|
|
|
|
extract *extractJobResult
|
|
|
|
|
completion *laneCompletion
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-22 14:24:03 +00:00
|
|
|
type laneEngine struct {
|
|
|
|
|
runner *Runner
|
|
|
|
|
laneEngineConfig
|
2026-11-22 01:45:31 +00:00
|
|
|
ctx context.Context
|
|
|
|
|
cancel context.CancelFunc
|
|
|
|
|
workerCount int
|
|
|
|
|
jobs chan laneJob
|
|
|
|
|
results chan laneJobResult
|
|
|
|
|
workers sync.WaitGroup
|
|
|
|
|
pending []*laneExtractState
|
|
|
|
|
completedOutputs []RunOutput
|
|
|
|
|
runErrors []orderedRunError
|
|
|
|
|
nextExtract int
|
|
|
|
|
active int
|
2026-07-22 14:24:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (r *Runner) runLaneEngine(parent context.Context, config laneEngineConfig) ([]RunOutput, []orderedRunError) {
|
|
|
|
|
engine := newLaneEngine(r, parent, config)
|
|
|
|
|
return engine.run()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func newLaneEngine(r *Runner, parent context.Context, config laneEngineConfig) *laneEngine {
|
2026-11-22 01:45:31 +00:00
|
|
|
workerCount := config.input.Workers
|
|
|
|
|
if workerCount == 0 {
|
|
|
|
|
workerCount = DefaultWorkers
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
|
|
|
|
ctx, cancel := context.WithCancel(parent)
|
2026-07-22 14:24:03 +00:00
|
|
|
return &laneEngine{
|
|
|
|
|
runner: r, laneEngineConfig: config, ctx: ctx, cancel: cancel, workerCount: workerCount,
|
2026-11-22 01:45:31 +00:00
|
|
|
jobs: make(chan laneJob), results: make(chan laneJobResult, workerCount),
|
2026-07-22 14:24:03 +00:00
|
|
|
completedOutputs: make([]RunOutput, len(config.states)),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *laneEngine) run() ([]RunOutput, []orderedRunError) {
|
|
|
|
|
defer e.cancel()
|
2026-11-22 01:45:31 +00:00
|
|
|
e.startWorkers()
|
|
|
|
|
e.collect()
|
|
|
|
|
close(e.jobs)
|
|
|
|
|
e.workers.Wait()
|
|
|
|
|
return e.completedOutputs, e.runErrors
|
2026-07-22 14:24:03 +00:00
|
|
|
}
|
2026-07-17 08:37:52 +00:00
|
|
|
|
2026-11-22 01:45:31 +00:00
|
|
|
func (e *laneEngine) startWorkers() {
|
2026-07-22 14:24:03 +00:00
|
|
|
for i := 0; i < e.workerCount; i++ {
|
2026-11-22 01:45:31 +00:00
|
|
|
e.workers.Add(1)
|
2026-07-17 08:37:52 +00:00
|
|
|
go func() {
|
2026-11-22 01:45:31 +00:00
|
|
|
defer e.workers.Done()
|
2026-07-22 14:24:03 +00:00
|
|
|
for job := range e.jobs {
|
2026-11-22 01:45:31 +00:00
|
|
|
if job.chunk != nil {
|
|
|
|
|
result := e.runner.runExtractJob(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, extractJob{lane: job.lane, chunk: *job.chunk})
|
|
|
|
|
e.results <- laneJobResult{extract: &result}
|
2026-07-17 08:37:52 +00:00
|
|
|
continue
|
|
|
|
|
}
|
2026-11-22 01:45:31 +00:00
|
|
|
completion := laneCompletion{index: job.lane.index}
|
|
|
|
|
if err := e.ctx.Err(); err != nil {
|
|
|
|
|
completion.err = err
|
|
|
|
|
} else {
|
2026-11-22 02:24:45 +00:00
|
|
|
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
|
2026-11-22 01:45:31 +00:00
|
|
|
}
|
|
|
|
|
e.results <- laneJobResult{completion: &completion}
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
}
|
2026-07-22 14:24:03 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 01:45:31 +00:00
|
|
|
// nextJob gives ready continuations priority, then preserves chunk-first,
|
|
|
|
|
// lane-second extraction order. Dispatch remains lazy rather than creating a
|
|
|
|
|
// queue or goroutine for every extraction.
|
|
|
|
|
func (e *laneEngine) nextJob() (laneJob, bool) {
|
|
|
|
|
if len(e.pending) > 0 {
|
|
|
|
|
return laneJob{lane: e.pending[0]}, true
|
2026-07-22 14:24:03 +00:00
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
if e.nextExtract < len(e.chunks)*len(e.states) {
|
2026-11-22 01:45:31 +00:00
|
|
|
state := e.states[e.nextExtract%len(e.states)]
|
2026-11-22 02:24:45 +00:00
|
|
|
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
|
2026-07-17 08:49:08 +00:00
|
|
|
}
|
2026-11-22 01:45:31 +00:00
|
|
|
return laneJob{}, false
|
2026-07-22 02:13:50 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 01:45:31 +00:00
|
|
|
// collect alone owns dispatch and result accounting. Cancellation disables new
|
|
|
|
|
// dispatch, but every dispatched job still returns one result before workers exit.
|
2026-07-22 14:24:03 +00:00
|
|
|
func (e *laneEngine) collect() {
|
2026-11-22 01:45:31 +00:00
|
|
|
for {
|
|
|
|
|
if e.ctx.Err() != nil {
|
|
|
|
|
if e.active == 0 {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
e.receiveResult(<-e.results)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
job, ready := e.nextJob()
|
|
|
|
|
if !ready && e.active == 0 {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
var jobs chan<- laneJob
|
|
|
|
|
if ready && e.active < e.workerCount {
|
|
|
|
|
jobs = e.jobs
|
2026-07-17 08:49:08 +00:00
|
|
|
}
|
2026-07-17 08:37:52 +00:00
|
|
|
select {
|
2026-11-22 01:45:31 +00:00
|
|
|
case <-e.ctx.Done():
|
|
|
|
|
case result := <-e.results:
|
|
|
|
|
e.receiveResult(result)
|
|
|
|
|
case jobs <- job:
|
|
|
|
|
e.active++
|
|
|
|
|
if job.chunk == nil {
|
|
|
|
|
e.pending = e.pending[1:]
|
|
|
|
|
} else {
|
|
|
|
|
e.nextExtract++
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-07-22 02:03:27 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 01:45:31 +00:00
|
|
|
func (e *laneEngine) receiveResult(result laneJobResult) {
|
|
|
|
|
e.active--
|
|
|
|
|
if result.extract != nil {
|
|
|
|
|
e.handleExtractResult(*result.extract)
|
|
|
|
|
} else {
|
|
|
|
|
e.handleCompletion(*result.completion)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-22 14:24:03 +00:00
|
|
|
func (e *laneEngine) handleExtractResult(result extractJobResult) {
|
|
|
|
|
state := e.states[result.laneIndex]
|
|
|
|
|
state.remaining--
|
|
|
|
|
if result.err != nil {
|
|
|
|
|
state.failed = true
|
|
|
|
|
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err})
|
|
|
|
|
e.cancel()
|
|
|
|
|
} else {
|
|
|
|
|
state.results[result.chunkIndex] = result
|
|
|
|
|
}
|
|
|
|
|
if state.remaining != 0 || state.failed || e.ctx.Err() != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
finalizeLaneExtract(state)
|
2026-07-22 14:24:03 +00:00
|
|
|
e.pending = append(e.pending, state)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (e *laneEngine) handleCompletion(completion laneCompletion) {
|
|
|
|
|
e.completedOutputs[completion.index] = completion.output
|
|
|
|
|
if completion.err != nil {
|
|
|
|
|
e.runErrors = append(e.runErrors, classifyLaneError(completion.index, len(e.chunks), completion.err))
|
|
|
|
|
e.cancel()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:34:40 +00:00
|
|
|
func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) {
|
2026-07-17 08:37:52 +00:00
|
|
|
for i := range completedOutputs {
|
2026-11-22 02:34:40 +00:00
|
|
|
mergeLaneOutput(output, completedOutputs[i])
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 03:19:43 +00:00
|
|
|
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) (result extractJobResult) {
|
2026-07-17 23:16:16 -05:00
|
|
|
state := job.lane
|
2026-07-17 08:37:52 +00:00
|
|
|
lane, typed := state.prepared.resolved, state.prepared.typed
|
2026-11-22 03:19:43 +00:00
|
|
|
result = extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
|
2026-11-21 23:48:51 +00:00
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
result.err = err
|
2026-07-17 23:16:16 -05:00
|
|
|
return result
|
|
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
chunk, err := cloneSourceChunk(job.chunk)
|
|
|
|
|
if err != nil {
|
|
|
|
|
result.err = err
|
2026-08-27 01:27:25 +00:00
|
|
|
return result
|
|
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
metadata, err := cloneMetadata(input.Metadata)
|
|
|
|
|
if err != nil {
|
|
|
|
|
result.err = err
|
|
|
|
|
return result
|
2026-08-27 15:29:10 +00:00
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
references := operationReferenceSet(input, lane.ExtractReferences)
|
2026-11-22 03:19:43 +00:00
|
|
|
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
|
|
|
|
|
var capturedArtifact contracts.SerializedArtifact
|
|
|
|
|
defer func() {
|
|
|
|
|
result.err = captureFailure(result.err, captureOperationResult(execution, capturedArtifact, result.outcome, result.err))
|
|
|
|
|
}()
|
|
|
|
|
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution, chunk: &chunk, ref: chunk.Ref}
|
|
|
|
|
if err := captureOperationInputs(execution, target, lane.Extract, nil); err != nil {
|
|
|
|
|
result.err = err
|
|
|
|
|
return result
|
|
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
var assessment contracts.CandidateAssessment
|
|
|
|
|
assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators)
|
2026-11-22 03:19:43 +00:00
|
|
|
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
|
2026-11-21 23:48:51 +00:00
|
|
|
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
|
|
|
|
|
var err error
|
|
|
|
|
assessment, err = assess(ctx, value)
|
|
|
|
|
return assessment, err
|
|
|
|
|
},
|
|
|
|
|
}
|
2026-11-22 02:10:05 +00:00
|
|
|
var cacheDiagnostics []contracts.ProducerDiagnostic
|
|
|
|
|
operation.Cache = artifactProducerCache(input, typed, lane.ExtractExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, doc, &chunk, &cacheDiagnostics)
|
2026-11-21 23:48:51 +00:00
|
|
|
extracted, err := typed.extract(ctx, typed.extractor, contracts.TypedExtractionRequest[any]{Operation: operation, Source: doc, Chunk: &chunk, SourceInput: target.sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Extract.LLMProfile, Metadata: metadata})
|
|
|
|
|
if err != nil {
|
|
|
|
|
result.err = fmt.Errorf("extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
|
2026-07-17 08:37:52 +00:00
|
|
|
return result
|
|
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
value, outcome, err := completeProducerOperation(ctx, lane.ExtractExecutionClass, operation, extracted.Value, extracted.Outcome, extracted.Fallback)
|
|
|
|
|
if err != nil {
|
|
|
|
|
result.err = fmt.Errorf("accept extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
|
|
|
|
|
return result
|
|
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
stored, err := serializeArtifact(typed.codec, value, false)
|
2026-11-21 23:48:51 +00:00
|
|
|
if err != nil {
|
|
|
|
|
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
|
|
|
|
|
return result
|
2026-08-27 01:06:17 +00:00
|
|
|
}
|
2026-11-22 03:19:43 +00:00
|
|
|
capturedArtifact, result.outcome = stored, outcome
|
2026-11-22 02:10:05 +00:00
|
|
|
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
|
2026-11-21 23:48:51 +00:00
|
|
|
if err != nil {
|
|
|
|
|
result.err = err
|
|
|
|
|
return result
|
|
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value, artifact: stored}
|
|
|
|
|
result.outcome = outcome
|
2026-07-17 08:37:52 +00:00
|
|
|
return result
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:24:45 +00:00
|
|
|
func finalizeLaneExtract(state *laneExtractState) {
|
2026-07-17 08:37:52 +00:00
|
|
|
indexes := make([]int, 0, len(state.results))
|
|
|
|
|
for index := range state.results {
|
|
|
|
|
indexes = append(indexes, index)
|
|
|
|
|
}
|
|
|
|
|
sort.Ints(indexes)
|
|
|
|
|
for _, index := range indexes {
|
|
|
|
|
result := state.results[index]
|
|
|
|
|
state.values = append(state.values, result.value)
|
2026-08-27 15:29:10 +00:00
|
|
|
state.diagnostics = append(state.diagnostics, result.diagnostics...)
|
2026-11-21 23:48:51 +00:00
|
|
|
if !result.outcome.ReuseEligible {
|
|
|
|
|
state.reuseEligible = false
|
|
|
|
|
}
|
|
|
|
|
if result.outcome.Disposition == contracts.ProducerFallback {
|
|
|
|
|
state.fallbackCount++
|
2026-08-27 01:06:17 +00:00
|
|
|
}
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:24:45 +00:00
|
|
|
func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
|
2026-07-17 08:37:52 +00:00
|
|
|
lane := state.prepared.resolved
|
2026-11-22 02:34:40 +00:00
|
|
|
local := RunOutput{FallbackCount: state.fallbackCount}
|
2026-07-17 14:05:21 +00:00
|
|
|
results := finalizedExtractResults{
|
2026-11-21 23:48:51 +00:00
|
|
|
accepted: state.values,
|
|
|
|
|
diagnostics: state.diagnostics,
|
|
|
|
|
reuseEligible: state.reuseEligible,
|
2026-07-17 14:05:21 +00:00
|
|
|
}
|
2026-08-27 15:29:10 +00:00
|
|
|
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
|
2026-11-22 03:19:43 +00:00
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "metadata": input.Metadata}}); err != nil {
|
2026-07-17 14:05:21 +00:00
|
|
|
return local, &laneRunError{stage: StageExtract, err: err}
|
|
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil {
|
2026-07-17 14:05:21 +00:00
|
|
|
return local, &laneRunError{stage: StageExtract, err: err}
|
|
|
|
|
}
|
|
|
|
|
if len(results.accepted) == 0 {
|
|
|
|
|
return local, nil
|
|
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
err := r.continueTypedLane(ctx, input, doc, sourceInput, sessionID, state.prepared, results, &local)
|
2026-07-17 08:37:52 +00:00
|
|
|
return local, err
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:24:45 +00:00
|
|
|
func debugExtractArtifacts(values []erasedExtractArtifact) []any {
|
|
|
|
|
outputs := make([]any, 0, len(values))
|
|
|
|
|
for _, value := range values {
|
|
|
|
|
outputs = append(outputs, map[string]any{"chunk_id": value.ChunkID, "chunk_index": value.ChunkIndex, "chunk_ref": value.ChunkRef, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{LaneID: value.LaneID, SourceID: value.SourceID, Artifact: value.artifact})})
|
|
|
|
|
}
|
|
|
|
|
return outputs
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-17 08:37:52 +00:00
|
|
|
func classifyLaneError(lane, sentinel int, err error) orderedRunError {
|
|
|
|
|
stage := 1
|
|
|
|
|
var laneErr *laneRunError
|
|
|
|
|
if errors.As(err, &laneErr) {
|
|
|
|
|
switch laneErr.stage {
|
|
|
|
|
case StageExtract:
|
|
|
|
|
stage = 0
|
|
|
|
|
case StageNormalize:
|
|
|
|
|
stage = 2
|
|
|
|
|
}
|
|
|
|
|
} else if strings.Contains(strings.ToLower(err.Error()), "normalize") {
|
|
|
|
|
stage = 2
|
|
|
|
|
}
|
|
|
|
|
return orderedRunError{stage: stage, lane: lane, chunk: sentinel, err: err}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func selectRunError(parent context.Context, values []orderedRunError) error {
|
|
|
|
|
if err := parent.Err(); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if len(values) == 0 {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
hasReal := false
|
|
|
|
|
for _, value := range values {
|
|
|
|
|
if !errors.Is(value.err, context.Canceled) && !errors.Is(value.err, context.DeadlineExceeded) {
|
|
|
|
|
hasReal = true
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
filtered := values[:0]
|
|
|
|
|
for _, value := range values {
|
|
|
|
|
if hasReal && (errors.Is(value.err, context.Canceled) || errors.Is(value.err, context.DeadlineExceeded)) {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
filtered = append(filtered, value)
|
|
|
|
|
}
|
|
|
|
|
sort.SliceStable(filtered, func(i, j int) bool {
|
|
|
|
|
if filtered[i].stage != filtered[j].stage {
|
|
|
|
|
return filtered[i].stage < filtered[j].stage
|
|
|
|
|
}
|
|
|
|
|
if filtered[i].lane != filtered[j].lane {
|
|
|
|
|
return filtered[i].lane < filtered[j].lane
|
|
|
|
|
}
|
|
|
|
|
return filtered[i].chunk < filtered[j].chunk
|
|
|
|
|
})
|
|
|
|
|
return filtered[0].err
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:34:40 +00:00
|
|
|
func mergeLaneOutput(dst *RunOutput, src RunOutput) {
|
2026-07-17 08:37:52 +00:00
|
|
|
if dst == nil {
|
2026-11-22 02:34:40 +00:00
|
|
|
return
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
dst.FallbackCount += src.FallbackCount
|
2026-11-22 02:34:40 +00:00
|
|
|
dst.normalizeOutputs = append(dst.normalizeOutputs, cloneSerializedOutputs(src.normalizeOutputs)...)
|
2026-08-27 15:29:10 +00:00
|
|
|
appendDiagnosticGroups(dst, src.diagnosticGroups)
|
2026-08-27 11:50:37 +00:00
|
|
|
if len(src.normalizeReuseEligibility) > 0 {
|
|
|
|
|
if dst.normalizeReuseEligibility == nil {
|
|
|
|
|
dst.normalizeReuseEligibility = make(map[generatedOutputKey]bool, len(src.normalizeReuseEligibility))
|
|
|
|
|
}
|
|
|
|
|
for key, eligible := range src.normalizeReuseEligibility {
|
|
|
|
|
dst.normalizeReuseEligibility[key] = eligible
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-07-17 08:37:52 +00:00
|
|
|
}
|