629 lines
25 KiB
Go
629 lines
25 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
type laneExtractState struct {
|
|
fallbackCount int
|
|
index int
|
|
prepared preparedLaneExecutor
|
|
deps []CheckpointFingerprint
|
|
decision CheckpointDecision
|
|
reuseEligible bool
|
|
values []erasedExtractArtifact
|
|
serialized []CheckpointArtifact
|
|
diagnostics []contracts.DiagnosticGroup
|
|
rejected []contracts.RejectedOutput
|
|
results map[int]extractJobResult
|
|
remaining int
|
|
failed bool
|
|
terminal bool
|
|
output RunOutput
|
|
}
|
|
|
|
type finalizedExtractResults struct {
|
|
accepted []erasedExtractArtifact
|
|
serialized []CheckpointArtifact
|
|
diagnostics []contracts.DiagnosticGroup
|
|
rejected []contracts.RejectedOutput
|
|
decision CheckpointDecision
|
|
reuseEligible bool
|
|
}
|
|
|
|
func loadExtract(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
|
if stepAware, ok := loader.(StepCheckpointLoader); ok {
|
|
return stepAware.ExtractForStep(stepID, laneID, moduleKey, deps)
|
|
}
|
|
return loader.Extract(laneID, moduleKey, deps)
|
|
}
|
|
|
|
func recordExtract(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
|
|
return checkpointExtractSucceeded(recorder, stepID, laneID, moduleKey, deps, outputs, rejected)
|
|
}
|
|
|
|
type extractJob struct {
|
|
lane *laneExtractState
|
|
chunk source.Chunk
|
|
}
|
|
|
|
type extractJobResult struct {
|
|
laneIndex int
|
|
chunkIndex int
|
|
value erasedExtractArtifact
|
|
serialized CheckpointArtifact
|
|
diagnostics []contracts.DiagnosticGroup
|
|
rejected *contracts.RejectedOutput
|
|
outcome contracts.ProducerOutcome
|
|
err error
|
|
}
|
|
|
|
type laneCompletion struct {
|
|
index int
|
|
output RunOutput
|
|
err error
|
|
}
|
|
|
|
type orderedRunError struct {
|
|
stage int
|
|
lane int
|
|
chunk int
|
|
err error
|
|
}
|
|
|
|
func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
|
|
output := RunOutput{Manifest: manifestFromPipeline(input)}
|
|
states, err := initializeLaneStates(input, step, checkpoints, loader, doc, chunks, &output)
|
|
if err != nil {
|
|
if mergeErr := mergeTerminalLaneStates(&output, states); mergeErr != nil {
|
|
return output, errors.Join(err, mergeErr)
|
|
}
|
|
return output, err
|
|
}
|
|
completedOutputs, runErrors := r.runLaneEngine(parent, laneEngineConfig{
|
|
input: input, checkpoints: checkpoints, loader: loader, doc: doc,
|
|
sourceInput: sourceInput, sessionID: sessionID, chunks: chunks, states: states,
|
|
})
|
|
if err := mergeCompletedLanes(&output, completedOutputs); err != nil {
|
|
return output, err
|
|
}
|
|
if err := selectRunError(parent, runErrors); err != nil {
|
|
return output, err
|
|
}
|
|
return output, nil
|
|
}
|
|
|
|
func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, output *RunOutput) ([]*laneExtractState, error) {
|
|
states := make([]*laneExtractState, len(step.lanes))
|
|
for i, prepared := range step.lanes {
|
|
if prepared.typed == nil {
|
|
return states, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
|
|
}
|
|
if err := setTypedLaneManifestMetadata(output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
|
|
return states, err
|
|
}
|
|
if input.CheckpointPolicy.requiresReusable(input.stepID, prepared.resolved.ID) && !input.CheckpointPolicy.forced(input.stepID, prepared.resolved.ID) {
|
|
if input.reuseIneligible || !laneReferencesReuseEligible(input, prepared.resolved) {
|
|
return states, fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q: generated input lineage is validation-incomplete", input.stepID, prepared.resolved.ID)
|
|
}
|
|
state, err := hydrateRequiredLane(input, loader, doc, i, prepared)
|
|
states[i] = state
|
|
if err != nil {
|
|
return states, err
|
|
}
|
|
continue
|
|
}
|
|
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared, output)
|
|
if err != nil {
|
|
return states, err
|
|
}
|
|
if !state.decision.Reused && state.reuseEligible {
|
|
if err := checkpointExtractRunning(checkpoints, input.stepID, prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
|
|
return states, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
|
|
}
|
|
} else if err := finalizeLaneExtract(checkpoints, input.stepID, state); err != nil {
|
|
return states, err
|
|
}
|
|
states[i] = state
|
|
}
|
|
return states, nil
|
|
}
|
|
|
|
type laneEngineConfig struct {
|
|
input RunInput
|
|
checkpoints CheckpointRecorder
|
|
loader CheckpointLoader
|
|
doc *source.SourceDocument
|
|
sourceInput contracts.LLMInputMaterial
|
|
sessionID string
|
|
chunks []source.Chunk
|
|
states []*laneExtractState
|
|
}
|
|
|
|
// laneJob is ready to execute: a nil chunk selects the lane's merge/normalize
|
|
// continuation, otherwise it selects one extraction. Prerequisites are resolved
|
|
// by the coordinator before dispatch, never by an occupied worker.
|
|
type laneJob struct {
|
|
lane *laneExtractState
|
|
chunk *source.Chunk
|
|
}
|
|
|
|
type laneJobResult struct {
|
|
extract *extractJobResult
|
|
completion *laneCompletion
|
|
}
|
|
|
|
type laneEngine struct {
|
|
runner *Runner
|
|
laneEngineConfig
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
workerCount int
|
|
jobs chan laneJob
|
|
results chan laneJobResult
|
|
workers sync.WaitGroup
|
|
pending []*laneExtractState
|
|
completedOutputs []RunOutput
|
|
runErrors []orderedRunError
|
|
nextExtract int
|
|
active int
|
|
}
|
|
|
|
func (r *Runner) runLaneEngine(parent context.Context, config laneEngineConfig) ([]RunOutput, []orderedRunError) {
|
|
engine := newLaneEngine(r, parent, config)
|
|
return engine.run()
|
|
}
|
|
|
|
func newLaneEngine(r *Runner, parent context.Context, config laneEngineConfig) *laneEngine {
|
|
workerCount := config.input.Workers
|
|
if workerCount == 0 {
|
|
workerCount = DefaultWorkers
|
|
}
|
|
ctx, cancel := context.WithCancel(parent)
|
|
return &laneEngine{
|
|
runner: r, laneEngineConfig: config, ctx: ctx, cancel: cancel, workerCount: workerCount,
|
|
jobs: make(chan laneJob), results: make(chan laneJobResult, workerCount),
|
|
completedOutputs: make([]RunOutput, len(config.states)),
|
|
}
|
|
}
|
|
|
|
func (e *laneEngine) run() ([]RunOutput, []orderedRunError) {
|
|
defer e.cancel()
|
|
for _, state := range e.states {
|
|
if state.terminal {
|
|
e.completedOutputs[state.index] = state.output
|
|
} else if state.decision.Reused {
|
|
e.pending = append(e.pending, state)
|
|
}
|
|
}
|
|
e.startWorkers()
|
|
e.collect()
|
|
close(e.jobs)
|
|
e.workers.Wait()
|
|
return e.completedOutputs, e.runErrors
|
|
}
|
|
|
|
func (e *laneEngine) startWorkers() {
|
|
for i := 0; i < e.workerCount; i++ {
|
|
e.workers.Add(1)
|
|
go func() {
|
|
defer e.workers.Done()
|
|
for job := range e.jobs {
|
|
if job.chunk != nil {
|
|
result := e.runner.runExtractJob(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, extractJob{lane: job.lane, chunk: *job.chunk})
|
|
e.results <- laneJobResult{extract: &result}
|
|
continue
|
|
}
|
|
completion := laneCompletion{index: job.lane.index}
|
|
if err := e.ctx.Err(); err != nil {
|
|
completion.err = err
|
|
} else {
|
|
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.checkpoints, e.loader, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
|
|
}
|
|
e.results <- laneJobResult{completion: &completion}
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// nextJob gives ready continuations priority, then preserves chunk-first,
|
|
// lane-second extraction order. Dispatch remains lazy rather than creating a
|
|
// queue or goroutine for every extraction.
|
|
func (e *laneEngine) nextJob() (laneJob, bool) {
|
|
if len(e.pending) > 0 {
|
|
return laneJob{lane: e.pending[0]}, true
|
|
}
|
|
for e.nextExtract < len(e.chunks)*len(e.states) {
|
|
state := e.states[e.nextExtract%len(e.states)]
|
|
if !state.terminal && !state.decision.Reused {
|
|
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
|
|
}
|
|
e.nextExtract++
|
|
}
|
|
return laneJob{}, false
|
|
}
|
|
|
|
// collect alone owns dispatch and result accounting. Cancellation disables new
|
|
// dispatch, but every dispatched job still returns one result before workers exit.
|
|
func (e *laneEngine) collect() {
|
|
for {
|
|
if e.ctx.Err() != nil {
|
|
if e.active == 0 {
|
|
return
|
|
}
|
|
e.receiveResult(<-e.results)
|
|
continue
|
|
}
|
|
job, ready := e.nextJob()
|
|
if !ready && e.active == 0 {
|
|
return
|
|
}
|
|
var jobs chan<- laneJob
|
|
if ready && e.active < e.workerCount {
|
|
jobs = e.jobs
|
|
}
|
|
select {
|
|
case <-e.ctx.Done():
|
|
case result := <-e.results:
|
|
e.receiveResult(result)
|
|
case jobs <- job:
|
|
e.active++
|
|
if job.chunk == nil {
|
|
e.pending = e.pending[1:]
|
|
} else {
|
|
e.nextExtract++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *laneEngine) receiveResult(result laneJobResult) {
|
|
e.active--
|
|
if result.extract != nil {
|
|
e.handleExtractResult(*result.extract)
|
|
} else {
|
|
e.handleCompletion(*result.completion)
|
|
}
|
|
}
|
|
|
|
func (e *laneEngine) handleExtractResult(result extractJobResult) {
|
|
state := e.states[result.laneIndex]
|
|
state.remaining--
|
|
if result.err != nil {
|
|
state.failed = true
|
|
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err})
|
|
if state.reuseEligible {
|
|
_ = checkpointExtractFailed(e.checkpoints, e.input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
|
|
}
|
|
e.cancel()
|
|
} else {
|
|
state.results[result.chunkIndex] = result
|
|
}
|
|
if state.remaining != 0 || state.failed || e.ctx.Err() != nil {
|
|
return
|
|
}
|
|
if err := finalizeLaneExtract(e.checkpoints, e.input.stepID, state); err != nil {
|
|
state.failed = true
|
|
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: len(e.chunks), err: err})
|
|
e.cancel()
|
|
return
|
|
}
|
|
e.pending = append(e.pending, state)
|
|
}
|
|
|
|
func (e *laneEngine) handleCompletion(completion laneCompletion) {
|
|
e.completedOutputs[completion.index] = completion.output
|
|
if completion.err != nil {
|
|
e.runErrors = append(e.runErrors, classifyLaneError(completion.index, len(e.chunks), completion.err))
|
|
e.cancel()
|
|
}
|
|
}
|
|
|
|
func hydrateRequiredLane(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, index int, prepared preparedLaneExecutor) (*laneExtractState, error) {
|
|
lane, typed := prepared.resolved, prepared.typed
|
|
local := RunOutput{Manifest: manifestFromPipeline(input)}
|
|
checkpoint, decision := loader.AcceptedNormalize(input.stepID, lane.ID, lane.Normalize.Module)
|
|
if decision.Reused {
|
|
decision = checkpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
|
|
if checkpoint.Output.LaneID != lane.ID || checkpoint.Output.ModuleKey != lane.Normalize.Module || checkpoint.Output.SourceID != doc.ID {
|
|
decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactPayloadInvalid)
|
|
}
|
|
}
|
|
resolution, err := resolveCheckpointDecision(&local, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
|
|
decision = resolution.decision
|
|
state := &laneExtractState{index: index, prepared: prepared, decision: decision, reuseEligible: true, terminal: true, output: local}
|
|
if err != nil {
|
|
return state, err
|
|
}
|
|
hydrated := resolution.artifacts[0]
|
|
diagnostics, diagnosticErr := promoteCheckpointDiagnostics(hydrated.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
|
|
if diagnosticErr != nil {
|
|
return state, fmt.Errorf("promote reused accepted diagnostics: %w", diagnosticErr)
|
|
}
|
|
appendDiagnosticGroups(&local, diagnostics)
|
|
local.NormalizeOutputs = append(local.NormalizeOutputs, contracts.SerializedOutput{
|
|
StepID: input.stepID,
|
|
LaneID: lane.ID,
|
|
NormalizerKey: lane.Normalize.Module,
|
|
SourceID: doc.ID,
|
|
Artifact: contracts.CloneSerializedArtifact(hydrated.Artifact),
|
|
})
|
|
local.normalizeReuseEligibility = map[generatedOutputKey]bool{generatedOutputKeyFor(input.stepID, lane.ID): true}
|
|
state.output = local
|
|
return state, nil
|
|
}
|
|
|
|
func mergeTerminalLaneStates(output *RunOutput, states []*laneExtractState) error {
|
|
for _, state := range states {
|
|
if state == nil || !state.terminal {
|
|
continue
|
|
}
|
|
if err := mergeLaneOutput(output, state.output); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) error {
|
|
for i := range completedOutputs {
|
|
if err := mergeLaneOutput(output, completedOutputs[i]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, index int, prepared preparedLaneExecutor, output *RunOutput) (*laneExtractState, error) {
|
|
lane, typed := prepared.resolved, prepared.typed
|
|
digest, err := joinedChunkDigest(chunks)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
|
|
}
|
|
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
|
|
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
|
|
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
|
|
if !state.reuseEligible {
|
|
state.decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
|
|
recordCheckpointEvent(output, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, state.decision)
|
|
return state, nil
|
|
}
|
|
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
|
|
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
decision = resolution.decision
|
|
state.decision = decision
|
|
if decision.Reused {
|
|
state.remaining = 0
|
|
for i, stored := range resolution.artifacts {
|
|
value := resolution.values[i]
|
|
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: stored.ChunkID, ChunkIndex: stored.ChunkIndex, ChunkRef: stored.ChunkRef, Value: value}
|
|
if stored.ChunkIndex >= 0 && stored.ChunkIndex < len(chunks) && artifact.ChunkRef == (source.SourceRef{}) {
|
|
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
|
|
}
|
|
state.values = append(state.values, artifact)
|
|
state.serialized = append(state.serialized, cloneCheckpointArtifact(stored))
|
|
chunk := source.Chunk{ID: stored.ChunkID, Index: stored.ChunkIndex}
|
|
diagnostics, diagnosticErr := promoteCheckpointDiagnostics(stored.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
|
|
if diagnosticErr != nil {
|
|
return nil, fmt.Errorf("promote reused extract diagnostics: %w", diagnosticErr)
|
|
}
|
|
state.diagnostics = append(state.diagnostics, diagnostics...)
|
|
}
|
|
state.rejected = cloneRejectedOutputs(cp.Rejected)
|
|
}
|
|
return state, nil
|
|
}
|
|
|
|
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
|
|
state := job.lane
|
|
lane, typed := state.prepared.resolved, state.prepared.typed
|
|
result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
|
|
if err := ctx.Err(); err != nil {
|
|
result.err = err
|
|
return result
|
|
}
|
|
chunk, err := cloneSourceChunk(job.chunk)
|
|
if err != nil {
|
|
result.err = err
|
|
return result
|
|
}
|
|
metadata, err := cloneMetadata(input.Metadata)
|
|
if err != nil {
|
|
result.err = err
|
|
return result
|
|
}
|
|
references := operationReferenceSet(input, lane.ExtractReferences)
|
|
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution, chunk: &chunk, ref: chunk.Ref}
|
|
var assessment contracts.CandidateAssessment
|
|
assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators)
|
|
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
|
|
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
|
|
var err error
|
|
assessment, err = assess(ctx, value)
|
|
return assessment, err
|
|
},
|
|
}
|
|
var cacheDiagnostics []contracts.ProducerDiagnostic
|
|
operation.Cache = artifactProducerCache(input, typed, lane.ExtractExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, doc, &chunk, &cacheDiagnostics)
|
|
extracted, err := typed.extract(ctx, typed.extractor, contracts.TypedExtractionRequest[any]{Operation: operation, Source: doc, Chunk: &chunk, SourceInput: target.sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Extract.LLMProfile, Metadata: metadata})
|
|
if err != nil {
|
|
result.err = fmt.Errorf("extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
|
|
return result
|
|
}
|
|
value, outcome, err := completeProducerOperation(ctx, lane.ExtractExecutionClass, operation, extracted.Value, extracted.Outcome, extracted.Fallback)
|
|
if err != nil {
|
|
result.err = fmt.Errorf("accept extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
|
|
return result
|
|
}
|
|
stored, err := checkpointArtifact(typed.codec, lane.ID, lane.Extract.Module, doc.ID, value)
|
|
if err != nil {
|
|
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
|
|
return result
|
|
}
|
|
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = chunk.ID, chunk.Index, chunk.Ref
|
|
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
|
|
if err != nil {
|
|
result.err = err
|
|
return result
|
|
}
|
|
result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value}
|
|
result.serialized, result.outcome = stored, outcome
|
|
return result
|
|
}
|
|
|
|
func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *laneExtractState) error {
|
|
lane := state.prepared.resolved
|
|
indexes := make([]int, 0, len(state.results))
|
|
for index := range state.results {
|
|
indexes = append(indexes, index)
|
|
}
|
|
sort.Ints(indexes)
|
|
for _, index := range indexes {
|
|
result := state.results[index]
|
|
if result.rejected != nil {
|
|
state.rejected = append(state.rejected, *result.rejected)
|
|
state.diagnostics = append(state.diagnostics, result.diagnostics...)
|
|
continue
|
|
}
|
|
state.values = append(state.values, result.value)
|
|
state.serialized = append(state.serialized, result.serialized)
|
|
state.diagnostics = append(state.diagnostics, result.diagnostics...)
|
|
if !result.outcome.ReuseEligible {
|
|
state.reuseEligible = false
|
|
}
|
|
if result.outcome.Disposition == contracts.ProducerFallback {
|
|
state.fallbackCount++
|
|
}
|
|
}
|
|
sort.SliceStable(state.values, func(i, j int) bool { return state.values[i].ChunkIndex < state.values[j].ChunkIndex })
|
|
sort.SliceStable(state.serialized, func(i, j int) bool { return state.serialized[i].ChunkIndex < state.serialized[j].ChunkIndex })
|
|
sort.SliceStable(state.rejected, func(i, j int) bool { return state.rejected[i].ChunkIndex < state.rejected[j].ChunkIndex })
|
|
if !state.decision.Reused && state.reuseEligible {
|
|
if err := recordExtract(checkpoints, stepID, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected); err != nil {
|
|
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
|
|
lane := state.prepared.resolved
|
|
local := RunOutput{Manifest: manifestFromPipeline(input), FallbackCount: state.fallbackCount}
|
|
results := finalizedExtractResults{
|
|
accepted: state.values,
|
|
serialized: state.serialized,
|
|
diagnostics: state.diagnostics,
|
|
rejected: state.rejected,
|
|
decision: state.decision,
|
|
reuseEligible: state.reuseEligible,
|
|
}
|
|
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
|
|
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
|
|
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "decision": results.decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
|
|
return local, &laneRunError{stage: StageExtract, err: err}
|
|
}
|
|
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "outputs": debugCheckpointArtifacts(results.serialized), "rejected": debugRejectedOutputEnvelopes(results.rejected)}}); err != nil {
|
|
return local, &laneRunError{stage: StageExtract, err: err}
|
|
}
|
|
if len(results.accepted) == 0 {
|
|
return local, nil
|
|
}
|
|
err := r.continueTypedLane(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, state.prepared, results, &local)
|
|
return local, err
|
|
}
|
|
|
|
func classifyLaneError(lane, sentinel int, err error) orderedRunError {
|
|
stage := 1
|
|
var laneErr *laneRunError
|
|
if errors.As(err, &laneErr) {
|
|
switch laneErr.stage {
|
|
case StageExtract:
|
|
stage = 0
|
|
case StageNormalize:
|
|
stage = 2
|
|
}
|
|
} else if strings.Contains(strings.ToLower(err.Error()), "normalize") {
|
|
stage = 2
|
|
}
|
|
return orderedRunError{stage: stage, lane: lane, chunk: sentinel, err: err}
|
|
}
|
|
|
|
func selectRunError(parent context.Context, values []orderedRunError) error {
|
|
if err := parent.Err(); err != nil {
|
|
return err
|
|
}
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
hasReal := false
|
|
for _, value := range values {
|
|
if !errors.Is(value.err, context.Canceled) && !errors.Is(value.err, context.DeadlineExceeded) {
|
|
hasReal = true
|
|
break
|
|
}
|
|
}
|
|
filtered := values[:0]
|
|
for _, value := range values {
|
|
if hasReal && (errors.Is(value.err, context.Canceled) || errors.Is(value.err, context.DeadlineExceeded)) {
|
|
continue
|
|
}
|
|
filtered = append(filtered, value)
|
|
}
|
|
sort.SliceStable(filtered, func(i, j int) bool {
|
|
if filtered[i].stage != filtered[j].stage {
|
|
return filtered[i].stage < filtered[j].stage
|
|
}
|
|
if filtered[i].lane != filtered[j].lane {
|
|
return filtered[i].lane < filtered[j].lane
|
|
}
|
|
return filtered[i].chunk < filtered[j].chunk
|
|
})
|
|
return filtered[0].err
|
|
}
|
|
|
|
func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
|
|
if dst == nil {
|
|
return nil
|
|
}
|
|
dst.FallbackCount += src.FallbackCount
|
|
dst.NormalizeOutputs = append(dst.NormalizeOutputs, cloneSerializedOutputs(src.NormalizeOutputs)...)
|
|
dst.Rejected = append(dst.Rejected, cloneRejectedOutputs(src.Rejected)...)
|
|
appendDiagnosticGroups(dst, src.diagnosticGroups)
|
|
dst.CheckpointEvents = append(dst.CheckpointEvents, src.CheckpointEvents...)
|
|
dst.ValidationSummaries = append(dst.ValidationSummaries, cloneValidationSummaries(src.ValidationSummaries)...)
|
|
if len(src.normalizeReuseEligibility) > 0 {
|
|
if dst.normalizeReuseEligibility == nil {
|
|
dst.normalizeReuseEligibility = make(map[generatedOutputKey]bool, len(src.normalizeReuseEligibility))
|
|
}
|
|
for key, eligible := range src.normalizeReuseEligibility {
|
|
dst.normalizeReuseEligibility[key] = eligible
|
|
}
|
|
}
|
|
for i := range dst.Manifest.ArtifactLanes {
|
|
for j := range src.Manifest.ArtifactLanes {
|
|
if dst.Manifest.ArtifactLanes[i].ID == src.Manifest.ArtifactLanes[j].ID && dst.Manifest.ArtifactLanes[i].StepID == src.Manifest.ArtifactLanes[j].StepID && src.Manifest.ArtifactLanes[j].Metadata != nil {
|
|
metadata, err := cloneMetadata(src.Manifest.ArtifactLanes[j].Metadata)
|
|
if err != nil {
|
|
return fmt.Errorf("clone lane %q manifest metadata: %w", dst.Manifest.ArtifactLanes[i].ID, err)
|
|
}
|
|
dst.Manifest.ArtifactLanes[i].Metadata = metadata
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|