2026-07-03 10:01:28 -05:00
package pipeline
2026-07-03 12:50:09 +00:00
import (
"context"
2026-07-05 17:51:36 +00:00
"crypto/sha256"
"encoding/hex"
2026-11-22 03:19:43 +00:00
"errors"
2026-07-03 12:50:09 +00:00
"fmt"
2026-07-05 17:51:36 +00:00
"mime"
2026-07-04 00:45:21 +00:00
"path"
2026-07-05 17:51:36 +00:00
"path/filepath"
2026-07-04 00:45:21 +00:00
"strings"
"time"
2026-07-03 12:50:09 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
2026-07-23 14:58:14 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
2026-07-03 12:50:09 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
2026-11-22 03:19:43 +00:00
"go.maximumdirect.net/promptkit"
2026-07-03 12:50:09 +00:00
)
2026-07-03 15:46:42 +00:00
type Registries struct {
2026-07-27 18:04:12 +00:00
Inputs * InputAdapterRegistry
Chunkers * ChunkerRegistry
ArtifactCodecs * ArtifactCodecRegistry
ArtifactEvidence * ArtifactEvidenceRegistry
Extractors * ExtractorRegistry
Mergers * MergerRegistry
Normalizers * NormalizerRegistry
Validators * ValidatorRegistry
ValidatorChains * ValidatorChainRegistry
Outputs * OutputEncoderRegistry
2026-07-03 12:50:09 +00:00
}
2026-07-17 06:18:46 +00:00
type Runner struct { }
2026-07-03 12:50:09 +00:00
2026-11-22 01:45:31 +00:00
// DefaultWorkers is the shared ready-job limit when the caller omits one.
const DefaultWorkers = 64
2026-07-17 06:18:46 +00:00
func New ( ) * Runner {
return & Runner { }
2026-07-03 12:50:09 +00:00
}
type RunInput struct {
2026-11-22 02:10:05 +00:00
Cache contracts . ArtifactCache
// ObserveCache receives scoped transient evidence, including prepared details
// on a hit. Shared observers must synchronize concurrent calls.
2026-11-22 02:24:45 +00:00
ObserveCache func ( context . Context , contracts . DiagnosticOrigin , contracts . CacheObservation ) error
Execution contracts . ExecutionOptions
2026-11-22 03:19:43 +00:00
// Executions is the invocation's native group. The runner enrolls its
// requests; callers may use Snapshot without maintaining copied totals.
Executions * promptkit . ExecutionGroup
Prepared * PreparedPipeline
SourceID string
Path string
RawInput [ ] byte
SessionID string
RunID string
StartedAt time . Time
Metadata map [ string ] any
Diagnostics [ ] contracts . ProducerDiagnostic
2026-11-22 02:24:45 +00:00
Debug DebugRecorder
2026-11-22 01:45:31 +00:00
// Workers bounds ready extraction and lane-completion jobs together.
// Zero selects DefaultWorkers; configuration rejects explicit zero.
Workers int
2026-07-17 06:18:46 +00:00
2026-11-21 23:48:51 +00:00
pipeline ResolvedPipeline
// reuseIneligible carries chunk ancestry; reference eligibility stays scoped.
reuseIneligible bool
2026-08-27 11:50:37 +00:00
stepID string
references map [ referenceTargetKey ] contracts . ReferenceSet
referenceReuseEligibility map [ referenceTargetKey ] bool
2026-07-03 12:50:09 +00:00
}
type RunOutput struct {
2026-11-22 02:34:40 +00:00
FallbackCount int ` json:"fallback_count,omitempty" `
2026-11-22 02:24:45 +00:00
2026-11-22 02:34:40 +00:00
Diagnostics contracts . DiagnosticCollection ` json:"diagnostics,omitempty" `
OutputFiles [ ] contracts . OutputFile ` json:"-" `
2026-08-27 11:50:37 +00:00
2026-11-22 02:34:40 +00:00
normalizeOutputs [ ] contracts . SerializedOutput
2026-08-27 11:50:37 +00:00
normalizeReuseEligibility map [ generatedOutputKey ] bool
2026-08-27 15:29:10 +00:00
diagnosticGroups [ ] contracts . DiagnosticGroup
2026-07-03 12:50:09 +00:00
}
2026-11-22 02:34:40 +00:00
// ArtifactCount counts logical deliverables, not intermediate lane values,
// cache entries, diagnostic records, or debug files.
func ( output RunOutput ) ArtifactCount ( ) int {
return len ( output . OutputFiles )
}
2026-07-05 18:21:37 +00:00
func ( r * Runner ) Run ( ctx context . Context , input RunInput ) ( output RunOutput , err error ) {
2026-07-03 12:50:09 +00:00
if r == nil {
return output , fmt . Errorf ( "runner must not be nil" )
}
2026-08-09 00:50:02 +00:00
if err := ctx . Err ( ) ; err != nil {
return output , err
}
2026-07-03 15:46:42 +00:00
if err := validateRunInput ( input ) ; err != nil {
return output , err
2026-07-03 12:50:09 +00:00
}
2026-07-17 23:16:16 -05:00
input . Metadata , err = cloneMetadata ( input . Metadata )
if err != nil {
return output , fmt . Errorf ( "clone run metadata: %w" , err )
}
2026-07-17 06:18:46 +00:00
input . pipeline = input . Prepared . resolved
2026-07-03 15:46:42 +00:00
2026-08-03 19:45:27 +00:00
sessionID := strings . TrimSpace ( input . SessionID )
2026-07-08 03:14:59 +00:00
debugRecorder := input . Debug
if debugRecorder == nil {
debugRecorder = NoopDebugRecorder ( )
}
2026-07-17 08:37:52 +00:00
debugRecorder = synchronizedDebugRecorder ( debugRecorder )
2026-07-08 03:14:59 +00:00
input . Debug = debugRecorder
2026-11-22 03:19:43 +00:00
if input . Executions == nil {
input . Executions = & promptkit . ExecutionGroup { }
}
input . Execution . Observer = promptkit . ComposeObservers ( input . Execution . Observer , input . Executions . Observe )
defer func ( ) {
if debugRecorder . Enabled ( ) {
if captureErr := debugRecorder . WriteJSON ( "executions.json" , input . Executions . Snapshot ( ) ) ; captureErr != nil {
err = errors . Join ( err , & contracts . CaptureError { Cause : captureErr } )
output = failOutput ( output )
}
}
} ( )
2026-08-27 15:29:10 +00:00
inputDiagnostics , diagnosticErr := promoteProducerDiagnostics ( input . Diagnostics , contracts . DiagnosticOrigin { Stage : contracts . DiagnosticOriginStageReferences } , nil )
if diagnosticErr != nil {
return failOutput ( output ) , fmt . Errorf ( "promote input diagnostics: %w" , diagnosticErr )
}
appendDiagnosticGroups ( & output , inputDiagnostics )
2026-11-22 03:19:43 +00:00
if err := writeDebugTimed ( debugRecorder , "pipeline.json" , debugTimedEnvelope {
2026-07-08 03:14:59 +00:00
Stage : "run" ,
StartedAt : startedTime ( input . StartedAt ) ,
Payload : map [ string ] any {
2026-07-17 06:18:46 +00:00
"pipeline_id" : input . pipeline . ID ,
"pipeline_digest" : input . pipeline . Digest ,
2026-11-22 02:34:40 +00:00
"run_id" : input . RunID ,
2026-07-08 03:14:59 +00:00
} ,
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write debug run artifact: %w" , err )
}
2026-11-22 03:19:43 +00:00
if debugRecorder . Enabled ( ) {
if err := debugRecorder . WriteJSON ( "composition.json" , captureComposition ( input . pipeline ) ) ; err != nil {
return failOutput ( output ) , & contracts . CaptureError { Cause : err }
}
}
2026-07-03 15:46:42 +00:00
2026-07-17 06:18:46 +00:00
adapter := input . Prepared . input
2026-11-22 02:24:45 +00:00
var doc * source . SourceDocument
2026-07-08 03:14:59 +00:00
sourceStarted := time . Now ( ) . UTC ( )
if err := writeDebugTimed ( debugRecorder , "source/input.json" , debugTimedEnvelope {
Stage : "source" ,
ModuleKey : adapter . Key ( ) ,
StartedAt : sourceStarted ,
Payload : debugSourceInput {
SourceID : input . SourceID ,
Path : input . Path ,
Raw : debugContentEnvelope ( input . RawInput , sourceInputMediaType ( input . Path ) , nil , nil ) ,
2026-11-22 03:19:43 +00:00
Metadata : input . Metadata ,
2026-07-08 03:14:59 +00:00
} ,
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write source debug artifact: %w" , err )
}
2026-11-22 02:24:45 +00:00
requestMetadata , err := cloneMetadata ( input . Metadata )
if err != nil {
return failOutput ( output ) , fmt . Errorf ( "clone input adapter metadata: %w" , err )
}
if err := ctx . Err ( ) ; err != nil {
return failOutput ( output ) , err
}
doc , err = adapter . Parse ( ctx , contracts . ParseRequest {
SourceID : input . SourceID , Path : input . Path , Raw : input . RawInput ,
LLMProfile : input . pipeline . Input . LLMProfile , Metadata : requestMetadata ,
} )
if ctxErr := ctx . Err ( ) ; ctxErr != nil {
return failOutput ( output ) , ctxErr
2026-07-08 02:46:38 +00:00
}
2026-11-22 02:24:45 +00:00
if err != nil {
return failOutput ( output ) , fmt . Errorf ( "parse input with adapter %q: %w" , adapter . Key ( ) , err )
}
if err := source . ValidateDocument ( doc ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "validate source document: %w" , err )
}
2026-07-08 03:14:59 +00:00
if err := writeDebugTimed ( debugRecorder , "source/output.json" , debugTimedEnvelope {
Stage : "source" ,
ModuleKey : adapter . Key ( ) ,
StartedAt : sourceStarted ,
Payload : map [ string ] any {
"document" : debugSourceDocumentEnvelope ( doc ) ,
} ,
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write source debug artifact: %w" , err )
}
2026-07-05 17:51:36 +00:00
sourceInput := sourceInputMaterial ( input . Path , input . RawInput )
2026-07-03 15:46:42 +00:00
2026-07-17 06:18:46 +00:00
chunker := input . Prepared . chunker
2026-07-08 03:14:59 +00:00
chunkStarted := time . Now ( ) . UTC ( )
if err := writeDebugTimed ( debugRecorder , "chunk/input.json" , debugTimedEnvelope {
Stage : string ( StageChunk ) ,
ModuleKey : chunker . Key ( ) ,
StartedAt : chunkStarted ,
Payload : map [ string ] any {
2026-11-22 02:24:45 +00:00
"cache_mode" : input . pipeline . ChunkControls . Cache ,
2026-07-08 03:14:59 +00:00
"source" : debugSourceDocumentEnvelope ( doc ) ,
"source_input" : debugContentEnvelope ( sourceInput . Content , sourceInput . MediaType , nil , nil ) ,
2026-11-22 03:19:43 +00:00
"metadata" : input . Metadata ,
2026-07-08 03:14:59 +00:00
} ,
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write chunk debug artifact: %w" , err )
}
2026-07-18 00:20:56 +00:00
chunkResult , err := r . runChunkPlan ( ctx , input , doc , sourceInput , sessionID )
if err != nil {
return failOutput ( output ) , err
}
2026-08-27 15:29:10 +00:00
appendDiagnosticGroups ( & output , chunkResult . diagnostics )
2026-07-08 03:14:59 +00:00
chunkDebugPayload := map [ string ] any {
2026-11-22 02:24:45 +00:00
"cache_mode" : input . pipeline . ChunkControls . Cache ,
2026-07-18 00:20:56 +00:00
"accepted" : chunkResult . accepted ,
"materialized_chunks" : debugSourceChunkEnvelopes ( chunkResult . chunks ) ,
2026-07-17 23:57:59 +00:00
}
2026-07-18 00:20:56 +00:00
if chunkResult . plan != nil {
chunkDebugPayload [ "plan" ] = debugChunkPlanEnvelope ( * chunkResult . plan )
2026-07-08 03:14:59 +00:00
}
if err := writeDebugTimed ( debugRecorder , "chunk/output.json" , debugTimedEnvelope {
Stage : string ( StageChunk ) ,
ModuleKey : chunker . Key ( ) ,
StartedAt : chunkStarted ,
Payload : chunkDebugPayload ,
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write chunk debug artifact: %w" , err )
}
2026-07-03 12:50:09 +00:00
2026-07-23 14:58:14 +00:00
var acceptedChunkMap * contracts . SerializedArtifact
if chunkResult . accepted {
2026-11-22 02:24:45 +00:00
if chunkResult . plan == nil {
return failOutput ( output ) , fmt . Errorf ( "accepted chunk plan is missing" )
}
chunkProfile := ""
if input . pipeline . ChunkExecutionClass == contracts . ExecutionClassLLMBacked {
chunkProfile = input . pipeline . Chunk . LLMProfile
2026-07-23 14:58:14 +00:00
}
artifact , buildErr := chunkmap . Serialize ( chunkmap . BuildRequest {
Source : doc ,
Plan : * chunkResult . plan ,
Chunks : chunkResult . chunks ,
RequestedChunker : input . pipeline . Chunk . Module ,
Producer : chunkmap . Producer {
2026-11-22 02:24:45 +00:00
InputModule : adapter . Key ( ) ,
ChunkModule : chunker . Key ( ) ,
LLMProfile : chunkProfile ,
2026-07-23 14:58:14 +00:00
} ,
} )
if buildErr != nil {
return failOutput ( output ) , fmt . Errorf ( "serialize accepted chunk map: %w" , buildErr )
}
acceptedChunkMap = & artifact
}
2026-07-18 00:20:56 +00:00
if chunkResult . accepted {
2026-11-21 23:48:51 +00:00
input . reuseIneligible = ! chunkResult . outcome . ReuseEligible
if chunkResult . outcome . Disposition == contracts . ProducerFallback {
output . FallbackCount ++
}
2026-11-22 02:24:45 +00:00
if err := r . runPreparedSteps ( ctx , input , doc , sourceInput , sessionID , chunkResult . chunks , & output ) ; err != nil {
2026-07-22 02:03:27 +00:00
return failOutput ( output ) , err
2026-07-03 12:50:09 +00:00
}
2026-07-03 15:46:42 +00:00
}
2026-08-09 00:50:02 +00:00
if err := ctx . Err ( ) ; err != nil {
return failOutput ( output ) , err
}
2026-07-03 15:46:42 +00:00
2026-08-27 15:29:10 +00:00
if err := finalizeDiagnostics ( & output ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "aggregate diagnostics: %w" , err )
}
2026-07-03 15:46:42 +00:00
2026-07-17 06:18:46 +00:00
encoder := input . Prepared . output
2026-08-09 00:50:02 +00:00
if err := ctx . Err ( ) ; err != nil {
return failOutput ( output ) , err
}
2026-07-08 03:14:59 +00:00
outputStarted := time . Now ( ) . UTC ( )
2026-11-22 02:34:40 +00:00
evidenceArtifact , evidenceSummary , err := buildOutputEvidenceContext ( input . Prepared , doc , output . normalizeOutputs )
2026-07-27 18:17:54 +00:00
if err != nil {
return failOutput ( output ) , err
}
2026-08-09 00:50:02 +00:00
if err := ctx . Err ( ) ; err != nil {
return failOutput ( output ) , err
}
2026-07-27 18:17:54 +00:00
if evidenceSummary != nil {
if err := writeDebugTimed ( debugRecorder , "output/evidence-context.json" , debugTimedEnvelope {
Stage : string ( StageOutput ) ,
ModuleKey : encoder . Key ( ) ,
StartedAt : outputStarted ,
Payload : * evidenceSummary ,
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write evidence context debug artifact: %w" , err )
}
2026-08-09 00:50:02 +00:00
if err := ctx . Err ( ) ; err != nil {
return failOutput ( output ) , err
}
2026-07-27 18:17:54 +00:00
}
2026-07-23 14:58:14 +00:00
outputDebugPayload := map [ string ] any {
2026-11-22 02:34:40 +00:00
"normalize_outputs" : debugSerializedOutputEnvelopes ( output . normalizeOutputs ) ,
2026-11-22 03:19:43 +00:00
"metadata" : input . Metadata ,
2026-07-23 14:58:14 +00:00
}
if acceptedChunkMap != nil {
outputDebugPayload [ "chunk_map" ] = debugSerializedOutputEnvelope ( contracts . SerializedOutput { Artifact : * acceptedChunkMap } )
}
2026-07-08 03:14:59 +00:00
if err := writeDebugTimed ( debugRecorder , "output/input.json" , debugTimedEnvelope {
Stage : string ( StageOutput ) ,
ModuleKey : encoder . Key ( ) ,
StartedAt : outputStarted ,
2026-07-23 14:58:14 +00:00
Payload : outputDebugPayload ,
2026-07-08 03:14:59 +00:00
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write output debug artifact: %w" , err )
}
2026-08-09 00:50:02 +00:00
if err := ctx . Err ( ) ; err != nil {
return failOutput ( output ) , err
}
2026-07-03 15:46:42 +00:00
encoded , err := encoder . Encode ( ctx , contracts . OutputRequest {
2026-11-22 02:34:40 +00:00
NormalizeOutputs : cloneSerializedOutputs ( output . normalizeOutputs ) ,
2026-11-21 23:48:51 +00:00
ChunkMap : contracts . CloneSerializedArtifactPointer ( acceptedChunkMap ) ,
EvidenceContext : contracts . CloneSerializedArtifactPointer ( evidenceArtifact ) ,
2026-07-03 15:46:42 +00:00
} )
2026-08-09 00:50:02 +00:00
if ctxErr := ctx . Err ( ) ; ctxErr != nil {
return failOutput ( output ) , ctxErr
}
2026-07-03 15:46:42 +00:00
if err != nil {
return failOutput ( output ) , fmt . Errorf ( "encode output with encoder %q: %w" , encoder . Key ( ) , err )
}
2026-07-04 00:45:21 +00:00
files , err := outputFilesFromResult ( encoded )
if err != nil {
return failOutput ( output ) , fmt . Errorf ( "validate output files from encoder %q: %w" , encoder . Key ( ) , err )
}
2026-07-08 03:14:59 +00:00
if err := writeDebugTimed ( debugRecorder , "output/output.json" , debugTimedEnvelope {
Stage : string ( StageOutput ) ,
ModuleKey : encoder . Key ( ) ,
StartedAt : outputStarted ,
Payload : map [ string ] any {
2026-08-27 15:33:13 +00:00
"files" : debugOutputFiles ( files ) ,
2026-07-08 03:14:59 +00:00
} ,
} ) ; err != nil {
return failOutput ( output ) , fmt . Errorf ( "write output debug artifact: %w" , err )
}
2026-08-09 00:50:02 +00:00
if err := ctx . Err ( ) ; err != nil {
return failOutput ( output ) , err
}
output . OutputFiles = files
2026-07-03 15:46:42 +00:00
return output , nil
}
2026-11-22 02:24:45 +00:00
func ( r * Runner ) runPreparedSteps ( ctx context . Context , input RunInput , doc * source . SourceDocument , sourceInput contracts . LLMInputMaterial , sessionID string , chunks [ ] source . Chunk , output * RunOutput ) error {
2026-07-22 02:03:27 +00:00
for _ , step := range input . Prepared . Steps {
stepInput := input
stepInput . stepID = step . ID
2026-11-22 02:34:40 +00:00
stepReferences , err := buildStepReferenceSets ( input , step , output . normalizeOutputs )
2026-07-22 02:03:27 +00:00
if err != nil {
return fmt . Errorf ( "prepare generated references for pipeline step %q: %w" , step . ID , err )
}
stepInput . references = stepReferences
2026-08-27 11:50:37 +00:00
stepInput . referenceReuseEligibility = buildStepReferenceReuseEligibility ( step , output . normalizeReuseEligibility )
2026-11-22 02:24:45 +00:00
laneOutput , laneErr := r . runLanes ( ctx , stepInput , step , doc , sourceInput , sessionID , chunks )
2026-11-22 02:34:40 +00:00
mergeLaneOutput ( output , laneOutput )
2026-07-22 02:03:27 +00:00
if laneErr != nil {
return fmt . Errorf ( "execute pipeline step %q: %w" , step . ID , laneErr )
}
}
return nil
}
2026-07-07 21:10:29 +00:00
func resolvedValidatorChain ( stage ModuleStage , laneID string , moduleKey string , chains [ ] ResolvedValidatorChain ) ResolvedValidatorChain {
for _ , chain := range chains {
if chain . Stage != stage {
continue
}
if chain . ModuleKey != moduleKey {
continue
}
if strings . TrimSpace ( chain . LaneID ) != strings . TrimSpace ( laneID ) {
continue
}
return ResolvedValidatorChain {
Stage : chain . Stage ,
LaneID : chain . LaneID ,
ModuleKey : chain . ModuleKey ,
Validators : cloneResolvedValidators ( chain . Validators ) ,
}
}
return ResolvedValidatorChain {
Stage : stage ,
LaneID : strings . TrimSpace ( laneID ) ,
ModuleKey : strings . TrimSpace ( moduleKey ) ,
}
}
2026-07-17 06:18:46 +00:00
func validateRunInput ( input RunInput ) error {
2026-11-22 01:45:31 +00:00
if input . Workers < 0 {
return fmt . Errorf ( "workers must be positive when specified" )
}
2026-07-17 06:18:46 +00:00
if input . Prepared == nil {
return fmt . Errorf ( "prepared pipeline must not be nil" )
2026-07-03 15:46:42 +00:00
}
2026-07-21 21:19:00 +00:00
if err := validateResolvedPipeline ( input . Prepared . resolved ) ; err != nil {
return err
}
2026-07-21 21:39:23 +00:00
return nil
2026-07-03 15:46:42 +00:00
}
2026-07-17 07:19:58 +00:00
func validateResolvedPipeline ( pipeline ResolvedPipeline ) error {
2026-07-17 06:18:46 +00:00
if pipeline . ID == "" {
2026-07-03 15:46:42 +00:00
return fmt . Errorf ( "resolved pipeline id must not be empty" )
}
2026-07-17 06:18:46 +00:00
if pipeline . Digest == "" {
2026-07-03 15:46:42 +00:00
return fmt . Errorf ( "resolved pipeline digest must not be empty" )
}
2026-07-17 06:18:46 +00:00
if pipeline . Input . Module == "" {
2026-07-03 15:46:42 +00:00
return fmt . Errorf ( "resolved pipeline input module must not be empty" )
}
2026-07-17 06:18:46 +00:00
if pipeline . Chunk . Module == "" {
2026-07-03 15:46:42 +00:00
return fmt . Errorf ( "resolved pipeline chunk module must not be empty" )
}
2026-07-17 06:18:46 +00:00
if pipeline . Output . Module == "" {
2026-07-03 15:46:42 +00:00
return fmt . Errorf ( "resolved pipeline output module must not be empty" )
}
2026-07-21 21:05:28 +00:00
if len ( pipeline . Steps ) == 0 {
2026-07-03 15:46:42 +00:00
return fmt . Errorf ( "resolved pipeline artifact lanes must not be empty" )
}
2026-07-21 21:05:28 +00:00
for _ , lane := range pipeline . AllArtifactLanes ( ) {
2026-07-03 15:46:42 +00:00
if lane . ID == "" {
return fmt . Errorf ( "resolved pipeline artifact lane id must not be empty" )
}
if lane . Extract . Module == "" {
return fmt . Errorf ( "resolved pipeline lane %q extract module must not be empty" , lane . ID )
}
if lane . Merge . Module == "" {
return fmt . Errorf ( "resolved pipeline lane %q merge module must not be empty" , lane . ID )
}
if lane . Normalize . Module == "" {
return fmt . Errorf ( "resolved pipeline lane %q normalize module must not be empty" , lane . ID )
}
2026-07-07 15:32:35 -05:00
if len ( lane . Validators ) > 0 {
2026-07-07 21:21:12 +00:00
return fmt . Errorf ( "resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators" , lane . ID )
2026-07-03 15:46:42 +00:00
}
}
return nil
}
2026-07-08 03:14:59 +00:00
func startedTime ( t time . Time ) time . Time {
if t . IsZero ( ) {
return time . Now ( ) . UTC ( )
}
return t . UTC ( )
}
2026-07-03 15:46:42 +00:00
func failOutput ( output RunOutput ) RunOutput {
2026-08-27 16:40:17 +00:00
// Preserve any diagnostics already accepted by the pipeline when a later
// operation fails. Producers and validators validate each diagnostic before
// it is appended, so finalization here cannot introduce a new failure path.
_ = finalizeDiagnostics ( & output )
2026-07-03 15:46:42 +00:00
return output
}
2026-07-04 00:45:21 +00:00
func outputFilesFromResult ( result contracts . OutputResult ) ( [ ] contracts . OutputFile , error ) {
2026-07-03 21:26:05 -05:00
out := make ( [ ] contracts . OutputFile , 0 , len ( result . Files ) )
for _ , file := range result . Files {
2026-07-04 00:45:21 +00:00
if err := validateOutputFileName ( file . Name ) ; err != nil {
return nil , err
}
out = append ( out , contracts . OutputFile {
Name : file . Name ,
ContentType : file . ContentType ,
Bytes : append ( [ ] byte ( nil ) , file . Bytes ... ) ,
} )
}
return out , nil
}
func validateOutputFileName ( name string ) error {
if strings . TrimSpace ( name ) == "" {
return fmt . Errorf ( "output file name must not be empty" )
}
if strings . Contains ( name , "\\" ) {
return fmt . Errorf ( "output file name %q must use slash-separated relative paths" , name )
}
if path . IsAbs ( name ) {
return fmt . Errorf ( "output file name %q must be relative" , name )
}
if strings . Contains ( name , ".." ) {
return fmt . Errorf ( "output file name %q must not contain .." , name )
}
cleaned := path . Clean ( name )
if cleaned == "." || cleaned != name {
return fmt . Errorf ( "output file name %q must be clean" , name )
}
return nil
}
2026-07-17 23:16:16 -05:00
func cloneMetadata ( metadata map [ string ] any ) ( map [ string ] any , error ) {
return source . CloneMetadata ( metadata )
2026-07-18 00:45:01 +00:00
}
2026-07-05 17:51:36 +00:00
func sourceInputMaterial ( inputPath string , content [ ] byte ) contracts . LLMInputMaterial {
return contracts . NewLLMInputMaterial (
"source" ,
sourceInputMediaType ( inputPath ) ,
content ,
sourceInputDigest ( content ) ,
sourceInputOriginURI ( inputPath ) ,
)
}
2026-07-17 05:32:11 +00:00
func chunkInputMaterial ( sourceInput contracts . LLMInputMaterial , chunk source . Chunk ) contracts . LLMInputMaterial {
2026-07-07 15:32:35 -05:00
return contracts . NewLLMInputMaterial (
"source" ,
chunk . MediaType ,
chunk . Content ,
sourceInputDigest ( chunk . Content ) ,
sourceInput . OriginURI ,
)
}
2026-07-05 17:51:36 +00:00
func sourceInputMediaType ( inputPath string ) string {
extension := strings . ToLower ( filepath . Ext ( strings . TrimSpace ( inputPath ) ) )
if extension == ".json" {
return "application/json"
}
mediaType := mime . TypeByExtension ( extension )
if strings . TrimSpace ( mediaType ) == "" {
return unknownMediaType
}
return canonicalMediaType ( mediaType )
}
func sourceInputDigest ( content [ ] byte ) string {
sum := sha256 . Sum256 ( content )
return "sha256:" + hex . EncodeToString ( sum [ : ] )
}
func sourceInputOriginURI ( inputPath string ) string {
if strings . TrimSpace ( inputPath ) == "" {
return ""
}
return fileURI ( inputPath )
}
2026-07-17 23:16:16 -05:00
func cloneSourceChunkPtr ( chunk * source . Chunk ) ( * source . Chunk , error ) {
2026-07-07 21:27:28 +00:00
if chunk == nil {
2026-07-17 23:16:16 -05:00
return nil , nil
}
cloned , err := cloneSourceChunk ( * chunk )
if err != nil {
return nil , err
2026-07-07 21:27:28 +00:00
}
2026-07-17 23:16:16 -05:00
return & cloned , nil
2026-07-07 21:27:28 +00:00
}
2026-07-17 23:16:16 -05:00
func cloneSourceChunk ( chunk source . Chunk ) ( source . Chunk , error ) {
2026-07-07 21:27:28 +00:00
chunk . Content = append ( [ ] byte ( nil ) , chunk . Content ... )
2026-07-17 23:16:16 -05:00
var err error
chunk . Units , err = cloneSourceUnits ( chunk . Units )
if err != nil {
return source . Chunk { } , err
}
chunk . Metadata , err = cloneMetadata ( chunk . Metadata )
if err != nil {
return source . Chunk { } , fmt . Errorf ( "clone chunk metadata: %w" , err )
}
2026-07-17 23:47:22 +00:00
chunk . Annotations = source . CloneChunkAnnotations ( chunk . Annotations )
chunk . PlanAnnotations = source . CloneChunkAnnotations ( chunk . PlanAnnotations )
2026-07-17 23:16:16 -05:00
return chunk , nil
2026-07-07 21:27:28 +00:00
}
2026-07-17 23:16:16 -05:00
func cloneSourceChunks ( chunks [ ] source . Chunk ) ( [ ] source . Chunk , error ) {
2026-07-07 21:27:28 +00:00
if len ( chunks ) == 0 {
2026-07-17 23:16:16 -05:00
return nil , nil
2026-07-07 21:27:28 +00:00
}
2026-07-17 05:32:11 +00:00
out := make ( [ ] source . Chunk , 0 , len ( chunks ) )
2026-07-07 21:27:28 +00:00
for _ , chunk := range chunks {
2026-07-17 23:16:16 -05:00
cloned , err := cloneSourceChunk ( chunk )
if err != nil {
return nil , err
}
out = append ( out , cloned )
2026-07-07 21:27:28 +00:00
}
2026-07-17 23:16:16 -05:00
return out , nil
2026-07-07 21:27:28 +00:00
}
2026-07-17 23:16:16 -05:00
func cloneSourceUnits ( units [ ] source . SourceUnit ) ( [ ] source . SourceUnit , error ) {
2026-07-07 21:27:28 +00:00
if len ( units ) == 0 {
2026-07-17 23:16:16 -05:00
return nil , nil
2026-07-07 21:27:28 +00:00
}
out := make ( [ ] source . SourceUnit , 0 , len ( units ) )
for _ , unit := range units {
2026-07-17 23:16:16 -05:00
cloned , err := cloneSourceUnit ( unit )
if err != nil {
return nil , err
}
out = append ( out , cloned )
2026-07-07 21:27:28 +00:00
}
2026-07-17 23:16:16 -05:00
return out , nil
2026-07-07 21:27:28 +00:00
}
2026-07-17 07:33:49 +00:00
func cloneSerializedOutputs ( outputs [ ] contracts . SerializedOutput ) [ ] contracts . SerializedOutput {
2026-07-07 18:58:23 +00:00
if len ( outputs ) == 0 {
return nil
2026-07-03 12:50:09 +00:00
}
2026-07-17 07:33:49 +00:00
out := make ( [ ] contracts . SerializedOutput , 0 , len ( outputs ) )
2026-07-07 18:58:23 +00:00
for _ , output := range outputs {
2026-07-17 07:33:49 +00:00
out = append ( out , contracts . CloneSerializedOutput ( output ) )
2026-07-07 18:58:23 +00:00
}
return out
2026-07-03 12:50:09 +00:00
}