96 lines
4.3 KiB
Go
96 lines
4.3 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
type chunkPlanExecution struct {
|
|
outcome contracts.ProducerOutcome
|
|
accepted bool
|
|
chunks []source.Chunk
|
|
plan *source.ChunkPlan
|
|
diagnostics []contracts.DiagnosticGroup
|
|
}
|
|
|
|
// Chunk planning remains serial before lane workers. Only a native chunk
|
|
// producer uses the accepted-artifact cache; deterministic plans run directly.
|
|
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
|
|
chunker := input.Prepared.chunker
|
|
result := chunkPlanExecution{}
|
|
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}
|
|
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
|
|
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
|
|
var assessment contracts.CandidateAssessment
|
|
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible,
|
|
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
|
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
if err != nil {
|
|
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic {
|
|
return contracts.CandidateAssessment{}, err
|
|
}
|
|
assessment = contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "chunk materialization", Decision: contracts.AssessmentFailed, Failure: err}}}
|
|
return assessment, nil
|
|
}
|
|
assessment, err = assessChunks(ctx, chunks)
|
|
return assessment, err
|
|
},
|
|
}
|
|
var cacheDiagnostics []contracts.ProducerDiagnostic
|
|
if input.Cache != nil && input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
|
|
operation.Cache = &contracts.ProducerCache[source.ChunkPlan]{
|
|
Store: input.Cache,
|
|
Encode: func(plan source.ChunkPlan) ([]byte, error) {
|
|
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(canonical)
|
|
},
|
|
Decode: func(content []byte) (source.ChunkPlan, error) {
|
|
var plan source.ChunkPlan
|
|
decoder := json.NewDecoder(bytes.NewReader(content))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&plan); err != nil {
|
|
return source.ChunkPlan{}, err
|
|
}
|
|
var trailing any
|
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
|
return source.ChunkPlan{}, fmt.Errorf("cached chunk plan must contain one JSON value")
|
|
}
|
|
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
return canonical, err
|
|
},
|
|
Observe: producerCacheObserver(input, origin, nil, &cacheDiagnostics),
|
|
}
|
|
}
|
|
metadata, err := cloneMetadata(input.Metadata)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return result, err
|
|
}
|
|
produced, err := chunker.Plan(ctx, contracts.ChunkRequest{Operation: operation, Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(target.references), LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: metadata})
|
|
if err != nil {
|
|
return result, fmt.Errorf("chunk source with %q: %w", chunker.Key(), err)
|
|
}
|
|
plan, outcome, err := completeProducerOperation(ctx, input.pipeline.ChunkExecutionClass, operation, produced.Plan, produced.Outcome, produced.Fallback)
|
|
if err != nil {
|
|
return result, fmt.Errorf("accept chunk plan with %q: %w", chunker.Key(), err)
|
|
}
|
|
plan, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
if err != nil {
|
|
return result, fmt.Errorf("materialize final chunk plan: %w", err)
|
|
}
|
|
result.plan, result.chunks, result.accepted, result.outcome = &plan, chunks, true, outcome
|
|
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, cacheDiagnostics, assessment, outcome, origin, nil)
|
|
return result, err
|
|
}
|