247 lines
9.8 KiB
Go
247 lines
9.8 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
type chunkPlanExecution struct {
|
|
outcome contracts.ProducerOutcome
|
|
accepted bool
|
|
chunks []source.Chunk
|
|
plan *source.ChunkPlan
|
|
diagnostics []contracts.DiagnosticGroup
|
|
rejection *contracts.RejectedOutput
|
|
lookup ChunkPlanDecision
|
|
record *ChunkPlanRecord
|
|
action string
|
|
summary artifacts.ChunkPlanSummary
|
|
validation *artifacts.ValidationSummary
|
|
}
|
|
|
|
func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode {
|
|
if mode == "" {
|
|
return ChunkCacheBypass
|
|
}
|
|
parsed, _ := ParseChunkCacheMode(string(mode))
|
|
return parsed
|
|
}
|
|
|
|
// Chunk-plan lookup and publication happen serially before lane workers start.
|
|
// The runner therefore does not add synchronization around the store.
|
|
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
|
|
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
|
|
chunker := input.Prepared.chunker
|
|
result := chunkPlanExecution{lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"}, summary: artifacts.ChunkPlanSummary{Mode: string(mode), SourceDigest: doc.Digest, RequestedModule: chunker.Key(), LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped", ValidationStatus: "not_run", PublicationStatus: "not_requested"}}
|
|
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
|
|
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
|
|
var assessment contracts.CandidateAssessment
|
|
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: true,
|
|
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
|
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
if err != nil {
|
|
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic {
|
|
return contracts.CandidateAssessment{}, err
|
|
}
|
|
assessment = contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "chunk materialization", Decision: contracts.AssessmentFailed, Failure: err}}}
|
|
return assessment, nil
|
|
}
|
|
assessment, err = assessChunks(ctx, chunks)
|
|
return assessment, err
|
|
},
|
|
}
|
|
if mode == ChunkCacheAuto {
|
|
record, decision, err := input.ChunkPlans.Load(doc.Digest)
|
|
if err != nil {
|
|
return result, fmt.Errorf("load chunk plan: %w", err)
|
|
}
|
|
result.lookup = decision
|
|
result.summary.LookupStatus, result.summary.LookupReason = chunkPlanLookupStatus(decision.Status), chunkPlanLookupReason(decision.Status)
|
|
if decision.Status == ChunkPlanHit {
|
|
plan, chunks, err := validateAndMaterializeChunkPlan(doc, record.Plan)
|
|
if err == nil {
|
|
cachedAssessment, err := assessChunks(ctx, chunks)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if cachedAssessment.Accepted() {
|
|
if err := result.setCandidate(record, "reused"); err != nil {
|
|
return result, err
|
|
}
|
|
result.plan, result.chunks, result.accepted = &plan, chunks, true
|
|
result.outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}
|
|
result.diagnostics, err = operationDiagnosticGroups(record.Diagnostics, cachedAssessment, result.outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
|
|
result.setValidation(nil, err)
|
|
return result, err
|
|
}
|
|
}
|
|
result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: chunkPlanLookupReason(ChunkPlanInvalid)}
|
|
result.summary.LookupStatus, result.summary.LookupReason = "invalid", chunkPlanLookupReason(ChunkPlanInvalid)
|
|
} else if decision.Status != ChunkPlanMissing && decision.Status != ChunkPlanInvalid {
|
|
return result, fmt.Errorf("load chunk plan returned unsupported decision status")
|
|
}
|
|
}
|
|
metadata, err := cloneMetadata(input.Metadata)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return result, err
|
|
}
|
|
produced, err := chunker.Plan(ctx, contracts.ChunkRequest{Operation: operation, Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(target.references), LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: metadata})
|
|
if err != nil {
|
|
return result, fmt.Errorf("chunk source with %q: %w", chunker.Key(), err)
|
|
}
|
|
plan, outcome, err := completeProducerOperation(ctx, input.pipeline.ChunkExecutionClass, operation, produced.Plan, produced.Outcome, produced.Fallback)
|
|
if err != nil {
|
|
return result, fmt.Errorf("accept chunk plan with %q: %w", chunker.Key(), err)
|
|
}
|
|
plan, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
if err != nil {
|
|
return result, fmt.Errorf("materialize final chunk plan: %w", err)
|
|
}
|
|
digest, err := source.DigestChunkPlan(plan)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
producerMetadata, _, err := moduleManifestMetadata(chunker)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
profile := ""
|
|
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
|
|
profile = input.pipeline.Chunk.LLMProfile
|
|
}
|
|
record := ChunkPlanRecord{SchemaVersion: ChunkPlanSchemaVersion, SourceDigest: doc.Digest, PlanDigest: digest, Plan: source.CloneChunkPlan(plan), CreatedAt: time.Now().UTC(),
|
|
Producer: ChunkPlanProducer{InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile, References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...), Metadata: producerMetadata},
|
|
}
|
|
if err := result.setCandidate(record, "generated"); err != nil {
|
|
return result, err
|
|
}
|
|
if mode == ChunkCacheRefresh {
|
|
result.action, result.summary.Action = "refreshed", "refreshed"
|
|
}
|
|
if mode == ChunkCacheBypass {
|
|
result.action, result.summary.Action = "bypassed", "bypassed"
|
|
}
|
|
result.plan, result.chunks, result.accepted, result.outcome = &plan, chunks, true, outcome
|
|
result.setValidation(nil, nil)
|
|
if outcome.Disposition == contracts.ProducerFallback {
|
|
result.summary.ValidationStatus = "incomplete"
|
|
}
|
|
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
|
|
result.summary.PublicationStatus = "not_published"
|
|
if outcome.ReuseEligible {
|
|
record.Diagnostics = contracts.CloneProducerDiagnostics(produced.Diagnostics)
|
|
if err := input.ChunkPlans.Save(record); err != nil {
|
|
return result, fmt.Errorf("save chunk plan: %w", err)
|
|
}
|
|
result.summary.PublicationStatus = "published"
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func chunkPlanLookupStatus(status ChunkPlanStatus) string {
|
|
switch status {
|
|
case ChunkPlanHit:
|
|
return "hit"
|
|
case ChunkPlanMissing:
|
|
return "missing"
|
|
case ChunkPlanInvalid:
|
|
return "invalid"
|
|
default:
|
|
return "skipped"
|
|
}
|
|
}
|
|
|
|
func chunkPlanLookupReason(status ChunkPlanStatus) string {
|
|
switch status {
|
|
case ChunkPlanHit:
|
|
return "stored chunk plan is valid"
|
|
case ChunkPlanMissing:
|
|
return "chunk plan not found"
|
|
case ChunkPlanInvalid:
|
|
return "stored chunk plan is invalid"
|
|
default:
|
|
return "chunk plan lookup skipped"
|
|
}
|
|
}
|
|
|
|
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) error {
|
|
cloned, err := cloneChunkPlanRecord(record)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
result.record = &cloned
|
|
result.action = action
|
|
result.summary.Action = action
|
|
result.summary.SourceDigest = record.SourceDigest
|
|
result.summary.CandidateDigest = record.PlanDigest
|
|
return nil
|
|
}
|
|
|
|
func (result *chunkPlanExecution) setValidation(rejection *contracts.RejectedOutput, err error) {
|
|
switch {
|
|
case err != nil:
|
|
result.summary.ValidationStatus = "error"
|
|
case rejection != nil:
|
|
result.summary.ValidationStatus = "rejected"
|
|
default:
|
|
result.summary.ValidationStatus = "approved"
|
|
}
|
|
}
|
|
|
|
func cloneChunkPlanRecord(record ChunkPlanRecord) (ChunkPlanRecord, error) {
|
|
record.Plan = source.CloneChunkPlan(record.Plan)
|
|
record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
|
|
metadata, err := cloneMetadata(record.Producer.Metadata)
|
|
if err != nil {
|
|
return ChunkPlanRecord{}, fmt.Errorf("clone chunk plan producer metadata: %w", err)
|
|
}
|
|
record.Producer.Metadata = metadata
|
|
record.Diagnostics = contracts.CloneProducerDiagnostics(record.Diagnostics)
|
|
return record, nil
|
|
}
|
|
|
|
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) error {
|
|
if output == nil {
|
|
return nil
|
|
}
|
|
summary := result.summary
|
|
output.ChunkPlan = &summary
|
|
if output.Manifest.ChunkPlan == nil {
|
|
return nil
|
|
}
|
|
manifest := output.Manifest.ChunkPlan
|
|
manifest.Action = result.action
|
|
if result.record == nil {
|
|
return nil
|
|
}
|
|
record := result.record
|
|
manifest.SourceDigest = record.SourceDigest
|
|
manifest.PlanDigest = record.PlanDigest
|
|
manifest.PlanSchemaVersion = record.SchemaVersion
|
|
manifest.ProducerInputModule = record.Producer.InputModule
|
|
manifest.ProducerModule = record.Producer.ChunkModule
|
|
manifest.ProducerLLMProfile = record.Producer.LLMProfile
|
|
manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
|
|
metadata, err := cloneMetadata(record.Producer.Metadata)
|
|
if err != nil {
|
|
return fmt.Errorf("clone chunk plan manifest producer metadata: %w", err)
|
|
}
|
|
manifest.ProducerMetadata = metadata
|
|
createdAt := record.CreatedAt
|
|
manifest.CreatedAt = &createdAt
|
|
return nil
|
|
}
|