package pipeline import ( "context" "fmt" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) type chunkPlanExecution struct { outcome contracts.ProducerOutcome accepted bool chunks []source.Chunk plan *source.ChunkPlan diagnostics []contracts.DiagnosticGroup rejection *contracts.RejectedOutput lookup ChunkPlanDecision record *ChunkPlanRecord action string summary artifacts.ChunkPlanSummary validation *artifacts.ValidationSummary } func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode { if mode == "" { return ChunkCacheBypass } parsed, _ := ParseChunkCacheMode(string(mode)) return parsed } // Chunk-plan lookup and publication happen serially before lane workers start. // The runner therefore does not add synchronization around the store. func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) { mode := effectiveChunkCacheMode(input.ChunkCacheMode) chunker := input.Prepared.chunker result := chunkPlanExecution{lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"}, summary: artifacts.ChunkPlanSummary{Mode: string(mode), SourceDigest: doc.Digest, RequestedModule: chunker.Key(), LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped", ValidationStatus: "not_run", PublicationStatus: "not_requested"}} target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution} assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators) var assessment contracts.CandidateAssessment operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: true, Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) { _, chunks, err := validateAndMaterializeChunkPlan(doc, plan) if err != nil { if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic { return contracts.CandidateAssessment{}, err } assessment = contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "chunk materialization", Decision: contracts.AssessmentFailed, Failure: err}}} return assessment, nil } assessment, err = assessChunks(ctx, chunks) return assessment, err }, } if mode == ChunkCacheAuto { record, decision, err := input.ChunkPlans.Load(doc.Digest) if err != nil { return result, fmt.Errorf("load chunk plan: %w", err) } result.lookup = decision result.summary.LookupStatus, result.summary.LookupReason = chunkPlanLookupStatus(decision.Status), chunkPlanLookupReason(decision.Status) if decision.Status == ChunkPlanHit { plan, chunks, err := validateAndMaterializeChunkPlan(doc, record.Plan) if err == nil { cachedAssessment, err := assessChunks(ctx, chunks) if err != nil { return result, err } if cachedAssessment.Accepted() { if err := result.setCandidate(record, "reused"); err != nil { return result, err } result.plan, result.chunks, result.accepted = &plan, chunks, true result.outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true} result.diagnostics, err = operationDiagnosticGroups(record.Diagnostics, cachedAssessment, result.outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil) result.setValidation(nil, err) return result, err } } result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: chunkPlanLookupReason(ChunkPlanInvalid)} result.summary.LookupStatus, result.summary.LookupReason = "invalid", chunkPlanLookupReason(ChunkPlanInvalid) } else if decision.Status != ChunkPlanMissing && decision.Status != ChunkPlanInvalid { return result, fmt.Errorf("load chunk plan returned unsupported decision status") } } metadata, err := cloneMetadata(input.Metadata) if err != nil { return result, err } if err := ctx.Err(); err != nil { return result, err } produced, err := chunker.Plan(ctx, contracts.ChunkRequest{Operation: operation, Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(target.references), LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: metadata}) if err != nil { return result, fmt.Errorf("chunk source with %q: %w", chunker.Key(), err) } plan, outcome, err := completeProducerOperation(ctx, input.pipeline.ChunkExecutionClass, operation, produced.Plan, produced.Outcome, produced.Fallback) if err != nil { return result, fmt.Errorf("accept chunk plan with %q: %w", chunker.Key(), err) } plan, chunks, err := validateAndMaterializeChunkPlan(doc, plan) if err != nil { return result, fmt.Errorf("materialize final chunk plan: %w", err) } digest, err := source.DigestChunkPlan(plan) if err != nil { return result, err } producerMetadata, _, err := moduleManifestMetadata(chunker) if err != nil { return result, err } profile := "" if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked { profile = input.pipeline.Chunk.LLMProfile } record := ChunkPlanRecord{SchemaVersion: ChunkPlanSchemaVersion, SourceDigest: doc.Digest, PlanDigest: digest, Plan: source.CloneChunkPlan(plan), CreatedAt: time.Now().UTC(), Producer: ChunkPlanProducer{InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile, References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...), Metadata: producerMetadata}, } if err := result.setCandidate(record, "generated"); err != nil { return result, err } if mode == ChunkCacheRefresh { result.action, result.summary.Action = "refreshed", "refreshed" } if mode == ChunkCacheBypass { result.action, result.summary.Action = "bypassed", "bypassed" } result.plan, result.chunks, result.accepted, result.outcome = &plan, chunks, true, outcome result.setValidation(nil, nil) if outcome.Disposition == contracts.ProducerFallback { result.summary.ValidationStatus = "incomplete" } result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil) if err != nil { return result, err } if mode == ChunkCacheAuto || mode == ChunkCacheRefresh { result.summary.PublicationStatus = "not_published" if outcome.ReuseEligible { record.Diagnostics = contracts.CloneProducerDiagnostics(produced.Diagnostics) if err := input.ChunkPlans.Save(record); err != nil { return result, fmt.Errorf("save chunk plan: %w", err) } result.summary.PublicationStatus = "published" } } return result, nil } func chunkPlanLookupStatus(status ChunkPlanStatus) string { switch status { case ChunkPlanHit: return "hit" case ChunkPlanMissing: return "missing" case ChunkPlanInvalid: return "invalid" default: return "skipped" } } func chunkPlanLookupReason(status ChunkPlanStatus) string { switch status { case ChunkPlanHit: return "stored chunk plan is valid" case ChunkPlanMissing: return "chunk plan not found" case ChunkPlanInvalid: return "stored chunk plan is invalid" default: return "chunk plan lookup skipped" } } func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) error { cloned, err := cloneChunkPlanRecord(record) if err != nil { return err } result.record = &cloned result.action = action result.summary.Action = action result.summary.SourceDigest = record.SourceDigest result.summary.CandidateDigest = record.PlanDigest return nil } func (result *chunkPlanExecution) setValidation(rejection *contracts.RejectedOutput, err error) { switch { case err != nil: result.summary.ValidationStatus = "error" case rejection != nil: result.summary.ValidationStatus = "rejected" default: result.summary.ValidationStatus = "approved" } } func cloneChunkPlanRecord(record ChunkPlanRecord) (ChunkPlanRecord, error) { record.Plan = source.CloneChunkPlan(record.Plan) record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...) metadata, err := cloneMetadata(record.Producer.Metadata) if err != nil { return ChunkPlanRecord{}, fmt.Errorf("clone chunk plan producer metadata: %w", err) } record.Producer.Metadata = metadata record.Diagnostics = contracts.CloneProducerDiagnostics(record.Diagnostics) return record, nil } func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) error { if output == nil { return nil } summary := result.summary output.ChunkPlan = &summary if output.Manifest.ChunkPlan == nil { return nil } manifest := output.Manifest.ChunkPlan manifest.Action = result.action if result.record == nil { return nil } record := result.record manifest.SourceDigest = record.SourceDigest manifest.PlanDigest = record.PlanDigest manifest.PlanSchemaVersion = record.SchemaVersion manifest.ProducerInputModule = record.Producer.InputModule manifest.ProducerModule = record.Producer.ChunkModule manifest.ProducerLLMProfile = record.Producer.LLMProfile manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...) metadata, err := cloneMetadata(record.Producer.Metadata) if err != nil { return fmt.Errorf("clone chunk plan manifest producer metadata: %w", err) } manifest.ProducerMetadata = metadata createdAt := record.CreatedAt manifest.CreatedAt = &createdAt return nil }