2026-07-18 00:20:56 +00:00
|
|
|
package pipeline
|
|
|
|
|
|
|
|
|
|
import (
|
2026-11-22 02:24:45 +00:00
|
|
|
"bytes"
|
2026-07-18 00:20:56 +00:00
|
|
|
"context"
|
2026-11-22 02:24:45 +00:00
|
|
|
"encoding/json"
|
2026-07-18 00:20:56 +00:00
|
|
|
"fmt"
|
2026-11-22 02:24:45 +00:00
|
|
|
"io"
|
2026-07-18 00:20:56 +00:00
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type chunkPlanExecution struct {
|
2026-11-21 23:48:51 +00:00
|
|
|
outcome contracts.ProducerOutcome
|
2026-08-27 15:29:10 +00:00
|
|
|
accepted bool
|
|
|
|
|
chunks []source.Chunk
|
|
|
|
|
plan *source.ChunkPlan
|
|
|
|
|
diagnostics []contracts.DiagnosticGroup
|
2026-07-18 00:20:56 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:24:45 +00:00
|
|
|
// Chunk planning remains serial before lane workers. Only a native chunk
|
|
|
|
|
// producer uses the accepted-artifact cache; deterministic plans run directly.
|
2026-11-22 03:19:43 +00:00
|
|
|
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (result chunkPlanExecution, err error) {
|
2026-07-18 00:20:56 +00:00
|
|
|
chunker := input.Prepared.chunker
|
2026-11-22 02:24:45 +00:00
|
|
|
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}
|
2026-11-22 03:19:43 +00:00
|
|
|
execution := operationExecution(input, origin, nil)
|
|
|
|
|
defer func() {
|
|
|
|
|
errorText := ""
|
|
|
|
|
if err != nil {
|
|
|
|
|
errorText = err.Error()
|
|
|
|
|
}
|
|
|
|
|
err = captureFailure(err, execution.RecordEvidence("outcome.json", map[string]any{"outcome": result.outcome, "plan": result.plan, "chunks": debugSourceChunkEnvelopes(result.chunks), "error": errorText}))
|
|
|
|
|
}()
|
|
|
|
|
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: execution}
|
|
|
|
|
if err := captureOperationInputs(execution, target, input.pipeline.Chunk, nil); err != nil {
|
|
|
|
|
return result, err
|
|
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
|
|
|
|
|
var assessment contracts.CandidateAssessment
|
2026-11-22 03:19:43 +00:00
|
|
|
candidateNumber := 0
|
|
|
|
|
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: execution, ReuseEligible: !input.reuseIneligible,
|
2026-11-21 23:48:51 +00:00
|
|
|
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
|
2026-11-22 03:19:43 +00:00
|
|
|
if execution.Capture != nil {
|
|
|
|
|
candidateNumber++
|
|
|
|
|
if err := execution.RecordEvidence(fmt.Sprintf("plans/candidate-%d.json", candidateNumber), plan); err != nil {
|
|
|
|
|
return contracts.CandidateAssessment{}, err
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
|
|
|
if err != nil {
|
|
|
|
|
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic {
|
|
|
|
|
return contracts.CandidateAssessment{}, err
|
|
|
|
|
}
|
|
|
|
|
assessment = contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "chunk materialization", Decision: contracts.AssessmentFailed, Failure: err}}}
|
|
|
|
|
return assessment, nil
|
|
|
|
|
}
|
|
|
|
|
assessment, err = assessChunks(ctx, chunks)
|
|
|
|
|
return assessment, err
|
2026-07-18 00:45:01 +00:00
|
|
|
},
|
|
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
var cacheDiagnostics []contracts.ProducerDiagnostic
|
|
|
|
|
if input.Cache != nil && input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
|
|
|
|
|
operation.Cache = &contracts.ProducerCache[source.ChunkPlan]{
|
|
|
|
|
Store: input.Cache,
|
|
|
|
|
Encode: func(plan source.ChunkPlan) ([]byte, error) {
|
|
|
|
|
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
|
2026-08-27 00:35:37 +00:00
|
|
|
if err != nil {
|
2026-11-22 02:24:45 +00:00
|
|
|
return nil, err
|
2026-07-17 23:16:16 -05:00
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
return json.Marshal(canonical)
|
|
|
|
|
},
|
|
|
|
|
Decode: func(content []byte) (source.ChunkPlan, error) {
|
|
|
|
|
var plan source.ChunkPlan
|
|
|
|
|
decoder := json.NewDecoder(bytes.NewReader(content))
|
|
|
|
|
decoder.DisallowUnknownFields()
|
|
|
|
|
if err := decoder.Decode(&plan); err != nil {
|
|
|
|
|
return source.ChunkPlan{}, err
|
2026-08-27 11:50:37 +00:00
|
|
|
}
|
2026-11-22 02:24:45 +00:00
|
|
|
var trailing any
|
|
|
|
|
if err := decoder.Decode(&trailing); err != io.EOF {
|
|
|
|
|
return source.ChunkPlan{}, fmt.Errorf("cached chunk plan must contain one JSON value")
|
|
|
|
|
}
|
|
|
|
|
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
|
|
|
return canonical, err
|
|
|
|
|
},
|
|
|
|
|
Observe: producerCacheObserver(input, origin, nil, &cacheDiagnostics),
|
2026-07-18 00:20:56 +00:00
|
|
|
}
|
|
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
metadata, err := cloneMetadata(input.Metadata)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return result, err
|
2026-07-18 00:45:01 +00:00
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return result, err
|
2026-08-27 01:27:25 +00:00
|
|
|
}
|
2026-11-21 23:48:51 +00:00
|
|
|
produced, err := chunker.Plan(ctx, contracts.ChunkRequest{Operation: operation, Source: doc, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(target.references), LLMProfile: input.pipeline.Chunk.LLMProfile, Metadata: metadata})
|
|
|
|
|
if err != nil {
|
|
|
|
|
return result, fmt.Errorf("chunk source with %q: %w", chunker.Key(), err)
|
|
|
|
|
}
|
|
|
|
|
plan, outcome, err := completeProducerOperation(ctx, input.pipeline.ChunkExecutionClass, operation, produced.Plan, produced.Outcome, produced.Fallback)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return result, fmt.Errorf("accept chunk plan with %q: %w", chunker.Key(), err)
|
|
|
|
|
}
|
|
|
|
|
plan, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return result, fmt.Errorf("materialize final chunk plan: %w", err)
|
|
|
|
|
}
|
|
|
|
|
result.plan, result.chunks, result.accepted, result.outcome = &plan, chunks, true, outcome
|
2026-11-22 02:24:45 +00:00
|
|
|
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, cacheDiagnostics, assessment, outcome, origin, nil)
|
|
|
|
|
return result, err
|
2026-07-18 00:45:01 +00:00
|
|
|
}
|