Unify chunk-plan caching and remove checkpoint execution paths

This commit is contained in:
2026-11-22 02:24:45 +00:00
parent 36906cb3d1
commit 3b1b4c9bcf
26 changed files with 717 additions and 4258 deletions

View File

@@ -1,504 +0,0 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type CheckpointFingerprint struct {
Name string `json:"name"`
Value string `json:"value"`
}
// CheckpointFingerprintProvider supplies stable, non-secret semantic identity
// for a prepared module or validator. Values must not contain source content,
// credentials, local paths, timestamps, or other invocation-specific data.
type CheckpointFingerprintProvider interface {
CheckpointFingerprints() []CheckpointFingerprint
}
type CheckpointRecorder interface {
SourceRunning(moduleKey string) error
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
SourceFailed(moduleKey string, err error) error
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
// StepCheckpointRecorder is implemented by checkpoint stores that isolate
// lane artifacts by their ordered pipeline step. The legacy recorder methods
// remain available for callers that do not have step context.
type StepCheckpointRecorder interface {
ExtractRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
ExtractSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error
ExtractFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
MergeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
MergeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
MergeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
MergeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
NormalizeRunningForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
NormalizeSucceededForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, output CheckpointArtifact) error
NormalizeRejectedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
NormalizeFailedForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
}
type CheckpointDecisionCategory string
const (
CheckpointDecisionExecuted CheckpointDecisionCategory = "executed"
CheckpointDecisionReused CheckpointDecisionCategory = "reused"
CheckpointDecisionForcedRecompute CheckpointDecisionCategory = "forced_recompute"
CheckpointDecisionDependencyInvalidated CheckpointDecisionCategory = "dependency_invalidated"
)
type CheckpointReasonCode string
const (
CheckpointReasonLoadingDisabled CheckpointReasonCode = "loading_disabled"
CheckpointReasonMissing CheckpointReasonCode = "checkpoint_missing"
CheckpointReasonPathInvalid CheckpointReasonCode = "checkpoint_path_invalid"
CheckpointReasonReadFailed CheckpointReasonCode = "checkpoint_read_failed"
CheckpointReasonDecodeFailed CheckpointReasonCode = "checkpoint_decode_failed"
CheckpointReasonWorkspaceSchemaIncompatible CheckpointReasonCode = "workspace_schema_incompatible"
CheckpointReasonIdentityMismatch CheckpointReasonCode = "identity_mismatch"
CheckpointReasonStageMismatch CheckpointReasonCode = "stage_mismatch"
CheckpointReasonStepMismatch CheckpointReasonCode = "step_mismatch"
CheckpointReasonLaneMismatch CheckpointReasonCode = "lane_mismatch"
CheckpointReasonModuleMismatch CheckpointReasonCode = "module_mismatch"
CheckpointReasonStatusNotReusable CheckpointReasonCode = "status_not_reusable"
CheckpointReasonDependencyMismatch CheckpointReasonCode = "dependency_mismatch"
CheckpointReasonArtifactPayloadInvalid CheckpointReasonCode = "artifact_payload_invalid"
CheckpointReasonArtifactDigestMismatch CheckpointReasonCode = "artifact_digest_mismatch"
CheckpointReasonArtifactCodecIncompatible CheckpointReasonCode = "artifact_codec_incompatible"
CheckpointReasonArtifactNotCanonical CheckpointReasonCode = "artifact_not_canonical"
CheckpointReasonReused CheckpointReasonCode = "checkpoint_reused"
CheckpointReasonAcceptedArtifactReused CheckpointReasonCode = "accepted_artifact_reused"
CheckpointReasonRecomputeStep CheckpointReasonCode = "recompute_step"
CheckpointReasonValidationIncompleteLineage CheckpointReasonCode = "validation_incomplete_lineage"
)
type CheckpointDecision struct {
Reused bool `json:"reused"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
// Reason is retained as a compatibility/debug field for existing callers.
// Detail and Reason are derived from the stable reason code.
Reason string `json:"reason,omitempty"`
}
const checkpointDecisionDetailLimit = 512
func NewCheckpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode) CheckpointDecision {
return checkpointDecision(category, reasonCode)
}
func checkpointDecision(category CheckpointDecisionCategory, reasonCode CheckpointReasonCode) CheckpointDecision {
detail := normalizeCheckpointDecisionDetail(checkpointDecisionDetail(reasonCode))
return CheckpointDecision{
Reused: category == CheckpointDecisionReused,
Category: category,
ReasonCode: reasonCode,
Detail: detail,
Reason: detail,
}
}
func checkpointDecisionDetail(reasonCode CheckpointReasonCode) string {
switch reasonCode {
case CheckpointReasonLoadingDisabled:
return "checkpoint loading is disabled"
case CheckpointReasonMissing:
return "checkpoint artifact is missing"
case CheckpointReasonPathInvalid:
return "checkpoint location is invalid"
case CheckpointReasonReadFailed:
return "checkpoint artifact could not be read"
case CheckpointReasonDecodeFailed:
return "checkpoint artifact could not be decoded"
case CheckpointReasonWorkspaceSchemaIncompatible:
return "checkpoint workspace schema is incompatible"
case CheckpointReasonIdentityMismatch:
return "checkpoint identity does not match the current invocation"
case CheckpointReasonStageMismatch:
return "checkpoint stage does not match"
case CheckpointReasonStepMismatch:
return "checkpoint step does not match"
case CheckpointReasonLaneMismatch:
return "checkpoint lane does not match"
case CheckpointReasonModuleMismatch:
return "checkpoint module does not match"
case CheckpointReasonStatusNotReusable:
return "checkpoint status is not reusable"
case CheckpointReasonDependencyMismatch:
return "checkpoint dependencies do not match"
case CheckpointReasonArtifactPayloadInvalid:
return "checkpoint artifact payload is invalid"
case CheckpointReasonArtifactDigestMismatch:
return "checkpoint artifact digest does not match"
case CheckpointReasonArtifactCodecIncompatible:
return "checkpoint artifact is incompatible with the registered codec"
case CheckpointReasonArtifactNotCanonical:
return "checkpoint artifact is not canonical"
case CheckpointReasonReused:
return "checkpoint is reusable"
case CheckpointReasonAcceptedArtifactReused:
return "accepted normalized artifact is reusable"
case CheckpointReasonRecomputeStep:
return "selected step requires execution"
case CheckpointReasonValidationIncompleteLineage:
return "checkpoint reuse is disabled by validation-incomplete input lineage"
default:
return "checkpoint decision"
}
}
func normalizeCheckpointDecisionDetail(detail string) string {
detail = strings.TrimSpace(strings.ToValidUTF8(detail, "?"))
var b strings.Builder
for _, r := range detail {
if r < 0x20 || r == 0x7f {
r = ' '
}
if b.Len()+len(string(r)) > checkpointDecisionDetailLimit {
break
}
b.WriteRune(r)
}
return strings.Join(strings.Fields(b.String()), " ")
}
type CheckpointEvent struct {
Stage string `json:"stage"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Action CheckpointDecisionCategory `json:"action"`
Category CheckpointDecisionCategory `json:"category,omitempty"`
ReasonCode CheckpointReasonCode `json:"reason_code,omitempty"`
Detail string `json:"detail,omitempty"`
Reason string `json:"reason,omitempty"`
}
type CheckpointExecutionPolicy struct {
ForcedLanes map[string]struct{}
RequireReusableLanes map[string]struct{}
}
func CheckpointLaneKey(stepID, laneID string) string {
return strings.TrimSpace(stepID) + "\x00" + strings.TrimSpace(laneID)
}
func (policy CheckpointExecutionPolicy) forced(stepID, laneID string) bool {
_, ok := policy.ForcedLanes[CheckpointLaneKey(stepID, laneID)]
return ok
}
func (policy CheckpointExecutionPolicy) requiresReusable(stepID, laneID string) bool {
_, ok := policy.RequireReusableLanes[CheckpointLaneKey(stepID, laneID)]
return ok
}
func forceCheckpointDecision(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) CheckpointDecision {
if policy.forced(stepID, laneID) {
return checkpointDecision(CheckpointDecisionForcedRecompute, CheckpointReasonRecomputeStep)
}
return decision
}
func requireReusableCheckpoint(policy CheckpointExecutionPolicy, stepID, laneID string, decision CheckpointDecision) error {
if policy.requiresReusable(stepID, laneID) && !policy.forced(stepID, laneID) && !decision.Reused {
return fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q (%s)", strings.TrimSpace(stepID), strings.TrimSpace(laneID), decision.ReasonCode)
}
return nil
}
// resolveCheckpointDecision applies runner policy and canonical payload
// validation at the single point where a stage's observable decision is made.
type checkpointResolution struct {
decision CheckpointDecision
values []any
artifacts []CheckpointArtifact
}
func resolveCheckpointDecision(output *RunOutput, loader CheckpointLoader, policy CheckpointExecutionPolicy, stage ModuleStage, stepID, laneID, moduleKey string, decision CheckpointDecision, codec artifactCodecEntry, artifacts []CheckpointArtifact) (checkpointResolution, error) {
decision = forceCheckpointDecision(policy, stepID, laneID, decision)
resolution := checkpointResolution{decision: decision}
if decision.Reused {
resolution.values = make([]any, 0, len(artifacts))
resolution.artifacts = make([]CheckpointArtifact, 0, len(artifacts))
for _, artifact := range artifacts {
value, hydrated, err := decodeCanonicalCheckpointArtifact(codec, artifact)
if err != nil {
decision = checkpointDecision(CheckpointDecisionExecuted, checkpointArtifactReasonCode(err))
resolution.values = nil
resolution.artifacts = nil
break
}
resolution.values = append(resolution.values, value)
resolution.artifacts = append(resolution.artifacts, hydrated)
}
}
resolution.decision = decision
if output != nil {
recordCheckpointEvent(output, loader, string(stage), stepID, laneID, moduleKey, decision)
}
if err := requireReusableCheckpoint(policy, stepID, laneID, decision); err != nil {
return resolution, err
}
return resolution, nil
}
type SourceCheckpoint struct {
Document *source.SourceDocument
}
// CheckpointArtifact is the durable, domain-neutral value stored at a lane
// checkpoint boundary.
type CheckpointArtifact struct {
LaneID string
ModuleKey string
SourceID string
ChunkID string
ChunkIndex int
ChunkRef source.SourceRef
Artifact contracts.SerializedArtifact
SchemaDigest string
Diagnostics []CheckpointDiagnostic
}
// CheckpointDiagnostic stores a producer-local diagnostic alongside a
// reusable artifact. The runner supplies the current run's origin when it
// promotes this value into a diagnostic group.
type CheckpointDiagnostic struct {
Diagnostic contracts.ProducerDiagnostic `json:"diagnostic"`
ValidatorKey string `json:"validator_key,omitempty"`
}
type ExtractCheckpoint struct {
Outputs []CheckpointArtifact
Rejected []contracts.RejectedOutput
}
type MergeCheckpoint struct {
Output CheckpointArtifact
}
type NormalizeCheckpoint struct {
Output CheckpointArtifact
}
type CheckpointLoader interface {
Enabled() bool
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
AcceptedNormalize(stepID, laneID, moduleKey string) (NormalizeCheckpoint, CheckpointDecision)
}
// StepCheckpointLoader is the step-aware counterpart used by the persistent
// checkpoint implementation. Loaders without this optional interface remain
// usable by framework callers and test doubles through the legacy methods.
type StepCheckpointLoader interface {
ExtractForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
MergeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
NormalizeForStep(stepID, laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
}
type noopCheckpointRecorder struct{}
type noopCheckpointLoader struct{}
func NoopCheckpointRecorder() CheckpointRecorder { return noopCheckpointRecorder{} }
func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{} }
func (noopCheckpointRecorder) SourceRunning(string) error { return nil }
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []CheckpointArtifact, []contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact) error {
return nil
}
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
return nil
}
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, CheckpointArtifact) error {
return nil
}
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
return nil
}
func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFingerprint, error) error {
return nil
}
func (noopCheckpointLoader) Enabled() bool { return false }
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
return SourceCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return ExtractCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func (noopCheckpointLoader) AcceptedNormalize(string, string, string) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonLoadingDisabled)
}
func checkpointExtractRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.ExtractRunning(laneID, moduleKey, deps)
}
func checkpointExtractSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractSucceededForStep(stepID, laneID, moduleKey, deps, outputs, rejected)
}
return recorder.ExtractSucceeded(laneID, moduleKey, deps, outputs, rejected)
}
func checkpointExtractFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.ExtractFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.ExtractFailed(laneID, moduleKey, deps, err)
}
func checkpointMergeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.MergeRunning(laneID, moduleKey, deps)
}
func checkpointMergeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeSucceededForStep(stepID, laneID, moduleKey, deps, output)
}
return recorder.MergeSucceeded(laneID, moduleKey, deps, output)
}
func checkpointMergeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeRejectedForStep(stepID, laneID, moduleKey, deps, rejected)
}
return recorder.MergeRejected(laneID, moduleKey, deps, rejected)
}
func checkpointMergeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.MergeFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.MergeFailed(laneID, moduleKey, deps, err)
}
func checkpointNormalizeRunning(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeRunningForStep(stepID, laneID, moduleKey, deps)
}
return recorder.NormalizeRunning(laneID, moduleKey, deps)
}
func checkpointNormalizeSucceeded(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeSucceededForStep(stepID, laneID, moduleKey, deps, output)
}
return recorder.NormalizeSucceeded(laneID, moduleKey, deps, output)
}
func checkpointNormalizeRejected(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeRejectedForStep(stepID, laneID, moduleKey, deps, rejected)
}
return recorder.NormalizeRejected(laneID, moduleKey, deps, rejected)
}
func checkpointNormalizeFailed(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, err error) error {
if stepAware, ok := recorder.(StepCheckpointRecorder); ok {
return stepAware.NormalizeFailedForStep(stepID, laneID, moduleKey, deps, err)
}
return recorder.NormalizeFailed(laneID, moduleKey, deps, err)
}
func digestFingerprints(name string, digest string) []CheckpointFingerprint {
digest = strings.TrimSpace(digest)
if digest == "" {
return nil
}
return []CheckpointFingerprint{{Name: name, Value: digest}}
}
func joinedChunkDigest(chunks []source.Chunk) (string, error) {
if len(chunks) == 0 {
return "", nil
}
values := make([]string, 0, len(chunks))
for _, chunk := range chunks {
digest, err := source.DigestChunk(chunk)
if err != nil {
return "", fmt.Errorf("digest chunk %q: %w", chunk.ID, err)
}
values = append(values, chunk.ID+"="+digest)
}
sort.Strings(values)
sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint {
if len(values) == 0 {
return nil
}
byName := make(map[string]string, len(values))
for _, value := range values {
name := strings.TrimSpace(value.Name)
fingerprint := strings.TrimSpace(value.Value)
if name == "" || fingerprint == "" {
continue
}
byName[name] = fingerprint
}
if len(byName) == 0 {
return nil
}
names := make([]string, 0, len(byName))
for name := range byName {
names = append(names, name)
}
sort.Strings(names)
out := make([]CheckpointFingerprint, 0, len(names))
for _, name := range names {
out = append(out, CheckpointFingerprint{Name: name, Value: byName[name]})
}
return out
}
func checkpointContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}

View File

@@ -1,29 +0,0 @@
package pipeline
import (
"fmt"
"strings"
)
type ChunkCacheMode string
const (
ChunkCacheAuto ChunkCacheMode = "auto"
ChunkCacheBypass ChunkCacheMode = "bypass"
ChunkCacheRefresh ChunkCacheMode = "refresh"
)
func ParseChunkCacheMode(raw string) (ChunkCacheMode, error) {
mode := ChunkCacheMode(strings.TrimSpace(raw))
switch mode {
case ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh:
return mode, nil
default:
return "", fmt.Errorf("chunk cache mode %q is not supported", strings.TrimSpace(raw))
}
}
func (m ChunkCacheMode) Validate() error {
_, err := ParseChunkCacheMode(string(m))
return err
}

View File

@@ -1,20 +0,0 @@
package pipeline
import "testing"
func TestParseChunkCacheMode(t *testing.T) {
for _, value := range []ChunkCacheMode{ChunkCacheAuto, ChunkCacheBypass, ChunkCacheRefresh} {
got, err := ParseChunkCacheMode(string(value))
if err != nil || got != value {
t.Fatalf("ParseChunkCacheMode(%q) = %q, %v", value, got, err)
}
if err := value.Validate(); err != nil {
t.Fatalf("%q.Validate() error = %v", value, err)
}
}
for _, value := range []string{"", "enabled", "AUTO", "auto,refresh"} {
if _, err := ParseChunkCacheMode(value); err == nil {
t.Fatalf("ParseChunkCacheMode(%q) error = nil, want error", value)
}
}
}

View File

@@ -0,0 +1,425 @@
package pipeline
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
// This producer uses real native preparation/continuation and returns domain
// plans; only the provider is faked by the tests.
type cachedPlanChunker struct{ engine *promptkit.Engine }
func (cachedPlanChunker) Key() string { return "cached-plan" }
func (cachedPlanChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c cachedPlanChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
references := map[string][]string{}
for name, slot := range request.References.Slots {
for _, item := range slot.Items {
references[name] = append(references[name], string(item.Content))
}
}
referenceContent, err := json.Marshal(references)
if err != nil {
return contracts.ChunkPlanResult{}, err
}
initial := nativeProducerInitial()
initial.ProfileID, initial.SessionID = request.LLMProfile, request.SessionID
initial.Inputs["source"] = promptkit.Inline(string(request.SourceInput.Content) + "\n" + string(referenceContent))
plan, outcome, err := RunProducer(ctx, c.engine, request.Operation, initial,
func(_ context.Context, result *promptkit.RunResult) (*source.ChunkPlan, []string, error) {
var plan source.ChunkPlan
err := json.Unmarshal([]byte(result.RawOutput), &plan)
return &plan, nil, err
}, producerCorrection, func(context.Context, *source.ChunkPlan) (source.ChunkPlan, error) {
return source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: request.Source.Units[0].ID, EndUnitID: request.Source.Units[len(request.Source.Units)-1].ID}}}, nil
})
return contracts.ChunkPlanResult{Plan: plan, Outcome: outcome}, err
}
func planCacheRunInput(t *testing.T, engine *promptkit.Engine, validator chunkAssessmentValidator) RunInput {
t.Helper()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
if err := catalog.Chunkers.RegisterBuilderWithSpec(ModuleSpec{Key: "cached-plan", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Chunker, error) {
return cachedPlanChunker{request.Dependencies.Engine}, nil
}); err != nil {
t.Fatal(err)
}
if err := RegisterChunkValidator(catalog.Validators, ValidatorSpec{Key: "plan-policy", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.ChunkValidator, error) {
return namedPlanValidator{"plan-policy", validator}, nil
}); err != nil {
t.Fatal(err)
}
profile := typedResolutionProfile()
profile.Chunk = Binding("cached-plan")
profile.Chunk.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("plan-policy")}}
prepared := prepareRunnerProfile(t, catalog, profile, engine)
return RunInput{Prepared: prepared, pipeline: prepared.resolved}
}
func planCacheSource(doc *source.SourceDocument) contracts.LLMInputMaterial {
var texts []string
for _, unit := range doc.Units {
texts = append(texts, unit.Text)
}
return contracts.NewLLMInputMaterial("source", "text/plain", []byte(strings.Join(texts, "\n")), "", "")
}
func planResponse(t *testing.T, plan source.ChunkPlan) *promptkit.GenerateResponse {
t.Helper()
content, err := json.Marshal(plan)
if err != nil {
t.Fatal(err)
}
return &promptkit.GenerateResponse{Content: string(content)}
}
func TestChunkPlanCacheStoresFinalPlanAndReusesWithoutAssessment(t *testing.T) {
root, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
whole := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}}
final := typedTestPlan(doc)
calls, assessments := 0, 0
client := producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if calls == 1 {
return planResponse(t, whole), nil
}
if calls == 2 {
if len(request.Prompt.Messages) != 4 {
t.Fatalf("correction history=%#v", request.Prompt.Messages)
}
}
return planResponse(t, final), nil
})
engine := nativeProducerEngine(t, client)
input := planCacheRunInput(t, engine, func(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
if len(request.Chunks) == 1 {
return contracts.ValidationResult{ReasonCode: "partition", CorrectionGuidance: "Use separate scenes."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
})
input.Cache = store
var keys, decisions []string
input.ObserveCache = func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if origin.Stage != contracts.DiagnosticOriginStageChunk || origin.ModuleKey != "cached-plan" {
t.Fatalf("cache scope=%#v", origin)
}
if observation.Prepared != nil {
keys = append(keys, observation.MessageContentHash)
}
decisions = append(decisions, observation.Decision)
return nil
}
store.beforeWrite = func(_ string, _ []byte) {
if calls != 2 || assessments != 2 {
t.Fatal("plan stored before correction acceptance")
}
}
first, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "first-session")
if err != nil || !first.accepted || len(first.chunks) != 2 || calls != 2 || assessments != 2 || store.writes != 1 {
t.Fatalf("first plan=%#v err=%v calls=%d assessments=%d", first, err, calls, assessments)
}
bytes, err := os.ReadFile(filepath.Join(root, keys[0]+".json"))
var saved source.ChunkPlan
if err != nil || json.Unmarshal(bytes, &saved) != nil || !reflect.DeepEqual(saved, *first.plan) {
t.Fatalf("cache is not the final raw plan: %s %v", bytes, err)
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(bytes, &fields); err != nil {
t.Fatal(err)
}
for _, field := range []string{"producer", "checkpoint", "document", "provenance", "usage"} {
if _, ok := fields[field]; ok {
t.Fatalf("cache contains reporting field %q", field)
}
}
store.beforeWrite = nil
input.pipeline.Chunk.LLMProfile = "chosen"
input.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"}
second, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "changed-session")
if err != nil || !second.outcome.ReuseEligible || len(second.chunks) != 2 || calls != 2 || assessments != 2 || len(keys) != 2 || keys[0] != keys[1] {
t.Fatalf("settings-only reuse=%#v %v calls=%d assessments=%d keys=%v", second, err, calls, assessments, keys)
}
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
t.Fatalf("cache decisions=%v", decisions)
}
// Materialization is against the current source, not cached transcript bytes.
changed := *doc
changed.ID = "current-source"
changed.Units = append([]source.SourceUnit(nil), doc.Units...)
for i := range changed.Units {
changed.Units[i].Ref.SourceID = changed.ID
}
changed.Digest, _ = source.DigestDocument(&changed)
// Identity is part of ordinary plan usability if it changes the source digest.
// Supplying the matching digest lets this check focus on rematerialized refs.
saved.SourceDigest = changed.Digest
replacement, _ := json.Marshal(saved)
if err := store.inner.Store(context.Background(), contracts.CacheRefresh, keys[0], replacement); err != nil {
t.Fatal(err)
}
third, err := New().runChunkPlan(context.Background(), input, &changed, planCacheSource(doc), "session")
if err != nil || calls != 2 || assessments != 2 || third.chunks[0].Ref.SourceID != changed.ID || third.chunks[0].Units[0].Ref.SourceID != changed.ID {
t.Fatalf("rematerialization=%#v %v", third, err)
}
}
func TestChunkPlanCacheUsesMessageContentRatherThanSourceIdentity(t *testing.T) {
_, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
calls, assessments := 0, 0
client := producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
plan := typedTestPlan(doc)
if calls > 1 {
plan.Ranges = []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}
}
return planResponse(t, plan), nil
})
validator := chunkAssessmentValidator(func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
input := planCacheRunInput(t, nativeProducerEngine(t, client), validator)
input.Cache = store
var keys []string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
keys = append(keys, observation.MessageContentHash)
}
return nil
}
run := func(wantChunks int) {
t.Helper()
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || len(result.chunks) != wantChunks {
t.Fatalf("chunks=%d err=%v", len(result.chunks), err)
}
}
run(2)
input.pipeline.ChunkReferences.ReferenceSet = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte("new scene context")}}}}}
run(1)
// Replace only the prompt text, retaining source and reference bytes.
input.Prepared.chunker = cachedPlanChunker{nativeProducerEngine(t, client, promptkit.SettingsInput(configkit.Bytes("changed prompt", []byte(`prompts:
- id: initial
default_profile: base
inputs: [{name: source, required: true}]
messages:
- {role: system, content: 'A changed scene assignment.'}
- {role: user, content: 'Source: {{input "source"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
`))))}
run(1)
if calls != 3 || assessments != 3 || len(keys) != 3 || keys[0] == keys[1] || keys[1] == keys[2] {
t.Fatalf("content reuse: calls=%d assessments=%d keys=%v", calls, assessments, keys)
}
}
func TestChunkPlanCacheUnusableEntriesAreMissesNotAssistantCandidates(t *testing.T) {
for _, defect := range []string{"source digest", "source range", "malformed", "legacy envelope"} {
t.Run(defect, func(t *testing.T) {
root, store := filesystemProducerCache(t)
doc := typedTestDocument()
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if len(request.Prompt.Messages) != 2 {
t.Fatal("unusable cache replayed as an assistant candidate")
}
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
input.Cache = store
var key string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
key = observation.MessageContentHash
}
return nil
}
if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil {
t.Fatal(err)
}
bad := typedTestPlan(doc)
switch defect {
case "source digest":
bad.SourceDigest = "not-current"
case "source range":
bad.Ranges[0].EndUnitID = 999
}
content, _ := json.Marshal(bad)
if defect == "malformed" {
content = []byte("{")
}
if defect == "legacy envelope" {
content = []byte(`{"plan":{},"producer":{"key":"legacy"}}`)
}
// Simulate an existing corrupt entry without going through valid writes.
if err := os.WriteFile(filepath.Join(root, key+".json"), content, 0644); err != nil {
t.Fatal(err)
}
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || !result.accepted || calls != 2 || assessments != 2 || len(result.diagnostics) != 1 {
t.Fatalf("unusable plan=%#v err=%v calls=%d assessments=%d", result, err, calls, assessments)
}
})
}
}
func TestChunkPlanCacheModesAndTerminalExclusions(t *testing.T) {
for _, test := range []struct {
name string
cache contracts.CacheMode
mode contracts.ExecutionMode
reject, ineligible bool
reads, writes int
fallback bool
}{
{"auto", contracts.CacheAuto, contracts.ModeStrict, false, false, 1, 1, false},
{"bypass", contracts.CacheBypass, contracts.ModeStrict, false, false, 0, 0, false},
{"refresh", contracts.CacheRefresh, contracts.ModeStrict, false, false, 0, 1, false},
{"ineligible", contracts.CacheAuto, contracts.ModeStrict, false, true, 0, 0, false},
{"strict rejection", contracts.CacheAuto, contracts.ModeStrict, true, false, 1, 0, false},
{"permissive fallback", contracts.CacheAuto, contracts.ModePermissive, true, false, 1, 0, true},
} {
t.Run(test.name, func(t *testing.T) {
_, store := filesystemProducerCache(t)
doc := typedTestDocumentWithUnits(2)
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: !test.reject, ReasonCode: "policy", CorrectionGuidance: "Revise the scene partition."}, nil
})
input.Cache = store
input.reuseIneligible = test.ineligible
input.pipeline.ChunkControls = contracts.ProducerControls{Cache: test.cache, Mode: test.mode, MaxAdditionalTurns: 0}
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if (err != nil) != (test.reject && !test.fallback) || calls != 1 || assessments != 1 || store.reads != test.reads || store.writes != test.writes {
t.Fatalf("result=%#v err=%v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes)
}
if test.fallback && (result.outcome.Disposition != contracts.ProducerFallback || result.outcome.ReuseEligible || len(result.chunks) != 1 || len(result.chunks[0].Units) != 2) {
t.Fatalf("invalid whole-source fallback: %#v", result)
}
})
}
}
func TestChunkPlanFailedRefreshPreservesAcceptanceAndIgnoresLegacyRecords(t *testing.T) {
root, store := filesystemProducerCache(t)
legacyNames := []string{"checkpoints/run/manifest.json", "chunk-plans/source/plan.json"}
for _, name := range legacyNames {
filename := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("unread legacy bytes"), 0644); err != nil {
t.Fatal(err)
}
}
doc := typedTestDocumentWithUnits(2)
reject := false
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(doc)), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: !reject, ReasonCode: "partition", CorrectionGuidance: "Reconsider the partition."}, nil
})
input.Cache = store
input.pipeline.ChunkControls.MaxAdditionalTurns = 0
var key string
input.ObserveCache = func(_ context.Context, _ contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
key = observation.MessageContentHash
}
return nil
}
if _, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session"); err != nil {
t.Fatal(err)
}
original, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil {
t.Fatal(err)
}
reject = true
input.pipeline.ChunkControls.Cache = contracts.CacheRefresh
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
input.pipeline.ChunkControls.Mode = mode
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if (err != nil) != (mode == contracts.ModeStrict) {
t.Fatalf("refresh mode=%s result=%#v error=%v", mode, result, err)
}
if mode == contracts.ModePermissive && (result.outcome.ReuseEligible || result.outcome.Disposition != contracts.ProducerFallback) {
t.Fatalf("fallback=%#v", result.outcome)
}
current, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil || string(current) != string(original) {
t.Fatal("failed refresh changed old acceptance")
}
}
input.pipeline.ChunkControls.Cache = contracts.CacheAuto
result, err := New().runChunkPlan(context.Background(), input, doc, planCacheSource(doc), "session")
if err != nil || result.outcome.Disposition != contracts.ProducerAccepted || calls != 3 || assessments != 3 || store.reads != 2 || store.writes != 1 {
t.Fatalf("old acceptance not reused: %#v %v calls=%d assessments=%d cache=%d/%d", result, err, calls, assessments, store.reads, store.writes)
}
for _, name := range legacyNames {
content, err := os.ReadFile(filepath.Join(root, name))
if err != nil || string(content) != "unread legacy bytes" {
t.Fatalf("legacy record modified: %s %q %v", name, content, err)
}
}
}
func TestDeterministicChunkingIgnoresCacheAndLegacyRecords(t *testing.T) {
root, store := filesystemProducerCache(t)
legacy := map[string][]byte{"checkpoints/identity/manifest.json": []byte("old checkpoint"), "chunk-plans/source.json": []byte("old plan")}
for name, content := range legacy {
filename := filepath.Join(root, name)
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, content, 0644); err != nil {
t.Fatal(err)
}
}
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
input := RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, ObserveCache: func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error {
return fmt.Errorf("deterministic chunker prepared a synthetic request")
}}
for range 2 {
if _, err := New().Run(context.Background(), input); err != nil {
t.Fatal(err)
}
}
if store.reads != 0 || store.writes != 0 {
t.Fatalf("deterministic cache I/O=%d/%d", store.reads, store.writes)
}
for name, want := range legacy {
got, err := os.ReadFile(filepath.Join(root, name))
if err != nil || string(got) != string(want) {
t.Fatalf("legacy record modified: %s %q %v", name, got, err)
}
}
}

View File

@@ -1,49 +0,0 @@
package pipeline
import (
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
const ChunkPlanSchemaVersion = "notarius.chunk-plan.v3"
type ChunkPlanProducer struct {
InputModule string `json:"input_module"`
ChunkModule string `json:"chunk_module"`
LLMProfile string `json:"llm_profile,omitempty"`
References []artifacts.ReferenceProvenance `json:"references,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
type ChunkPlanRecord struct {
SchemaVersion string `json:"schema_version"`
SourceDigest string `json:"source_digest"`
PlanDigest string `json:"plan_digest"`
Plan source.ChunkPlan `json:"plan"`
Producer ChunkPlanProducer `json:"producer"`
Diagnostics []contracts.ProducerDiagnostic `json:"diagnostics,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
type ChunkPlanStore interface {
Load(sourceDigest string) (ChunkPlanRecord, ChunkPlanDecision, error)
Save(ChunkPlanRecord) error
}
type ChunkPlanStoreFactory func(root string) (ChunkPlanStore, error)
type ChunkPlanDecision struct {
Status ChunkPlanStatus `json:"status"`
Reason string `json:"reason,omitempty"`
}
type ChunkPlanStatus string
const (
ChunkPlanHit ChunkPlanStatus = "hit"
ChunkPlanMissing ChunkPlanStatus = "missing"
ChunkPlanInvalid ChunkPlanStatus = "invalid"
)

View File

@@ -1,41 +1,11 @@
package pipeline
import (
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdiagnostics "gitea.maximumdirect.net/eric/notarius/internal/framework/diagnostics"
)
func promoteCheckpointDiagnostics(diagnostics []CheckpointDiagnostic, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) {
groups := make([]contracts.DiagnosticGroup, 0, len(diagnostics))
for _, record := range diagnostics {
validatorOrigin := origin
validatorOrigin.ValidatorKey = record.ValidatorKey
promoted, err := promoteProducerDiagnostics([]contracts.ProducerDiagnostic{record.Diagnostic}, validatorOrigin, chunk)
if err != nil {
return nil, fmt.Errorf("checkpoint diagnostic: %w", err)
}
groups = append(groups, promoted...)
}
return groups, nil
}
func cloneCheckpointDiagnostics(diagnostics []CheckpointDiagnostic) []CheckpointDiagnostic {
if len(diagnostics) == 0 {
return nil
}
cloned := make([]CheckpointDiagnostic, len(diagnostics))
for index, diagnostic := range diagnostics {
cloned[index] = CheckpointDiagnostic{
Diagnostic: contracts.CloneProducerDiagnostics([]contracts.ProducerDiagnostic{diagnostic.Diagnostic})[0],
ValidatorKey: diagnostic.ValidatorKey,
}
}
return cloned
}
func promoteProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) {
if err := contracts.ValidateProducerDiagnostics(diagnostics); err != nil {
return nil, err

View File

@@ -26,7 +26,7 @@ type debugEvidenceContextSummary struct {
// buildOutputEvidenceContext projects the prepared output policy from accepted
// normalized artifacts. It is intentionally separate from lane execution so
// checkpointed normalized outputs use the same reconstruction path.
// cached normalized outputs use the same reconstruction path.
func buildOutputEvidenceContext(prepared *PreparedPipeline, doc *source.SourceDocument, outputs []contracts.SerializedOutput) (*contracts.SerializedArtifact, *debugEvidenceContextSummary, error) {
if prepared == nil || prepared.evidencePlan == nil {
return nil, nil, nil

View File

@@ -1,9 +1,6 @@
package pipeline
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strings"
@@ -301,60 +298,3 @@ func referenceProvenanceForItem(target ResolvedReferenceTarget, item contracts.R
ProducerModule: item.Producer.ModuleKey,
}
}
type generatedReferenceFingerprintIdentity struct {
ProducerPipeline string `json:"producer_pipeline_id"`
ProducerStep string `json:"producer_step_id"`
ProducerLane string `json:"producer_lane_id"`
ProducerModule string `json:"producer_module_key"`
ArtifactKind string `json:"artifact_kind"`
SchemaID string `json:"schema_id"`
SchemaName string `json:"schema_name"`
SchemaVersion string `json:"schema_version"`
SchemaDigest string `json:"schema_digest"`
MediaType string `json:"media_type"`
ContentDigest string `json:"content_digest"`
SizeBytes int64 `json:"size_bytes"`
}
// generatedReferenceDependencies returns the canonical semantic dependency
// identity that receiving operation checkpoints must include.
func generatedReferenceDependencies(set contracts.ReferenceSet) []CheckpointFingerprint {
var fingerprints []CheckpointFingerprint
keys := make([]string, 0, len(set.Slots))
for key := range set.Slots {
keys = append(keys, key)
}
sort.Strings(keys)
for _, slotName := range keys {
for index, item := range set.Slots[slotName].Items {
if item.Origin.Type != "generated" {
continue
}
identity := generatedReferenceFingerprintIdentity{
ProducerPipeline: item.Producer.PipelineID,
ProducerStep: item.Producer.StepID,
ProducerLane: item.Producer.LaneID,
ProducerModule: item.Producer.ModuleKey,
ArtifactKind: string(item.ArtifactKind),
SchemaID: item.ArtifactSchema.ID,
SchemaName: item.ArtifactSchema.Name,
SchemaVersion: item.ArtifactSchema.Version,
SchemaDigest: contracts.DigestArtifactSchema(item.ArtifactSchema),
MediaType: item.MediaType,
ContentDigest: item.Digest,
SizeBytes: item.SizeBytes,
}
encoded, err := json.Marshal(identity)
if err != nil {
continue
}
sum := sha256.Sum256(encoded)
fingerprints = append(fingerprints, CheckpointFingerprint{
Name: fmt.Sprintf("generated-reference:%s:%d", slotName, index),
Value: "sha256:" + hex.EncodeToString(sum[:]),
})
}
}
return normalizeCheckpointFingerprints(fingerprints)
}

View File

@@ -19,15 +19,14 @@ type PreparedPipeline struct {
Steps []PreparedPipelineStep
Output ModuleBinding
resolved ResolvedPipeline
dependencies ModuleDependencies
input contracts.InputAdapter
chunker contracts.Chunker
chunkValidators preparedValidatorChain
output contracts.OutputEncoder
artifactCodecs *ArtifactCodecRegistry
evidencePlan *preparedEvidencePlan
checkpointFingerprints []CheckpointFingerprint
resolved ResolvedPipeline
dependencies ModuleDependencies
input contracts.InputAdapter
chunker contracts.Chunker
chunkValidators preparedValidatorChain
output contracts.OutputEncoder
artifactCodecs *ArtifactCodecRegistry
evidencePlan *preparedEvidencePlan
}
type PreparedPipelineStep struct {
@@ -158,10 +157,6 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
if err != nil {
return nil, err
}
prepared.checkpointFingerprints, err = collectPreparedCheckpointFingerprints(prepared)
if err != nil {
return nil, err
}
return prepared, nil
}

View File

@@ -1,88 +0,0 @@
package pipeline
import (
"fmt"
"sort"
"strings"
)
type checkpointFingerprintComponent struct {
scope string
module any
}
// CheckpointFingerprints returns a defensive copy of the stable semantic
// identities contributed by the pipeline's prepared modules and validators.
func (p *PreparedPipeline) CheckpointFingerprints() []CheckpointFingerprint {
if p == nil {
return nil
}
return append([]CheckpointFingerprint(nil), p.checkpointFingerprints...)
}
func collectPreparedCheckpointFingerprints(prepared *PreparedPipeline) ([]CheckpointFingerprint, error) {
components := []checkpointFingerprintComponent{
{scope: "input:" + prepared.resolved.Input.Module, module: prepared.input},
{scope: "chunk:" + prepared.resolved.Chunk.Module, module: prepared.chunker},
}
components = appendValidatorFingerprintComponents(components, "chunk:"+prepared.resolved.Chunk.Module, prepared.chunkValidators)
for _, step := range prepared.Steps {
for _, lane := range step.lanes {
laneScope := func(stage ModuleStage, moduleKey string) string {
return string(stage) + ":" + lane.resolved.ID + ":" + moduleKey
}
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageExtract, lane.resolved.Extract.Module), module: lane.typed.extractor})
components = appendValidatorFingerprintComponents(components, laneScope(StageExtract, lane.resolved.Extract.Module), lane.extractValidators)
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageMerge, lane.resolved.Merge.Module), module: lane.typed.merger})
components = appendValidatorFingerprintComponents(components, laneScope(StageMerge, lane.resolved.Merge.Module), lane.mergeValidators)
components = append(components, checkpointFingerprintComponent{scope: laneScope(StageNormalize, lane.resolved.Normalize.Module), module: lane.typed.normalizer})
components = appendValidatorFingerprintComponents(components, laneScope(StageNormalize, lane.resolved.Normalize.Module), lane.normalizeValidators)
}
}
components = append(components, checkpointFingerprintComponent{scope: "output:" + prepared.resolved.Output.Module, module: prepared.output})
var values []CheckpointFingerprint
seen := make(map[string]struct{})
for _, item := range components {
provider, ok := item.module.(CheckpointFingerprintProvider)
if !ok {
continue
}
for _, fingerprint := range provider.CheckpointFingerprints() {
name := strings.TrimSpace(fingerprint.Name)
value := strings.TrimSpace(fingerprint.Value)
if name == "" || value == "" {
return nil, fmt.Errorf("pipeline %q checkpoint fingerprint for %s must have a non-empty name and value", prepared.resolved.ID, item.scope)
}
scopedName := item.scope + ":" + name
if _, exists := seen[scopedName]; exists {
return nil, fmt.Errorf("pipeline %q checkpoint fingerprint %q is duplicated", prepared.resolved.ID, scopedName)
}
seen[scopedName] = struct{}{}
values = append(values, CheckpointFingerprint{Name: scopedName, Value: value})
}
}
sort.Slice(values, func(i, j int) bool { return values[i].Name < values[j].Name })
return values, nil
}
func appendValidatorFingerprintComponents(components []checkpointFingerprintComponent, ownerScope string, chain preparedValidatorChain) []checkpointFingerprintComponent {
for index, validator := range chain.validators {
scope := fmt.Sprintf("%s:validator:%d:%s", ownerScope, index, validator.resolved.Binding.Module)
components = append(components, checkpointFingerprintComponent{scope: scope, module: preparedValidatorImplementation(validator)})
}
return components
}
func preparedValidatorImplementation(validator preparedValidator) any {
switch validator.resolved.Target {
case ValidatorTargetTyped:
return validator.typed
case ValidatorTargetChunk:
return validator.chunk
case ValidatorTargetSerialized:
return validator.serialized
default:
return nil
}
}

View File

@@ -44,24 +44,20 @@ type RunInput struct {
Cache contracts.ArtifactCache
// ObserveCache receives scoped transient evidence, including prepared details
// on a hit. Shared observers must synchronize concurrent calls.
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
ChunkCacheMode ChunkCacheMode
ChunkPlans ChunkPlanStore
Checkpoints CheckpointRecorder
Checkpoint CheckpointLoader
CheckpointPolicy CheckpointExecutionPolicy
Debug DebugRecorder
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
LLMProfiles []artifacts.LLMProfileManifest
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
Debug DebugRecorder
// Workers bounds ready extraction and lane-completion jobs together.
// Zero selects DefaultWorkers; configuration rejects explicit zero.
Workers int
@@ -75,15 +71,15 @@ type RunInput struct {
}
type RunOutput struct {
FallbackCount int `json:"fallback_count,omitempty"`
Manifest artifacts.RunManifest `json:"manifest"`
ChunkPlan *artifacts.ChunkPlanSummary `json:"chunk_plan,omitempty"`
NormalizeOutputs []contracts.SerializedOutput `json:"normalize_outputs,omitempty"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Diagnostics contracts.DiagnosticCollection `json:"diagnostics,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
FallbackCount int `json:"fallback_count,omitempty"`
Manifest artifacts.RunManifest `json:"manifest"`
NormalizeOutputs []contracts.SerializedOutput `json:"normalize_outputs,omitempty"`
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
Diagnostics contracts.DiagnosticCollection `json:"diagnostics,omitempty"`
OutputFiles []contracts.OutputFile `json:"-"`
ValidationSummaries []artifacts.ValidationSummary `json:"validation_summaries,omitempty"`
normalizeReuseEligibility map[generatedOutputKey]bool
diagnosticGroups []contracts.DiagnosticGroup
@@ -111,25 +107,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err != nil {
return failOutput(output), err
}
output.ChunkPlan = &artifacts.ChunkPlanSummary{
Mode: string(effectiveChunkCacheMode(input.ChunkCacheMode)), RequestedModule: input.pipeline.Chunk.Module,
LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped",
ValidationStatus: "not_run", PublicationStatus: "not_requested",
}
checkpoints := input.Checkpoints
if checkpoints == nil {
checkpoints = NoopCheckpointRecorder()
}
checkpointLoader := input.Checkpoint
if checkpointLoader == nil {
checkpointLoader = NoopCheckpointLoader()
}
debugRecorder := input.Debug
if debugRecorder == nil {
debugRecorder = NoopDebugRecorder()
}
checkpoints = synchronizedCheckpointRecorder(checkpoints)
checkpointLoader = synchronizedCheckpointLoader(checkpointLoader)
debugRecorder = synchronizedDebugRecorder(debugRecorder)
input.Debug = debugRecorder
inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil)
@@ -153,9 +134,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if err := attachModuleManifestMetadata(&output, "input", adapter); err != nil {
return failOutput(output), err
}
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
recordCheckpointEvent(&output, checkpointLoader, "source", "", "", adapter.Key(), sourceDecision)
doc := sourceCheckpoint.Document
var doc *source.SourceDocument
sourceStarted := time.Now().UTC()
if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
Stage: "source",
@@ -171,48 +150,32 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
}
if !sourceDecision.Reused {
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return failOutput(output), fmt.Errorf("clone input adapter metadata: %w", metadataErr)
}
if err := ctx.Err(); err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), err
}
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID,
Path: input.Path,
Raw: input.RawInput,
LLMProfile: input.pipeline.Input.LLMProfile,
Metadata: requestMetadata,
})
if ctxErr := ctx.Err(); ctxErr != nil {
_ = checkpoints.SourceFailed(adapter.Key(), ctxErr)
return failOutput(output), ctxErr
}
if err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
}
if err := source.ValidateDocument(doc); err != nil {
_ = checkpoints.SourceFailed(adapter.Key(), err)
return failOutput(output), fmt.Errorf("validate source document: %w", err)
}
if err := checkpoints.SourceSucceeded(adapter.Key(), doc); err != nil {
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
}
requestMetadata, err := cloneMetadata(input.Metadata)
if err != nil {
return failOutput(output), fmt.Errorf("clone input adapter metadata: %w", err)
}
if err := ctx.Err(); err != nil {
return failOutput(output), err
}
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
SourceID: input.SourceID, Path: input.Path, Raw: input.RawInput,
LLMProfile: input.pipeline.Input.LLMProfile, Metadata: requestMetadata,
})
if ctxErr := ctx.Err(); ctxErr != nil {
return failOutput(output), ctxErr
}
if err != nil {
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
}
if err := source.ValidateDocument(doc); err != nil {
return failOutput(output), fmt.Errorf("validate source document: %w", err)
}
if err := writeDebugTimed(debugRecorder, "source/output.json", debugTimedEnvelope{
Stage: "source",
ModuleKey: adapter.Key(),
StartedAt: sourceStarted,
Payload: map[string]any{
"reused": sourceDecision.Reused,
"decision": sourceDecision,
"document": debugSourceDocumentEnvelope(doc),
},
}); err != nil {
@@ -226,13 +189,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), err
}
chunkStarted := time.Now().UTC()
chunkMode := effectiveChunkCacheMode(input.ChunkCacheMode)
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
StartedAt: chunkStarted,
Payload: map[string]any{
"cache_mode": chunkMode,
"cache_mode": input.pipeline.ChunkControls.Cache,
"source": debugSourceDocumentEnvelope(doc),
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
@@ -242,31 +204,18 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
}
chunkResult, err := r.runChunkPlan(ctx, input, doc, sourceInput, sessionID)
if applyErr := applyChunkPlanExecution(&output, chunkResult); applyErr != nil {
return failOutput(output), applyErr
}
if err != nil {
return failOutput(output), err
}
if chunkResult.rejection != nil {
output.Rejected = append(output.Rejected, *chunkResult.rejection)
}
if chunkResult.validation != nil {
output.ValidationSummaries = append(output.ValidationSummaries, artifacts.CloneValidationSummary(*chunkResult.validation))
}
appendDiagnosticGroups(&output, chunkResult.diagnostics)
chunkDebugPayload := map[string]any{
"cache_mode": chunkMode,
"lookup": chunkResult.lookup,
"cache_mode": input.pipeline.ChunkControls.Cache,
"accepted": chunkResult.accepted,
"materialized_chunks": debugSourceChunkEnvelopes(chunkResult.chunks),
}
if chunkResult.plan != nil {
chunkDebugPayload["plan"] = debugChunkPlanEnvelope(*chunkResult.plan)
}
if chunkResult.rejection != nil {
chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkResult.rejection)
}
if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
Stage: string(StageChunk),
ModuleKey: chunker.Key(),
@@ -278,8 +227,12 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
var acceptedChunkMap *contracts.SerializedArtifact
if chunkResult.accepted {
if chunkResult.plan == nil || chunkResult.record == nil {
return failOutput(output), fmt.Errorf("accepted chunk plan is missing plan or producer record")
if chunkResult.plan == nil {
return failOutput(output), fmt.Errorf("accepted chunk plan is missing")
}
chunkProfile := ""
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
chunkProfile = input.pipeline.Chunk.LLMProfile
}
artifact, buildErr := chunkmap.Serialize(chunkmap.BuildRequest{
Source: doc,
@@ -287,9 +240,9 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
Chunks: chunkResult.chunks,
RequestedChunker: input.pipeline.Chunk.Module,
Producer: chunkmap.Producer{
InputModule: chunkResult.record.Producer.InputModule,
ChunkModule: chunkResult.record.Producer.ChunkModule,
LLMProfile: chunkResult.record.Producer.LLMProfile,
InputModule: adapter.Key(),
ChunkModule: chunker.Key(),
LLMProfile: chunkProfile,
},
})
if buildErr != nil {
@@ -303,7 +256,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
if chunkResult.outcome.Disposition == contracts.ProducerFallback {
output.FallbackCount++
}
if err := r.runPreparedSteps(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, chunkResult.chunks, &output); err != nil {
if err := r.runPreparedSteps(ctx, input, doc, sourceInput, sessionID, chunkResult.chunks, &output); err != nil {
return failOutput(output), err
}
}
@@ -427,7 +380,7 @@ func hasIncompleteValidation(summaries []artifacts.ValidationSummary) bool {
return false
}
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, output *RunOutput) error {
for _, step := range input.Prepared.Steps {
stepInput := input
stepInput.stepID = step.ID
@@ -438,7 +391,7 @@ func (r *Runner) runPreparedSteps(ctx context.Context, input RunInput, checkpoin
stepInput.references = stepReferences
stepInput.referenceReuseEligibility = buildStepReferenceReuseEligibility(step, output.normalizeReuseEligibility)
output.Manifest.References = append(output.Manifest.References, referenceProvenance...)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, checkpoints, loader, doc, sourceInput, sessionID, chunks)
laneOutput, laneErr := r.runLanes(ctx, stepInput, step, doc, sourceInput, sessionID, chunks)
if err := mergeLaneOutput(output, laneOutput); err != nil {
return err
}
@@ -481,13 +434,6 @@ func validateRunInput(input RunInput) error {
if input.Prepared == nil {
return fmt.Errorf("prepared pipeline must not be nil")
}
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
if err := mode.Validate(); err != nil {
return err
}
if mode != ChunkCacheBypass && input.ChunkPlans == nil {
return fmt.Errorf("chunk plan store is required for %q mode", mode)
}
if err := validateResolvedPipeline(input.Prepared.resolved); err != nil {
return err
}
@@ -557,7 +503,7 @@ func manifestFromPipeline(input RunInput) artifacts.RunManifest {
InputModule: pipeline.Input.Module,
Chunker: pipeline.Chunk.Module,
ChunkPlan: &artifacts.ChunkPlanManifest{
Mode: string(effectiveChunkCacheMode(input.ChunkCacheMode)),
Mode: string(pipeline.ChunkControls.Cache),
RequestedModule: pipeline.Chunk.Module,
},
OutputEncoder: pipeline.Output.Module,
@@ -617,34 +563,6 @@ func failOutput(output RunOutput) RunOutput {
return output
}
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, stepID string, laneID string, moduleKey string, decision CheckpointDecision) {
if output == nil || loader == nil || !loader.Enabled() {
return
}
action := checkpointDecisionCategory(decision)
output.CheckpointEvents = append(output.CheckpointEvents, CheckpointEvent{
Stage: stage,
StepID: stepID,
LaneID: laneID,
ModuleKey: moduleKey,
Action: action,
Category: checkpointDecisionCategory(decision),
ReasonCode: decision.ReasonCode,
Detail: decision.Detail,
Reason: decision.Reason,
})
}
func checkpointDecisionCategory(decision CheckpointDecision) CheckpointDecisionCategory {
if decision.Category != "" {
return decision.Category
}
if decision.Reused {
return CheckpointDecisionReused
}
return CheckpointDecisionExecuted
}
func populateOutputManifest(output *RunOutput) {
if output == nil {
return
@@ -654,13 +572,6 @@ func populateOutputManifest(output *RunOutput) {
if len(output.ValidationSummaries) > 0 {
output.Manifest.ValidationSummaries = cloneValidationSummaries(output.ValidationSummaries)
}
if len(output.CheckpointEvents) > 0 {
decisions := make([]artifacts.CheckpointDecisionManifest, 0, len(output.CheckpointEvents))
for _, event := range output.CheckpointEvents {
decisions = append(decisions, artifacts.CheckpointDecisionManifest{Stage: event.Stage, StepID: event.StepID, LaneID: event.LaneID, ModuleKey: event.ModuleKey, Category: string(event.Category), ReasonCode: string(event.ReasonCode), Detail: event.Detail})
}
output.Manifest.CheckpointDecisions = decisions
}
}
func normalizedOutputManifests(outputs []contracts.SerializedOutput) []artifacts.NormalizedOutputManifest {

View File

@@ -1,396 +0,0 @@
package pipeline
import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type acceptedCheckpointLoader struct {
CheckpointLoader
accepted map[string]NormalizeCheckpoint
acceptedDecision map[string]CheckpointDecision
extractDeps map[string][]CheckpointFingerprint
}
func newAcceptedCheckpointLoader() *acceptedCheckpointLoader {
return &acceptedCheckpointLoader{
CheckpointLoader: NoopCheckpointLoader(),
accepted: make(map[string]NormalizeCheckpoint),
acceptedDecision: make(map[string]CheckpointDecision),
extractDeps: make(map[string][]CheckpointFingerprint),
}
}
func (l *acceptedCheckpointLoader) Enabled() bool { return true }
func (l *acceptedCheckpointLoader) AcceptedNormalize(stepID, laneID, _ string) (NormalizeCheckpoint, CheckpointDecision) {
key := CheckpointLaneKey(stepID, laneID)
return l.accepted[key], l.acceptedDecision[key]
}
func (l *acceptedCheckpointLoader) Extract(laneID string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.extractDeps[laneID] = append([]CheckpointFingerprint(nil), dependencies...)
return ExtractCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (l *acceptedCheckpointLoader) Merge(_ string, _ string, _ []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
return MergeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func (l *acceptedCheckpointLoader) Normalize(_ string, _ string, _ []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
return NormalizeCheckpoint{}, NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
}
func TestRunnerHydratesRequiredNormalizedArtifact(t *testing.T) {
value := codecNotes{Items: []string{"canonical producer value"}}
input, _, _ := handoffFixture(t, value)
prepared := input.Prepared
producer := &prepared.Steps[0].lanes[0]
consumer := &prepared.Steps[1].lanes[0]
doc := prepared.input.(*typedTestInput).doc
stored, err := checkpointArtifact(producer.typed.codec, producer.resolved.ID, producer.resolved.Normalize.Module, doc.ID, value)
if err != nil {
t.Fatal(err)
}
operationCalls := 0
producer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
operationCalls++
return erasedTypedResult{Value: value}, nil
}
producer.typed.merge = func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
operationCalls++
return erasedTypedResult{Value: value}, nil
}
producer.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
operationCalls++
return erasedTypedResult{Value: value}, nil
}
validatorCalls := 0
validator := preparedValidator{typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
validatorCalls++
return contracts.ValidationResult{Approved: true}, nil
}}
producer.extractValidators.validators = []preparedValidator{validator}
producer.mergeValidators.validators = []preparedValidator{validator}
producer.normalizeValidators.validators = []preparedValidator{validator}
var received contracts.ReferenceSet
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
received = CloneReferenceSet(request.References)
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
loader := newAcceptedCheckpointLoader()
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
consumerKey := CheckpointLaneKey(consumer.resolved.StepID, consumer.resolved.ID)
stored.Diagnostics = []CheckpointDiagnostic{{Diagnostic: contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "stored-warning", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "normalize", Message: "stored normalize warning"}}}}}
loader.accepted[producerKey] = NormalizeCheckpoint{Output: stored}
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
policy := CheckpointExecutionPolicy{
RequireReusableLanes: map[string]struct{}{producerKey: {}},
ForcedLanes: map[string]struct{}{consumerKey: {}},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if operationCalls != 0 || validatorCalls != 0 {
t.Fatalf("hydrated producer calls = operations %d validators %d, want zero", operationCalls, validatorCalls)
}
item := received.Slots["producer-output"].Items[0]
if string(item.Content) != string(stored.Artifact.Content) || item.Producer.StepID != producer.resolved.StepID || item.Producer.LaneID != producer.resolved.ID {
t.Fatalf("consumer generated reference = %#v, want exact hydrated producer bytes and identity", item)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "stored-warning" {
t.Fatalf("hydrated diagnostics = %#v, want normalize checkpoint diagnostics only", output.Diagnostics)
}
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, producer.resolved.StepID, producer.resolved.ID, CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
for _, event := range output.CheckpointEvents {
if event.StepID == producer.resolved.StepID && event.LaneID == producer.resolved.ID && event.Stage != string(StageNormalize) {
t.Fatalf("hydrated producer synthesized checkpoint event: %#v", event)
}
}
freshInput, _, _ := handoffFixture(t, value)
freshPrepared := freshInput.Prepared
freshPrepared.Steps[0].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
freshPrepared.Steps[0].lanes[0].typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: value}, nil
}
freshPrepared.Steps[1].lanes[0].typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecScore{Value: 3}}, nil
}
freshLoader := &handoffDependencyLoader{CheckpointLoader: NoopCheckpointLoader()}
freshOutput, err := New().Run(context.Background(), RunInput{Prepared: freshPrepared, RawInput: []byte("input"), Checkpoint: freshLoader})
if err != nil {
t.Fatalf("fresh Run() error = %v", err)
}
hydratedDependencies := loader.extractDeps[consumer.resolved.ID]
if generatedFingerprintCount(hydratedDependencies) != 1 {
t.Fatalf("hydrated consumer dependencies = %#v, want generated producer fingerprint", hydratedDependencies)
}
var matchedFreshDependencies bool
for _, dependencies := range freshLoader.extract {
if reflect.DeepEqual(dependencies, hydratedDependencies) {
matchedFreshDependencies = true
break
}
}
if !matchedFreshDependencies {
t.Fatalf("consumer dependencies differ: fresh %#v hydrated %#v", freshLoader.extract, hydratedDependencies)
}
if !reflect.DeepEqual(freshOutput.Manifest.References, output.Manifest.References) {
t.Fatalf("generated provenance differs: fresh %#v hydrated %#v", freshOutput.Manifest.References, output.Manifest.References)
}
}
func TestRunnerRejectsInvalidRequiredNormalizedArtifactBeforeConsumer(t *testing.T) {
const contentSentinel = "sensitive-campaign-payload-74291"
tests := []struct {
name string
decision CheckpointDecision
mutate func(*CheckpointArtifact)
wantCode CheckpointReasonCode
}{
{"missing", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing), nil, CheckpointReasonMissing},
{"rejected status", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonStatusNotReusable), nil, CheckpointReasonStatusNotReusable},
{"corrupt payload", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, CheckpointReasonArtifactPayloadInvalid},
{"non canonical", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items": ["stored"]}`) }, CheckpointReasonArtifactNotCanonical},
{"wrong codec identity", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused), func(v *CheckpointArtifact) { v.Artifact.Kind = "test/score" }, CheckpointReasonArtifactCodecIncompatible},
{"wrong content digest", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactDigestMismatch), nil, CheckpointReasonArtifactDigestMismatch},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
input, _, _ := handoffFixture(t, codecNotes{Items: []string{contentSentinel}})
prepared := input.Prepared
producer := &prepared.Steps[0].lanes[0]
consumer := &prepared.Steps[1].lanes[0]
doc := prepared.input.(*typedTestInput).doc
stored, err := checkpointArtifact(producer.typed.codec, producer.resolved.ID, producer.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{contentSentinel}})
if err != nil {
t.Fatal(err)
}
if test.mutate != nil {
test.mutate(&stored)
}
consumerCalls := 0
consumer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
consumerCalls++
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
loader := newAcceptedCheckpointLoader()
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
loader.accepted[producerKey] = NormalizeCheckpoint{Output: stored}
loader.acceptedDecision[producerKey] = test.decision
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{producerKey: {}}}
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if runErr == nil || !strings.Contains(runErr.Error(), string(test.wantCode)) || consumerCalls != 0 {
t.Fatalf("Run() error = %v consumer calls = %d, want %q before consumer", runErr, consumerCalls, test.wantCode)
}
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, producer.resolved.StepID, producer.resolved.ID, CheckpointDecisionExecuted, test.wantCode)
encoded, err := json.Marshal(struct {
Manifest any
Events any
Error string
}{output.Manifest, output.CheckpointEvents, runErr.Error()})
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), contentSentinel) || strings.Contains(string(encoded), string(stored.Artifact.Content)) {
t.Fatalf("failed hydration diagnostics leaked artifact content: %s", encoded)
}
})
}
}
func TestRunnerRetainsEarlierHydratedProducerWhenLaterRequiredProducerFails(t *testing.T) {
prepared := preparedPipelineWithSharedProducerStep(t)
first := &prepared.Steps[0].lanes[0]
second := &prepared.Steps[0].lanes[1]
consumer := &prepared.Steps[1].lanes[0]
doc := prepared.input.(*typedTestInput).doc
stored, err := checkpointArtifact(first.typed.codec, first.resolved.ID, first.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{"retained producer"}})
if err != nil {
t.Fatal(err)
}
consumerCalls := 0
consumer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
consumerCalls++
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
loader := newAcceptedCheckpointLoader()
firstKey := CheckpointLaneKey(first.resolved.StepID, first.resolved.ID)
secondKey := CheckpointLaneKey(second.resolved.StepID, second.resolved.ID)
stored.Diagnostics = []CheckpointDiagnostic{{Diagnostic: contracts.ProducerDiagnostic{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "retained-warning", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "normalize", Message: "retained warning"}}}}}
loader.accepted[firstKey] = NormalizeCheckpoint{Output: stored}
loader.acceptedDecision[firstKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
loader.acceptedDecision[secondKey] = NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing)
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{firstKey: {}, secondKey: {}}}
output, runErr := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if runErr == nil || !strings.Contains(runErr.Error(), string(CheckpointReasonMissing)) {
t.Fatalf("Run() error = %v, want missing required producer", runErr)
}
if consumerCalls != 0 {
t.Fatalf("consumer calls = %d, want zero", consumerCalls)
}
if len(output.NormalizeOutputs) != 1 || output.NormalizeOutputs[0].LaneID != first.resolved.ID || string(output.NormalizeOutputs[0].Artifact.Content) != string(stored.Artifact.Content) {
t.Fatalf("retained normalize outputs = %#v, want first producer", output.NormalizeOutputs)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "retained-warning" {
t.Fatalf("retained diagnostics = %#v", output.Diagnostics)
}
type decisionExpectation struct {
step, lane string
category CheckpointDecisionCategory
reason CheckpointReasonCode
}
want := []decisionExpectation{
{first.resolved.StepID, first.resolved.ID, CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused},
{second.resolved.StepID, second.resolved.ID, CheckpointDecisionExecuted, CheckpointReasonMissing},
}
var got []decisionExpectation
for _, event := range output.CheckpointEvents {
if event.Stage == string(StageNormalize) {
got = append(got, decisionExpectation{event.StepID, event.LaneID, event.Category, event.ReasonCode})
}
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("normalize decisions = %#v, want %#v", got, want)
}
var manifestGot []decisionExpectation
for _, decision := range output.Manifest.CheckpointDecisions {
if decision.Stage == string(StageNormalize) {
manifestGot = append(manifestGot, decisionExpectation{
decision.StepID, decision.LaneID,
CheckpointDecisionCategory(decision.Category), CheckpointReasonCode(decision.ReasonCode),
})
}
}
if !reflect.DeepEqual(manifestGot, want) {
t.Fatalf("manifest normalize decisions = %#v, want %#v", manifestGot, want)
}
}
func preparedPipelineWithSharedProducerStep(t *testing.T) *PreparedPipeline {
t.Helper()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
base := typedResolutionProfile()
profile := base
profile.Artifacts = nil
profile.Steps = []PipelineStepProfile{
{ID: "producers", Artifacts: map[string]ArtifactLaneProfile{"first": base.Artifacts["notes"], "second": base.Artifacts["notes"]}},
{ID: "consumer", Artifacts: map[string]ArtifactLaneProfile{"score": base.Artifacts["score"]}},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v", err)
}
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v", err)
}
doc := typedTestDocumentWithUnits(1)
prepared.input.(*typedTestInput).doc = doc
prepared.chunker = &typedTestChunker{key: "typed/chunk", plan: typedTestPlan(doc)}
return prepared
}
func TestForcedRequiredLaneExecutesInsteadOfHydrating(t *testing.T) {
prepared := preparedOrderedPipeline(t, 1,
orderedLaneSpec{id: "unrelated", profile: "score"},
orderedLaneSpec{id: "producer", profile: "notes"},
orderedLaneSpec{id: "consumer", profile: "score"},
)
unrelated := &prepared.Steps[0].lanes[0]
producer := &prepared.Steps[1].lanes[0]
consumer := &prepared.Steps[2].lanes[0]
installGeneratedReferenceTarget(&consumer.resolved.ExtractReferences, StageExtract, consumer, "step-2", "producer")
doc := prepared.input.(*typedTestInput).doc
unrelatedArtifact, err := checkpointArtifact(unrelated.typed.codec, unrelated.resolved.ID, unrelated.resolved.Normalize.Module, doc.ID, codecScore{Value: 9})
if err != nil {
t.Fatal(err)
}
producerArtifact, err := checkpointArtifact(producer.typed.codec, producer.resolved.ID, producer.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{"stale"}})
if err != nil {
t.Fatal(err)
}
unrelatedCalls, producerCalls := 0, 0
unrelated.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
unrelatedCalls++
return erasedTypedResult{Value: codecScore{Value: 9}}, nil
}
producer.typed.extract = func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error) {
producerCalls++
return erasedTypedResult{Value: codecNotes{Items: []string{"fresh"}}}, nil
}
producer.typed.normalize = func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
return erasedTypedResult{Value: codecNotes{Items: []string{"fresh"}}}, nil
}
var consumerReferences contracts.ReferenceSet
consumer.typed.extract = func(_ context.Context, _ any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
consumerReferences = CloneReferenceSet(request.References)
return erasedTypedResult{Value: codecScore{Value: 1}}, nil
}
loader := newAcceptedCheckpointLoader()
unrelatedKey := CheckpointLaneKey(unrelated.resolved.StepID, unrelated.resolved.ID)
producerKey := CheckpointLaneKey(producer.resolved.StepID, producer.resolved.ID)
consumerKey := CheckpointLaneKey(consumer.resolved.StepID, consumer.resolved.ID)
loader.accepted[unrelatedKey] = NormalizeCheckpoint{Output: unrelatedArtifact}
loader.acceptedDecision[unrelatedKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
loader.accepted[producerKey] = NormalizeCheckpoint{Output: producerArtifact}
loader.acceptedDecision[producerKey] = NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
policy := CheckpointExecutionPolicy{
ForcedLanes: map[string]struct{}{producerKey: {}, consumerKey: {}},
RequireReusableLanes: map[string]struct{}{unrelatedKey: {}, producerKey: {}},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if producerCalls == 0 || unrelatedCalls != 0 {
t.Fatalf("calls producer=%d unrelated=%d, want forced producer execution and unrelated hydration", producerCalls, unrelatedCalls)
}
if got := string(consumerReferences.Slots["producer-output"].Items[0].Content); !strings.Contains(got, "fresh") || strings.Contains(got, "stale") {
t.Fatalf("forced producer reference = %q, want freshly executed output", got)
}
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, unrelated.resolved.StepID, unrelated.resolved.ID, CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
assertAcceptedNormalizeEvent(t, output.CheckpointEvents, producer.resolved.StepID, producer.resolved.ID, CheckpointDecisionForcedRecompute, CheckpointReasonRecomputeStep)
}
func installGeneratedReferenceTarget(target *ResolvedReferenceTarget, stage ModuleStage, consumer *preparedLaneExecutor, producerStep, producerLane string) {
*target = ResolvedReferenceTarget{
Stage: stage,
StepID: consumer.resolved.StepID,
LaneID: consumer.resolved.ID,
Module: consumer.resolved.Extract.Module,
Bindings: []ReferenceBinding{{
Stage: stage,
LaneID: consumer.resolved.ID,
SlotName: "producer-output",
Artifact: &ArtifactReference{Step: producerStep, Lane: producerLane},
}},
ReferenceSet: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"producer-output": {Slot: contracts.ReferenceSlot{Name: "producer-output", AcceptedArtifactKinds: []contracts.ArtifactKind{"test/notes"}, AcceptedMediaTypes: []string{"application/json"}}},
}},
}
}
func assertAcceptedNormalizeEvent(t *testing.T, events []CheckpointEvent, stepID, laneID string, category CheckpointDecisionCategory, code CheckpointReasonCode) {
t.Helper()
for _, event := range events {
if event.Stage == string(StageNormalize) && event.StepID == stepID && event.LaneID == laneID {
if event.Category != category || event.ReasonCode != code {
t.Fatalf("accepted normalize event = %#v, want %q/%q", event, category, code)
}
return
}
}
t.Fatalf("accepted normalize event missing from %#v", events)
}

View File

@@ -36,29 +36,33 @@ func artifactProducerCache(input RunInput, typed *preparedTypedLane, class contr
}
return value, nil
},
Observe: func(ctx context.Context, observation contracts.CacheObservation) error {
if chunk != nil {
chunkIndex := chunk.Index
observation.ChunkID, observation.ChunkIndex = chunk.ID, &chunkIndex
}
if observation.Err != nil {
message := strings.ToValidUTF8(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err), "<22>")
if len(message) > contracts.MaxDiagnosticMessageBytes {
message = message[:contracts.MaxDiagnosticMessageBytes]
for !utf8.ValidString(message) {
message = message[:len(message)-1]
}
}
*warnings = append(*warnings, contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: "cache_" + observation.Decision, OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: message}},
})
}
if input.ObserveCache != nil {
return input.ObserveCache(ctx, origin, observation)
}
return nil
},
Observe: producerCacheObserver(input, origin, chunk, warnings),
}
}
func producerCacheObserver(input RunInput, origin contracts.DiagnosticOrigin, chunk *source.Chunk, warnings *[]contracts.ProducerDiagnostic) func(context.Context, contracts.CacheObservation) error {
return func(ctx context.Context, observation contracts.CacheObservation) error {
if chunk != nil {
chunkIndex := chunk.Index
observation.ChunkID, observation.ChunkIndex = chunk.ID, &chunkIndex
}
if observation.Err != nil {
message := strings.ToValidUTF8(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err), "<22>")
if len(message) > contracts.MaxDiagnosticMessageBytes {
message = message[:contracts.MaxDiagnosticMessageBytes]
for !utf8.ValidString(message) {
message = message[:len(message)-1]
}
}
*warnings = append(*warnings, contracts.ProducerDiagnostic{
Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: "cache_" + observation.Decision, OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: message}},
})
}
if input.ObserveCache != nil {
return input.ObserveCache(ctx, origin, observation)
}
return nil
}
}

View File

@@ -1,11 +1,12 @@
package pipeline
import (
"bytes"
"context"
"encoding/json"
"fmt"
"time"
"io"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -16,32 +17,18 @@ type chunkPlanExecution struct {
chunks []source.Chunk
plan *source.ChunkPlan
diagnostics []contracts.DiagnosticGroup
rejection *contracts.RejectedOutput
lookup ChunkPlanDecision
record *ChunkPlanRecord
action string
summary artifacts.ChunkPlanSummary
validation *artifacts.ValidationSummary
}
func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode {
if mode == "" {
return ChunkCacheBypass
}
parsed, _ := ParseChunkCacheMode(string(mode))
return parsed
}
// Chunk-plan lookup and publication happen serially before lane workers start.
// The runner therefore does not add synchronization around the store.
// Chunk planning remains serial before lane workers. Only a native chunk
// producer uses the accepted-artifact cache; deterministic plans run directly.
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
mode := effectiveChunkCacheMode(input.ChunkCacheMode)
chunker := input.Prepared.chunker
result := chunkPlanExecution{lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"}, summary: artifacts.ChunkPlanSummary{Mode: string(mode), SourceDigest: doc.Digest, RequestedModule: chunker.Key(), LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped", ValidationStatus: "not_run", PublicationStatus: "not_requested"}}
result := chunkPlanExecution{}
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
var assessment contracts.CandidateAssessment
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: true,
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible,
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
@@ -55,35 +42,32 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
return assessment, err
},
}
if mode == ChunkCacheAuto {
record, decision, err := input.ChunkPlans.Load(doc.Digest)
if err != nil {
return result, fmt.Errorf("load chunk plan: %w", err)
}
result.lookup = decision
result.summary.LookupStatus, result.summary.LookupReason = chunkPlanLookupStatus(decision.Status), chunkPlanLookupReason(decision.Status)
if decision.Status == ChunkPlanHit {
plan, chunks, err := validateAndMaterializeChunkPlan(doc, record.Plan)
if err == nil {
cachedAssessment, err := assessChunks(ctx, chunks)
var cacheDiagnostics []contracts.ProducerDiagnostic
if input.Cache != nil && input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
operation.Cache = &contracts.ProducerCache[source.ChunkPlan]{
Store: input.Cache,
Encode: func(plan source.ChunkPlan) ([]byte, error) {
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
return result, err
return nil, err
}
if cachedAssessment.Accepted() {
if err := result.setCandidate(record, "reused"); err != nil {
return result, err
}
result.plan, result.chunks, result.accepted = &plan, chunks, true
result.outcome = contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}
result.diagnostics, err = operationDiagnosticGroups(record.Diagnostics, nil, cachedAssessment, result.outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
result.setValidation(nil, err)
return result, err
return json.Marshal(canonical)
},
Decode: func(content []byte) (source.ChunkPlan, error) {
var plan source.ChunkPlan
decoder := json.NewDecoder(bytes.NewReader(content))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&plan); err != nil {
return source.ChunkPlan{}, err
}
}
result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: chunkPlanLookupReason(ChunkPlanInvalid)}
result.summary.LookupStatus, result.summary.LookupReason = "invalid", chunkPlanLookupReason(ChunkPlanInvalid)
} else if decision.Status != ChunkPlanMissing && decision.Status != ChunkPlanInvalid {
return result, fmt.Errorf("load chunk plan returned unsupported decision status")
var trailing any
if err := decoder.Decode(&trailing); err != io.EOF {
return source.ChunkPlan{}, fmt.Errorf("cached chunk plan must contain one JSON value")
}
canonical, _, err := validateAndMaterializeChunkPlan(doc, plan)
return canonical, err
},
Observe: producerCacheObserver(input, origin, nil, &cacheDiagnostics),
}
}
metadata, err := cloneMetadata(input.Metadata)
@@ -105,142 +89,7 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
if err != nil {
return result, fmt.Errorf("materialize final chunk plan: %w", err)
}
digest, err := source.DigestChunkPlan(plan)
if err != nil {
return result, err
}
producerMetadata, _, err := moduleManifestMetadata(chunker)
if err != nil {
return result, err
}
profile := ""
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
profile = input.pipeline.Chunk.LLMProfile
}
record := ChunkPlanRecord{SchemaVersion: ChunkPlanSchemaVersion, SourceDigest: doc.Digest, PlanDigest: digest, Plan: source.CloneChunkPlan(plan), CreatedAt: time.Now().UTC(),
Producer: ChunkPlanProducer{InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile, References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...), Metadata: producerMetadata},
}
if err := result.setCandidate(record, "generated"); err != nil {
return result, err
}
if mode == ChunkCacheRefresh {
result.action, result.summary.Action = "refreshed", "refreshed"
}
if mode == ChunkCacheBypass {
result.action, result.summary.Action = "bypassed", "bypassed"
}
result.plan, result.chunks, result.accepted, result.outcome = &plan, chunks, true, outcome
result.setValidation(nil, nil)
if outcome.Disposition == contracts.ProducerFallback {
result.summary.ValidationStatus = "incomplete"
}
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, nil, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}, nil)
if err != nil {
return result, err
}
if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
result.summary.PublicationStatus = "not_published"
if outcome.ReuseEligible {
record.Diagnostics = contracts.CloneProducerDiagnostics(produced.Diagnostics)
if err := input.ChunkPlans.Save(record); err != nil {
return result, fmt.Errorf("save chunk plan: %w", err)
}
result.summary.PublicationStatus = "published"
}
}
return result, nil
}
func chunkPlanLookupStatus(status ChunkPlanStatus) string {
switch status {
case ChunkPlanHit:
return "hit"
case ChunkPlanMissing:
return "missing"
case ChunkPlanInvalid:
return "invalid"
default:
return "skipped"
}
}
func chunkPlanLookupReason(status ChunkPlanStatus) string {
switch status {
case ChunkPlanHit:
return "stored chunk plan is valid"
case ChunkPlanMissing:
return "chunk plan not found"
case ChunkPlanInvalid:
return "stored chunk plan is invalid"
default:
return "chunk plan lookup skipped"
}
}
func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) error {
cloned, err := cloneChunkPlanRecord(record)
if err != nil {
return err
}
result.record = &cloned
result.action = action
result.summary.Action = action
result.summary.SourceDigest = record.SourceDigest
result.summary.CandidateDigest = record.PlanDigest
return nil
}
func (result *chunkPlanExecution) setValidation(rejection *contracts.RejectedOutput, err error) {
switch {
case err != nil:
result.summary.ValidationStatus = "error"
case rejection != nil:
result.summary.ValidationStatus = "rejected"
default:
result.summary.ValidationStatus = "approved"
}
}
func cloneChunkPlanRecord(record ChunkPlanRecord) (ChunkPlanRecord, error) {
record.Plan = source.CloneChunkPlan(record.Plan)
record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return ChunkPlanRecord{}, fmt.Errorf("clone chunk plan producer metadata: %w", err)
}
record.Producer.Metadata = metadata
record.Diagnostics = contracts.CloneProducerDiagnostics(record.Diagnostics)
return record, nil
}
func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) error {
if output == nil {
return nil
}
summary := result.summary
output.ChunkPlan = &summary
if output.Manifest.ChunkPlan == nil {
return nil
}
manifest := output.Manifest.ChunkPlan
manifest.Action = result.action
if result.record == nil {
return nil
}
record := result.record
manifest.SourceDigest = record.SourceDigest
manifest.PlanDigest = record.PlanDigest
manifest.PlanSchemaVersion = record.SchemaVersion
manifest.ProducerInputModule = record.Producer.InputModule
manifest.ProducerModule = record.Producer.ChunkModule
manifest.ProducerLLMProfile = record.Producer.LLMProfile
manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
metadata, err := cloneMetadata(record.Producer.Metadata)
if err != nil {
return fmt.Errorf("clone chunk plan manifest producer metadata: %w", err)
}
manifest.ProducerMetadata = metadata
createdAt := record.CreatedAt
manifest.CreatedAt = &createdAt
return nil
result.diagnostics, err = operationDiagnosticGroups(produced.Diagnostics, cacheDiagnostics, assessment, outcome, origin, nil)
return result, err
}

View File

@@ -2,837 +2,91 @@ package pipeline
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type recordingChunkPlanStore struct {
record ChunkPlanRecord
decision ChunkPlanDecision
loadErr error
saveErr error
loads int
saves int
saved ChunkPlanRecord
}
func (s *recordingChunkPlanStore) Load(string) (ChunkPlanRecord, ChunkPlanDecision, error) {
s.loads++
return s.record, s.decision, s.loadErr
}
func (s *recordingChunkPlanStore) Save(record ChunkPlanRecord) error {
s.saves++
s.saved = record
return s.saveErr
}
type countingChunkValidator struct {
calls int
result contracts.ValidationResult
err error
}
type capturingChunkMapOutput struct {
requests []contracts.OutputRequest
}
type capturingChunkMapOutput struct{ requests []contracts.OutputRequest }
func (*capturingChunkMapOutput) Key() string { return "capture/chunk-map" }
func (output *capturingChunkMapOutput) Encode(_ context.Context, request contracts.OutputRequest) (contracts.OutputResult, error) {
output.requests = append(output.requests, request)
return contracts.OutputResult{}, nil
}
func (*countingChunkValidator) Name() string { return "test/counting-chunks" }
func (*countingChunkValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (v *countingChunkValidator) Validate(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
v.calls++
return v.result, v.err
}
type manifestChunker struct {
terminalChunker
metadata map[string]any
}
func (c manifestChunker) ManifestMetadata() map[string]any { return c.metadata }
type llmCountingChunker struct {
terminalChunker
llmCalls *int
}
func (c llmCountingChunker) Plan(ctx context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
(*c.llmCalls)++
return c.terminalChunker.Plan(ctx, request)
}
type retryingChunker struct {
key string
plan source.ChunkPlan
calls int
}
type correctionAwareChunker struct {
key string
plan source.ChunkPlan
calls int
correction *contracts.SemanticCorrection
}
func (c *correctionAwareChunker) Key() string { return c.key }
func (*correctionAwareChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c *correctionAwareChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
c.calls++
if request.Correction != nil {
correction, err := contracts.CloneSemanticCorrection(request.Correction)
if err != nil {
return contracts.ChunkPlanResult{}, err
}
c.correction = correction
}
response := "initial chunk response"
if c.calls > 1 {
response = "corrected chunk response"
}
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
if err != nil {
return contracts.ChunkPlanResult{}, err
}
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), ModelCandidate: candidate}, nil
}
func (c *retryingChunker) Key() string { return c.key }
func (*retryingChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (c *retryingChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
c.calls++
if c.calls == 1 {
return contracts.ChunkPlanResult{Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("discarded", "discarded diagnostic")}}, errors.New("retry generation")
}
return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("accepted", "accepted diagnostic")}}, nil
}
type dependencyLoader struct {
CheckpointLoader
extractDependencies []CheckpointFingerprint
}
func (l *dependencyLoader) Extract(_ string, _ string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.extractDependencies = append([]CheckpointFingerprint(nil), dependencies...)
return ExtractCheckpoint{}, CheckpointDecision{Reason: "not found"}
}
func TestRunnerChunkPlanModeMatrix(t *testing.T) {
tests := []struct {
name string
mode ChunkCacheMode
decision ChunkPlanDecision
wantLoads int
wantSaves int
wantCalls int
wantAction string
}{
{name: "auto hit", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanHit}, wantLoads: 1, wantAction: "reused"},
{name: "auto missing", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanMissing}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
{name: "auto invalid", mode: ChunkCacheAuto, decision: ChunkPlanDecision{Status: ChunkPlanInvalid}, wantLoads: 1, wantSaves: 1, wantCalls: 1, wantAction: "generated"},
{name: "bypass", mode: ChunkCacheBypass, wantCalls: 1, wantAction: "bypassed"},
{name: "empty bypass", wantCalls: 1, wantAction: "bypassed"},
{name: "refresh", mode: ChunkCacheRefresh, wantSaves: 1, wantCalls: 1, wantAction: "refreshed"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
llmCalls := 0
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: tc.decision}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if store.loads != tc.wantLoads || store.saves != tc.wantSaves || calls != tc.wantCalls {
t.Fatalf("calls = load %d save %d module %d, want %d %d %d", store.loads, store.saves, calls, tc.wantLoads, tc.wantSaves, tc.wantCalls)
}
if output.Manifest.ChunkPlan == nil || output.Manifest.ChunkPlan.Action != tc.wantAction || output.ChunkPlan == nil || output.ChunkPlan.Action != tc.wantAction {
t.Fatalf("chunk plan manifest = %#v summary = %#v, want action %q", output.Manifest.ChunkPlan, output.ChunkPlan, tc.wantAction)
}
})
}
}
func TestRunnerUsesStatusDerivedChunkPlanSummaryReasons(t *testing.T) {
for _, tc := range []struct {
name string
status ChunkPlanStatus
wantStatus string
wantReason string
}{
{name: "hit", status: ChunkPlanHit, wantStatus: "hit", wantReason: "stored chunk plan is valid"},
{name: "missing", status: ChunkPlanMissing, wantStatus: "missing", wantReason: "chunk plan not found"},
{name: "invalid", status: ChunkPlanInvalid, wantStatus: "invalid", wantReason: "stored chunk plan is invalid"},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
store := &recordingChunkPlanStore{
record: chunkPlanRecord(t, prepared, plan),
decision: ChunkPlanDecision{Status: tc.status, Reason: "SENTINEL_STORE_REASON"},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if output.ChunkPlan.LookupStatus != tc.wantStatus || output.ChunkPlan.LookupReason != tc.wantReason || strings.Contains(output.ChunkPlan.LookupReason, "SENTINEL_STORE_REASON") {
t.Fatalf("summary = %#v", output.ChunkPlan)
}
})
}
prepared, _ := preparedTerminalDebugPipeline(t)
output, err := New().Run(context.Background(), RunInput{
Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto,
ChunkPlans: &recordingChunkPlanStore{loadErr: errors.New("read failed")},
})
if err == nil || output.ChunkPlan == nil || output.ChunkPlan.LookupStatus != "skipped" || output.ChunkPlan.LookupReason != "chunk plan lookup skipped" {
t.Fatalf("operational lookup output=%#v error=%v", output.ChunkPlan, err)
}
}
func TestRunnerChunkPlanHitUsesStoredProducerProvenance(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Module = "chunk/requested"
prepared.resolved.Chunk.LLMProfile = "current-profile"
record := chunkPlanRecord(t, prepared, plan)
record.Producer.InputModule = "input/original"
record.Producer.ChunkModule = "chunk/original"
record.Producer.LLMProfile = "original-profile"
record.Producer.Metadata = map[string]any{"prompt": map[string]any{"id": "original"}}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
manifest := output.Manifest.ChunkPlan
if manifest.RequestedModule != "chunk/requested" || manifest.ProducerModule != "chunk/original" || manifest.ProducerLLMProfile != "original-profile" {
t.Fatalf("chunk plan manifest = %#v", manifest)
}
storedNested := record.Producer.Metadata["prompt"].(map[string]any)
storedNested["id"] = "mutated"
if got := manifest.ProducerMetadata["prompt"].(map[string]any)["id"]; got != "original" {
t.Fatalf("producer metadata changed through stored alias: %v", got)
}
}
func TestRunnerReusesChunkPlanDiagnosticsWithCurrentValidatorDiagnostics(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryConfiguration,
ReasonCode: "stored_chunk_diagnostic",
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "reference", Message: "Stored configuration signal."}},
}}
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{{
Disposition: contracts.DiagnosticDispositionAdvisory,
Category: contracts.DiagnosticCategoryDataQuality,
ReasonCode: "current_validator_diagnostic",
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "chunk", Message: "Current validator finding."}},
}}}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}})
if err != nil {
t.Fatal(err)
}
if validator.calls != 1 || len(output.Diagnostics.Groups) != 2 {
t.Fatalf("validator calls = %d diagnostics = %#v", validator.calls, output.Diagnostics)
}
if output.Diagnostics.Groups[0].ReasonCode != "stored_chunk_diagnostic" || output.Diagnostics.Groups[1].ReasonCode != "current_validator_diagnostic" || output.Diagnostics.Groups[1].Origin.ValidatorKey != validator.Name() {
t.Fatalf("diagnostic groups = %#v", output.Diagnostics.Groups)
}
}
func TestRunnerProvidesAcceptedChunkMapToOutput(t *testing.T) {
for _, test := range []struct {
name string
configure func(*PreparedPipeline, source.ChunkPlan) (RunInput, chunkmap.Producer)
}{
{
name: "bypassed plan",
configure: func(prepared *PreparedPipeline, _ source.ChunkPlan) (RunInput, chunkmap.Producer) {
return RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheBypass}, chunkmap.Producer{
InputModule: prepared.input.Key(), ChunkModule: prepared.chunker.Key(),
}
},
},
{
name: "cached plan",
configure: func(prepared *PreparedPipeline, plan source.ChunkPlan) (RunInput, chunkmap.Producer) {
prepared.resolved.Chunk.Module = "chunk/requested"
record := chunkPlanRecord(t, prepared, plan)
record.Producer.InputModule = "input/original"
record.Producer.ChunkModule = "chunk/original"
record.Producer.LLMProfile = "original-profile"
return RunInput{
Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto,
ChunkPlans: &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}},
}, chunkmap.Producer{InputModule: "input/original", ChunkModule: "chunk/original", LLMProfile: "original-profile"}
},
},
} {
t.Run(test.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
input, wantProducer := test.configure(prepared, plan)
if _, err := New().Run(context.Background(), input); err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(encoder.requests) != 1 || encoder.requests[0].ChunkMap == nil {
t.Fatalf("output requests = %#v, want one accepted chunk map", encoder.requests)
}
artifact := encoder.requests[0].ChunkMap
if artifact.Kind != chunkmap.ArtifactKind || artifact.Schema.ID != chunkmap.SchemaID || artifact.MediaType != chunkmap.MediaType {
t.Fatalf("chunk map artifact = %#v, want fixed serialized identity", artifact)
}
value, err := chunkmap.New().Decode(artifact.Content)
if err != nil {
t.Fatalf("decode output chunk map: %v", err)
}
if value.RequestedChunker != prepared.resolved.Chunk.Module || value.Producer != wantProducer {
t.Fatalf("chunk map = %#v, want current request and producer %#v", value, wantProducer)
}
})
}
}
func TestRunnerOmitsChunkMapForRejectedPlan(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
calls, assessments := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return planResponse(t, typedTestPlan(typedTestDocument())), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
assessments++
return contracts.ValidationResult{Approved: true}, nil
})
_, store := filesystemProducerCache(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject/chunk"), Target: ValidatorTargetChunk},
chunk: &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable chunk plan"}},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 1 || len(encoder.requests) != 1 || encoder.requests[0].ChunkMap != nil {
t.Fatalf("output = %#v requests = %#v, want rejected plan without chunk map", output.Rejected, encoder.requests)
}
}
func TestRunnerCorrectsRejectedGeneratedChunkPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassLLMBacked
prepared.resolved.Chunk.Retries = 1
chunker := &correctionAwareChunker{key: prepared.resolved.Chunk.Module, plan: plan}
prepared.chunker = chunker
validatorCalls := 0
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject-then-approve"), Target: ValidatorTargetChunk},
chunk: chunkValidationFunc{name: "reject-then-approve", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
validatorCalls++
if validatorCalls == 1 {
return contracts.ValidationResult{ReasonCode: "chunk_scope", Message: "scope needs correction", CorrectionGuidance: "correct the chunk scope"}, nil
}
return contracts.ValidationResult{Approved: true}, nil
}},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 0 || chunker.calls != 2 || validatorCalls != 2 {
t.Fatalf("output = %#v chunker calls = %d validator calls = %d", output.Rejected, chunker.calls, validatorCalls)
}
if got := string(chunker.correction.AssistantResponse); got != "initial chunk response" {
t.Fatalf("correction response = %q", got)
}
}
func TestRunnerDoesNotPublishValidationIncompleteChunkPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("unavailable"), Target: ValidatorTargetChunk},
chunk: &countingChunkValidator{err: errors.New("validator unavailable")},
}}
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanMissing}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if output.ChunkPlan == nil || output.ChunkPlan.ValidationStatus != "incomplete" || output.ChunkPlan.PublicationStatus != "not_published" || store.saves != 0 {
t.Fatalf("chunk summary = %#v saves = %d", output.ChunkPlan, store.saves)
}
}
func TestRunnerRegeneratesValidationIncompleteChunkPlanHit(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validatorCalls := 0
prepared.chunkValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("cache-then-generated"), Target: ValidatorTargetChunk},
chunk: chunkValidationFunc{name: "cache-then-generated", validate: func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
validatorCalls++
if validatorCalls == 1 {
return contracts.ValidationResult{}, errors.New("cached candidate could not be validated")
}
return contracts.ValidationResult{Approved: true}, nil
}},
}}
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "discarded stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != 1 || validatorCalls != 2 || store.saves != 1 {
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 1", calls, validatorCalls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "approved" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Diagnostics.Groups) != 0 {
t.Fatalf("diagnostics = %#v, want discarded cache diagnostics omitted", output.Diagnostics)
}
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
}
func TestRunnerKeepsStoredPlanWhenCacheAndGeneratedValidationAreIncomplete(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "discarded stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if calls != 1 || validator.calls != 2 || store.saves != 0 {
t.Fatalf("calls = producer %d validator %d saves %d, want 1 2 0", calls, validator.calls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.Action != "generated" || output.ChunkPlan.LookupStatus != "invalid" || output.ChunkPlan.ValidationStatus != "incomplete" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "validator_execution_incomplete" {
t.Fatalf("diagnostics = %#v, want only generated incomplete diagnostic", output.Diagnostics)
}
if !reflect.DeepEqual(store.record, record) {
t.Fatal("discarded incomplete candidates mutated the stored record")
}
}
func TestRunnerFailsOnValidationIncompleteChunkPlanHitUnderFailRun(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
validator := &countingChunkValidator{err: errors.New("validator unavailable")}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
store := &recordingChunkPlanStore{record: chunkPlanRecord(t, prepared, plan), decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err == nil || !strings.Contains(err.Error(), "validator unavailable") {
t.Fatalf("Run() error = %v, want cached validation failure", err)
}
if calls != 0 || validator.calls != 1 || store.saves != 0 {
t.Fatalf("calls = producer %d validator %d saves %d, want 0 1 0", calls, validator.calls, store.saves)
}
if output.ChunkPlan == nil || output.ChunkPlan.ValidationStatus != "error" {
t.Fatalf("chunk plan summary = %#v", output.ChunkPlan)
}
}
func TestRunnerRetainsChunkMapAfterLaneRejection(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
prepared.Steps[0].lanes[0].mergeValidators.validators = []preparedValidator{{
resolved: ResolvedValidator{Binding: Binding("reject/lane"), Target: ValidatorTargetTyped, ArtifactKind: "test/notes"},
typedValidate: func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error) {
return contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "not accepted", CorrectionGuidance: "return an acceptable candidate"}, nil
},
}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if len(output.Rejected) != 1 || len(encoder.requests) != 1 || encoder.requests[0].ChunkMap == nil {
t.Fatalf("output = %#v requests = %#v, want lane rejection with accepted chunk map", output.Rejected, encoder.requests)
}
}
func TestRunnerChunkMapRequestDoesNotAliasStoredPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
record := chunkPlanRecord(t, prepared, plan)
record.Plan.Annotations = source.ChunkAnnotations{"dnd/scenes": json.RawMessage(`{"title":"Opening"}`)}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store}); err != nil {
t.Fatalf("Run() error = %v", err)
}
store.record.Plan.Annotations["dnd/scenes"][0] = '['
value, err := chunkmap.New().Decode(encoder.requests[0].ChunkMap.Content)
if err != nil {
t.Fatalf("decode captured chunk map after store mutation: %v", err)
}
if string(value.PlanAnnotations["dnd/scenes"]) != `{"title":"Opening"}` {
t.Fatalf("captured chunk map aliases stored plan: %s", value.PlanAnnotations["dnd/scenes"])
}
}
func TestRunnerChunkPlanManifestRetainsCandidateOnRejection(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: false, ReasonCode: "policy", Message: "no", CorrectionGuidance: "return a policy-compliant chunk plan"}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}})
if err != nil {
t.Fatal(err)
}
if output.Manifest.ChunkPlan.Action != "" || output.Manifest.ChunkPlan.PlanDigest != "" || output.ChunkPlan.ValidationStatus != "rejected" || output.ChunkPlan.PublicationStatus != "not_published" {
t.Fatalf("manifest = %#v summary = %#v; rejected output must not retain a usable chunk candidate", output.Manifest.ChunkPlan, output.ChunkPlan)
}
}
func TestRunnerValidatesChunkPlanPolicyInputs(t *testing.T) {
for _, tc := range []struct {
name string
mode ChunkCacheMode
want string
}{
{name: "auto requires store", mode: ChunkCacheAuto, want: "store is required"},
{name: "refresh requires store", mode: ChunkCacheRefresh, want: "store is required"},
{name: "invalid mode", mode: "sometimes", want: "not supported"},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, _ := preparedTerminalDebugPipeline(t)
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, ChunkCacheMode: tc.mode})
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("Run() error = %v, want %q", err, tc.want)
}
})
}
}
func TestRunnerChunkPlanStoreErrorsAreFrameworkErrors(t *testing.T) {
for _, tc := range []struct {
name string
mode ChunkCacheMode
decision ChunkPlanDecision
loadErr error
saveErr error
wantError string
wantCalls int
}{
{name: "load", mode: ChunkCacheAuto, loadErr: errors.New("read failed"), wantError: "load chunk plan", wantCalls: 0},
{name: "save", mode: ChunkCacheRefresh, saveErr: errors.New("write failed"), wantError: "save chunk plan", wantCalls: 1},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
llmCalls := 0
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
store := &recordingChunkPlanStore{decision: tc.decision, loadErr: tc.loadErr, saveErr: tc.saveErr}
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: tc.mode, ChunkPlans: store})
if err == nil || !strings.Contains(err.Error(), tc.wantError) || calls != tc.wantCalls {
t.Fatalf("Run() error = %v, module calls = %d, want %q and %d", err, calls, tc.wantError, tc.wantCalls)
}
})
}
}
func TestRunnerRegeneratesStructurallyInvalidHit(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
invalid := chunkPlanRecord(t, prepared, plan)
invalid.Plan.Ranges[0].StartUnitID = 999
store := &recordingChunkPlanStore{record: invalid, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store}); err != nil {
t.Fatal(err)
}
if calls != 1 || store.loads != 1 || store.saves != 1 {
t.Fatalf("calls = module %d load %d save %d, want 1 1 1", calls, store.loads, store.saves)
}
}
func TestRunnerReusesAnnotationsFromDifferentChunkerAsOptionalData(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
plan.Annotations = source.ChunkAnnotations{"test/chunker": json.RawMessage(`{"label":"fixture"}`)}
plan.Ranges[0].Annotations = source.ChunkAnnotations{"test/chunker": json.RawMessage(`{"category":"sample"}`)}
record := chunkPlanRecord(t, prepared, plan)
record.Producer.ChunkModule = "chunk/producer"
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if output.Manifest.ChunkPlan.ProducerModule != "chunk/producer" || output.Manifest.ChunkPlan.Action != "reused" {
t.Fatalf("different-chunker annotation plan was not reused: %#v", output.Manifest.ChunkPlan)
}
}
func TestRunnerRetriesBeforePublishingAcceptedPlan(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Retries = 1
chunker := &retryingChunker{key: prepared.resolved.Chunk.Module, plan: plan}
prepared.chunker = chunker
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanMissing}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if chunker.calls != 2 || store.saves != 1 {
t.Fatalf("module calls = %d saves = %d, want 2 and 1", chunker.calls, store.saves)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].Samples[0].Scope != "accepted" || len(store.saved.Diagnostics) != 1 || store.saved.Diagnostics[0].Samples[0].Scope != "accepted" {
t.Fatalf("output diagnostics = %#v stored diagnostics = %#v", output.Diagnostics, store.saved.Diagnostics)
}
}
func TestRunnerAutoHitValidatesOnceWithoutRegenerationOrMutation(t *testing.T) {
tests := []struct {
name string
result contracts.ValidationResult
validatorErr error
wantError string
wantReject bool
}{
{name: "diagnostic", result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("current", "current diagnostic")}}},
{name: "rejection", result: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected hit", CorrectionGuidance: "return an acceptable chunk plan"}, wantReject: true},
{name: "error", validatorErr: errors.New("validator failed"), wantError: "validator failed"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
calls := 0
llmCalls := 0
prepared.chunker = llmCountingChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}, llmCalls: &llmCalls}
validator := &countingChunkValidator{result: tc.result, err: tc.validatorErr}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
record := chunkPlanRecord(t, prepared, plan)
record.Diagnostics = []contracts.ProducerDiagnostic{producerDiagnostic("stored", "stored diagnostic")}
store := &recordingChunkPlanStore{record: record, decision: ChunkPlanDecision{Status: ChunkPlanHit}}
debug := newCapturedDebugRecorder()
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store, Debug: debug})
if tc.wantError != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("Run() error = %v, want %q", err, tc.wantError)
}
} else if err != nil {
t.Fatalf("Run() error = %v", err)
}
wantProducerCalls := 0
if tc.wantReject {
wantProducerCalls = 1
}
if validator.calls != 1+wantProducerCalls || calls != wantProducerCalls || llmCalls != wantProducerCalls || store.loads != 1 || store.saves != 0 {
t.Fatalf("calls = validator %d module %d llm %d load %d save %d", validator.calls, calls, llmCalls, store.loads, store.saves)
}
if tc.wantReject {
assertAttemptEnvelopeSequence(t, debug, "chunk", 1)
} else {
assertAttemptEnvelopeSequence(t, debug, "chunk")
}
if tc.wantError == "" {
encoded := string(debug.json["chunk/output.json"])
if tc.wantReject {
if !strings.Contains(encoded, `"status":"invalid"`) || strings.Contains(encoded, `"plan":`) {
t.Fatalf("rejected cache debug = %s", encoded)
}
} else if !strings.Contains(encoded, `"status":"hit"`) || !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) {
t.Fatalf("chunk hit debug = %s", encoded)
}
}
if tc.wantReject && (len(output.Rejected) != 1 || output.Rejected[0].ReasonCode != "rejected") {
t.Fatalf("rejected = %#v", output.Rejected)
}
if tc.wantError == "" && !tc.wantReject && len(output.Diagnostics.Groups) != 1+len(tc.result.Diagnostics) {
t.Fatalf("diagnostics = %#v, want stored diagnostic once plus current diagnostics", output.Diagnostics)
}
})
}
}
func TestRunnerPublishesOnlyAcceptedGeneratedPlans(t *testing.T) {
for _, tc := range []struct {
name string
moduleErr error
validator contracts.ValidationResult
cancel bool
wantCalls int
wantReject bool
}{
{name: "module error", moduleErr: errors.New("generation failed"), wantCalls: 2},
{name: "validator rejection", validator: contracts.ValidationResult{Approved: false, ReasonCode: "rejected", Message: "rejected generated plan", CorrectionGuidance: "return an acceptable chunk plan"}, wantCalls: 1, wantReject: true},
{name: "cancellation", cancel: true},
} {
t.Run(tc.name, func(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.Retries = 1
calls := 0
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls, err: tc.moduleErr}
if tc.wantReject {
validator := &countingChunkValidator{result: tc.validator}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
}
ctx := context.Background()
if tc.cancel {
cancelled, cancel := context.WithCancel(ctx)
cancel()
ctx = cancelled
}
store := &recordingChunkPlanStore{decision: ChunkPlanDecision{Status: ChunkPlanInvalid}}
output, err := New().Run(ctx, RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheAuto, ChunkPlans: store})
if tc.wantReject {
if err != nil || len(output.Rejected) != 1 {
t.Fatalf("Run() error = %v rejected = %#v", err, output.Rejected)
}
} else if err == nil {
t.Fatal("Run() error = nil")
}
if calls != tc.wantCalls || store.saves != 0 {
t.Fatalf("module calls = %d, saves = %d; want %d, 0", calls, store.saves, tc.wantCalls)
}
})
}
}
func TestRunnerStoresProducerProvenanceAndDiagnostics(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.LLMProfile = "chunk-profile"
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassLLMBacked
prepared.resolved.ChunkReferences = ResolvedReferenceTarget{
Stage: StageChunk,
ReferenceSet: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"guide": {Items: []contracts.ReferenceItem{{SlotName: "guide", Digest: "sha256:guide", Origin: contracts.ReferenceOrigin{Type: "file", URI: "file:///guide.txt"}, Content: []byte("sensitive")}}},
}},
}
prepared.chunker = manifestChunker{terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("producer", "producer diagnostic")}}, metadata: map[string]any{"prompt_id": "chunk/prompt"}}
validator := &countingChunkValidator{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{producerDiagnostic("validator", "validator diagnostic")}}}
prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk}, chunk: validator}}
store := &recordingChunkPlanStore{}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store}); err != nil {
t.Fatal(err)
}
producer := store.saved.Producer
if producer.InputModule != prepared.input.Key() || producer.ChunkModule != prepared.chunker.Key() || producer.LLMProfile != "chunk-profile" || producer.Metadata["prompt_id"] != "chunk/prompt" {
t.Fatalf("producer = %#v", producer)
}
if len(producer.References) != 1 || producer.References[0].Digest != "sha256:guide" {
t.Fatalf("producer references = %#v", producer.References)
}
if len(store.saved.Diagnostics) != 1 || store.saved.Diagnostics[0].Samples[0].Scope != "producer" {
t.Fatalf("stored diagnostics = %#v, want only producer diagnostic", store.saved.Diagnostics)
}
}
func TestRunnerRejectsUncloneableModuleManifestMetadata(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
cyclic := make(map[string]any)
cyclic["self"] = cyclic
prepared.chunker = manifestChunker{
terminalChunker: terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan},
metadata: map[string]any{"cyclic": cyclic},
}
_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")})
if err == nil || !strings.Contains(err.Error(), `clone manifest metadata for module "chunker": metadata.cyclic.self contains a cycle`) {
t.Fatalf("Run() error = %v, want contextual module metadata clone failure", err)
}
}
func TestRunnerOmitsProducerProfileForDeterministicChunker(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.Chunk.LLMProfile = "configured-but-unused"
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassDeterministic
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
store := &recordingChunkPlanStore{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if store.saved.Producer.LLMProfile != "" || output.Manifest.ChunkPlan.ProducerLLMProfile != "" {
t.Fatalf("deterministic producer profile = stored %q manifest %q", store.saved.Producer.LLMProfile, output.Manifest.ChunkPlan.ProducerLLMProfile)
}
}
func TestRunnerDoesNotInventProducerProfileForLLMChunker(t *testing.T) {
prepared, plan := preparedTerminalDebugPipeline(t)
prepared.resolved.ChunkExecutionClass = contracts.ExecutionClassLLMBacked
prepared.resolved.Chunk.LLMProfile = ""
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
store := &recordingChunkPlanStore{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: store})
if err != nil {
t.Fatal(err)
}
if store.saved.Producer.LLMProfile != "" || output.Manifest.ChunkPlan.ProducerLLMProfile != "" {
t.Fatalf("LLM producer profile = stored %q manifest %q, want empty", store.saved.Producer.LLMProfile, output.Manifest.ChunkPlan.ProducerLLMProfile)
}
}
func TestRunnerRefreshChangesDownstreamChunkFingerprint(t *testing.T) {
doc := typedTestDocumentWithUnits(2)
prepared := preparedConcurrentPipeline(t, 1)
prepared.Steps[0].lanes = prepared.Steps[0].lanes[:1]
prepared.resolved.Steps[0].ArtifactLanes = prepared.resolved.Steps[0].ArtifactLanes[:1]
prepared.Steps[0].ArtifactLanes = prepared.Steps[0].ArtifactLanes[:1]
prepared.input = &typedTestInput{key: prepared.resolved.Input.Module, doc: doc}
run := func(plan source.ChunkPlan) []CheckpointFingerprint {
prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
loader := &dependencyLoader{CheckpointLoader: NoopCheckpointLoader()}
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, ChunkCacheMode: ChunkCacheRefresh, ChunkPlans: &recordingChunkPlanStore{}, Checkpoint: loader}); err != nil {
input.Prepared.output = encoder
input.RawInput = []byte("input")
input.Cache = store
for range 2 {
if _, err := New().Run(context.Background(), input); err != nil {
t.Fatal(err)
}
return loader.extractDependencies
}
separate := typedTestPlan(doc)
combined := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}}}
if first, second := run(separate), run(combined); reflect.DeepEqual(first, second) {
t.Fatalf("downstream fingerprints did not change: %#v", first)
if calls != 1 || assessments != 1 || len(encoder.requests) != 2 {
t.Fatalf("calls=%d assessments=%d output requests=%d", calls, assessments, len(encoder.requests))
}
for _, request := range encoder.requests {
artifact := request.ChunkMap
if artifact == nil || artifact.Kind != chunkmap.ArtifactKind || artifact.Schema.ID != chunkmap.SchemaID || artifact.MediaType != chunkmap.MediaType {
t.Fatalf("chunk map artifact=%#v", artifact)
}
value, err := chunkmap.New().Decode(artifact.Content)
if err != nil {
t.Fatal(err)
}
wantProducer := chunkmap.Producer{InputModule: input.Prepared.input.Key(), ChunkModule: "cached-plan"}
if value.RequestedChunker != "cached-plan" || value.Producer != wantProducer || len(value.Chunks) != 1 {
t.Fatalf("chunk map=%#v", value)
}
}
}
func chunkPlanRecord(t *testing.T, prepared *PreparedPipeline, plan source.ChunkPlan) ChunkPlanRecord {
t.Helper()
digest, err := source.DigestChunkPlan(plan)
if err != nil {
t.Fatal(err)
}
return ChunkPlanRecord{
SchemaVersion: ChunkPlanSchemaVersion,
SourceDigest: plan.SourceDigest,
PlanDigest: digest,
Plan: source.CloneChunkPlan(plan),
Producer: ChunkPlanProducer{InputModule: prepared.input.Key(), ChunkModule: prepared.chunker.Key()},
CreatedAt: time.Now().UTC(),
func TestRunnerDoesNotPublishRejectedChunkPlan(t *testing.T) {
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return planResponse(t, typedTestPlan(typedTestDocument())), nil
}))
input := planCacheRunInput(t, engine, func(context.Context, contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
return contracts.ValidationResult{ReasonCode: "partition", CorrectionGuidance: "Choose a valid partition."}, nil
})
input.Prepared.resolved.ChunkControls.MaxAdditionalTurns = 0
encoder := &capturingChunkMapOutput{}
input.Prepared.output = encoder
input.RawInput = []byte("input")
_, err := New().Run(context.Background(), input)
if err == nil || len(encoder.requests) != 0 {
t.Fatalf("rejected plan error=%v output requests=%d", err, len(encoder.requests))
}
}
func TestDeterministicChunkMapDoesNotInventModelProvenance(t *testing.T) {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
prepared.resolved.Chunk.LLMProfile = "configured-but-not-used"
encoder := &capturingChunkMapOutput{}
prepared.output = encoder
if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
t.Fatal(err)
}
if len(encoder.requests) != 1 || encoder.requests[0].ChunkMap == nil {
t.Fatal("missing chunk map")
}
value, err := chunkmap.New().Decode(encoder.requests[0].ChunkMap.Content)
if err != nil || value.Producer.LLMProfile != "" {
t.Fatalf("model provenance=%#v %v", value.Producer, err)
}
}

View File

@@ -19,40 +19,20 @@ type laneExtractState struct {
fallbackCount int
index int
prepared preparedLaneExecutor
deps []CheckpointFingerprint
decision CheckpointDecision
reuseEligible bool
values []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
results map[int]extractJobResult
remaining int
failed bool
terminal bool
output RunOutput
}
type finalizedExtractResults struct {
accepted []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
decision CheckpointDecision
reuseEligible bool
}
func loadExtract(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.ExtractForStep(stepID, laneID, moduleKey, deps)
}
return loader.Extract(laneID, moduleKey, deps)
}
func recordExtract(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
return checkpointExtractSucceeded(recorder, stepID, laneID, moduleKey, deps, outputs, rejected)
}
type extractJob struct {
lane *laneExtractState
chunk source.Chunk
@@ -62,9 +42,7 @@ type extractJobResult struct {
laneIndex int
chunkIndex int
value erasedExtractArtifact
serialized CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected *contracts.RejectedOutput
outcome contracts.ProducerOutcome
err error
}
@@ -82,17 +60,14 @@ type orderedRunError struct {
err error
}
func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedPipelineStep, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk) (RunOutput, error) {
output := RunOutput{Manifest: manifestFromPipeline(input)}
states, err := initializeLaneStates(input, step, checkpoints, loader, doc, chunks, &output)
states, err := initializeLaneStates(input, step, chunks, &output)
if err != nil {
if mergeErr := mergeTerminalLaneStates(&output, states); mergeErr != nil {
return output, errors.Join(err, mergeErr)
}
return output, err
}
completedOutputs, runErrors := r.runLaneEngine(parent, laneEngineConfig{
input: input, checkpoints: checkpoints, loader: loader, doc: doc,
input: input, doc: doc,
sourceInput: sourceInput, sessionID: sessionID, chunks: chunks, states: states,
})
if err := mergeCompletedLanes(&output, completedOutputs); err != nil {
@@ -104,46 +79,22 @@ func (r *Runner) runLanes(parent context.Context, input RunInput, step PreparedP
return output, nil
}
func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, output *RunOutput) ([]*laneExtractState, error) {
func initializeLaneStates(input RunInput, step PreparedPipelineStep, chunks []source.Chunk, output *RunOutput) ([]*laneExtractState, error) {
states := make([]*laneExtractState, len(step.lanes))
for i, prepared := range step.lanes {
if prepared.typed == nil {
return states, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
return nil, fmt.Errorf("typed lane %q executor is not prepared", prepared.resolved.ID)
}
if err := setTypedLaneManifestMetadata(output, prepared.resolved.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return states, err
return nil, err
}
if input.CheckpointPolicy.requiresReusable(input.stepID, prepared.resolved.ID) && !input.CheckpointPolicy.forced(input.stepID, prepared.resolved.ID) {
if input.reuseIneligible || !laneReferencesReuseEligible(input, prepared.resolved) {
return states, fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q: generated input lineage is validation-incomplete", input.stepID, prepared.resolved.ID)
}
state, err := hydrateRequiredLane(input, loader, doc, i, prepared)
states[i] = state
if err != nil {
return states, err
}
continue
}
state, err := prepareLaneExtract(input, loader, doc, chunks, i, prepared, output)
if err != nil {
return states, err
}
if !state.decision.Reused && state.reuseEligible {
if err := checkpointExtractRunning(checkpoints, input.stepID, prepared.resolved.ID, prepared.resolved.Extract.Module, state.deps); err != nil {
return states, fmt.Errorf("write extract checkpoint for lane %q: %w", prepared.resolved.ID, err)
}
} else if err := finalizeLaneExtract(checkpoints, input.stepID, state); err != nil {
return states, err
}
states[i] = state
states[i] = &laneExtractState{index: i, prepared: prepared, reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, prepared.resolved.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
}
return states, nil
}
type laneEngineConfig struct {
input RunInput
checkpoints CheckpointRecorder
loader CheckpointLoader
doc *source.SourceDocument
sourceInput contracts.LLMInputMaterial
sessionID string
@@ -200,13 +151,6 @@ func newLaneEngine(r *Runner, parent context.Context, config laneEngineConfig) *
func (e *laneEngine) run() ([]RunOutput, []orderedRunError) {
defer e.cancel()
for _, state := range e.states {
if state.terminal {
e.completedOutputs[state.index] = state.output
} else if state.decision.Reused {
e.pending = append(e.pending, state)
}
}
e.startWorkers()
e.collect()
close(e.jobs)
@@ -229,7 +173,7 @@ func (e *laneEngine) startWorkers() {
if err := e.ctx.Err(); err != nil {
completion.err = err
} else {
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.checkpoints, e.loader, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
completion.output, completion.err = e.runner.continueLane(e.ctx, e.input, e.doc, e.sourceInput, e.sessionID, e.chunks, job.lane)
}
e.results <- laneJobResult{completion: &completion}
}
@@ -244,12 +188,9 @@ func (e *laneEngine) nextJob() (laneJob, bool) {
if len(e.pending) > 0 {
return laneJob{lane: e.pending[0]}, true
}
for e.nextExtract < len(e.chunks)*len(e.states) {
if e.nextExtract < len(e.chunks)*len(e.states) {
state := e.states[e.nextExtract%len(e.states)]
if !state.terminal && !state.decision.Reused {
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
}
e.nextExtract++
return laneJob{lane: state, chunk: &e.chunks[e.nextExtract/len(e.states)]}, true
}
return laneJob{}, false
}
@@ -303,9 +244,6 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
if result.err != nil {
state.failed = true
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: result.chunkIndex, err: result.err})
if state.reuseEligible {
_ = checkpointExtractFailed(e.checkpoints, e.input.stepID, state.prepared.resolved.ID, state.prepared.resolved.Extract.Module, state.deps, result.err)
}
e.cancel()
} else {
state.results[result.chunkIndex] = result
@@ -313,12 +251,7 @@ func (e *laneEngine) handleExtractResult(result extractJobResult) {
if state.remaining != 0 || state.failed || e.ctx.Err() != nil {
return
}
if err := finalizeLaneExtract(e.checkpoints, e.input.stepID, state); err != nil {
state.failed = true
e.runErrors = append(e.runErrors, orderedRunError{stage: 0, lane: state.index, chunk: len(e.chunks), err: err})
e.cancel()
return
}
finalizeLaneExtract(state)
e.pending = append(e.pending, state)
}
@@ -330,52 +263,6 @@ func (e *laneEngine) handleCompletion(completion laneCompletion) {
}
}
func hydrateRequiredLane(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, index int, prepared preparedLaneExecutor) (*laneExtractState, error) {
lane, typed := prepared.resolved, prepared.typed
local := RunOutput{Manifest: manifestFromPipeline(input)}
checkpoint, decision := loader.AcceptedNormalize(input.stepID, lane.ID, lane.Normalize.Module)
if decision.Reused {
decision = checkpointDecision(CheckpointDecisionReused, CheckpointReasonAcceptedArtifactReused)
if checkpoint.Output.LaneID != lane.ID || checkpoint.Output.ModuleKey != lane.Normalize.Module || checkpoint.Output.SourceID != doc.ID {
decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactPayloadInvalid)
}
}
resolution, err := resolveCheckpointDecision(&local, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
decision = resolution.decision
state := &laneExtractState{index: index, prepared: prepared, decision: decision, reuseEligible: true, terminal: true, output: local}
if err != nil {
return state, err
}
hydrated := resolution.artifacts[0]
diagnostics, diagnosticErr := promoteCheckpointDiagnostics(hydrated.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
if diagnosticErr != nil {
return state, fmt.Errorf("promote reused accepted diagnostics: %w", diagnosticErr)
}
appendDiagnosticGroups(&local, diagnostics)
local.NormalizeOutputs = append(local.NormalizeOutputs, contracts.SerializedOutput{
StepID: input.stepID,
LaneID: lane.ID,
NormalizerKey: lane.Normalize.Module,
SourceID: doc.ID,
Artifact: contracts.CloneSerializedArtifact(hydrated.Artifact),
})
local.normalizeReuseEligibility = map[generatedOutputKey]bool{generatedOutputKeyFor(input.stepID, lane.ID): true}
state.output = local
return state, nil
}
func mergeTerminalLaneStates(output *RunOutput, states []*laneExtractState) error {
for _, state := range states {
if state == nil || !state.terminal {
continue
}
if err := mergeLaneOutput(output, state.output); err != nil {
return err
}
}
return nil
}
func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) error {
for i := range completedOutputs {
if err := mergeLaneOutput(output, completedOutputs[i]); err != nil {
@@ -385,49 +272,6 @@ func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) error
return nil
}
func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.SourceDocument, chunks []source.Chunk, index int, prepared preparedLaneExecutor, output *RunOutput) (*laneExtractState, error) {
lane, typed := prepared.resolved, prepared.typed
digest, err := joinedChunkDigest(chunks)
if err != nil {
return nil, fmt.Errorf("digest chunks for lane %q: %w", lane.ID, err)
}
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
if !state.reuseEligible {
state.decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
recordCheckpointEvent(output, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, state.decision)
return state, nil
}
cp, decision := loadExtract(loader, input.stepID, lane.ID, lane.Extract.Module, state.deps)
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageExtract, input.stepID, lane.ID, lane.Extract.Module, decision, typed.codec, cp.Outputs)
if err != nil {
return nil, err
}
decision = resolution.decision
state.decision = decision
if decision.Reused {
state.remaining = 0
for i, stored := range resolution.artifacts {
value := resolution.values[i]
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: stored.ChunkID, ChunkIndex: stored.ChunkIndex, ChunkRef: stored.ChunkRef, Value: value}
if stored.ChunkIndex >= 0 && stored.ChunkIndex < len(chunks) && artifact.ChunkRef == (source.SourceRef{}) {
artifact.ChunkRef = chunks[stored.ChunkIndex].Ref
}
state.values = append(state.values, artifact)
state.serialized = append(state.serialized, cloneCheckpointArtifact(stored))
chunk := source.Chunk{ID: stored.ChunkID, Index: stored.ChunkIndex}
diagnostics, diagnosticErr := promoteCheckpointDiagnostics(stored.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if diagnosticErr != nil {
return nil, fmt.Errorf("promote reused extract diagnostics: %w", diagnosticErr)
}
state.diagnostics = append(state.diagnostics, diagnostics...)
}
state.rejected = cloneRejectedOutputs(cp.Rejected)
}
return state, nil
}
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
state := job.lane
lane, typed := state.prepared.resolved, state.prepared.typed
@@ -469,24 +313,22 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
result.err = fmt.Errorf("accept extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
stored, err := checkpointArtifact(typed.codec, lane.ID, lane.Extract.Module, doc.ID, value)
stored, err := serializeArtifact(typed.codec, value, false)
if err != nil {
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = chunk.ID, chunk.Index, chunk.Ref
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if err != nil {
result.err = err
return result
}
result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value}
result.serialized, result.outcome = stored, outcome
result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value, artifact: stored}
result.outcome = outcome
return result
}
func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *laneExtractState) error {
lane := state.prepared.resolved
func finalizeLaneExtract(state *laneExtractState) {
indexes := make([]int, 0, len(state.results))
for index := range state.results {
indexes = append(indexes, index)
@@ -494,13 +336,7 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
sort.Ints(indexes)
for _, index := range indexes {
result := state.results[index]
if result.rejected != nil {
state.rejected = append(state.rejected, *result.rejected)
state.diagnostics = append(state.diagnostics, result.diagnostics...)
continue
}
state.values = append(state.values, result.value)
state.serialized = append(state.serialized, result.serialized)
state.diagnostics = append(state.diagnostics, result.diagnostics...)
if !result.outcome.ReuseEligible {
state.reuseEligible = false
@@ -509,43 +345,38 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
state.fallbackCount++
}
}
sort.SliceStable(state.values, func(i, j int) bool { return state.values[i].ChunkIndex < state.values[j].ChunkIndex })
sort.SliceStable(state.serialized, func(i, j int) bool { return state.serialized[i].ChunkIndex < state.serialized[j].ChunkIndex })
sort.SliceStable(state.rejected, func(i, j int) bool { return state.rejected[i].ChunkIndex < state.rejected[j].ChunkIndex })
if !state.decision.Reused && state.reuseEligible {
if err := recordExtract(checkpoints, stepID, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
}
}
return nil
}
func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
lane := state.prepared.resolved
local := RunOutput{Manifest: manifestFromPipeline(input), FallbackCount: state.fallbackCount}
results := finalizedExtractResults{
accepted: state.values,
serialized: state.serialized,
diagnostics: state.diagnostics,
rejected: state.rejected,
decision: state.decision,
reuseEligible: state.reuseEligible,
}
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "decision": results.decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "outputs": debugCheckpointArtifacts(results.serialized), "rejected": debugRejectedOutputEnvelopes(results.rejected)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if len(results.accepted) == 0 {
return local, nil
}
err := r.continueTypedLane(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, state.prepared, results, &local)
err := r.continueTypedLane(ctx, input, doc, sourceInput, sessionID, state.prepared, results, &local)
return local, err
}
func debugExtractArtifacts(values []erasedExtractArtifact) []any {
outputs := make([]any, 0, len(values))
for _, value := range values {
outputs = append(outputs, map[string]any{"chunk_id": value.ChunkID, "chunk_index": value.ChunkIndex, "chunk_ref": value.ChunkRef, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{LaneID: value.LaneID, SourceID: value.SourceID, Artifact: value.artifact})})
}
return outputs
}
func classifyLaneError(lane, sentinel int, err error) orderedRunError {
stage := 1
var laneErr *laneRunError
@@ -603,7 +434,6 @@ func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
dst.NormalizeOutputs = append(dst.NormalizeOutputs, cloneSerializedOutputs(src.NormalizeOutputs)...)
dst.Rejected = append(dst.Rejected, cloneRejectedOutputs(src.Rejected)...)
appendDiagnosticGroups(dst, src.diagnosticGroups)
dst.CheckpointEvents = append(dst.CheckpointEvents, src.CheckpointEvents...)
dst.ValidationSummaries = append(dst.ValidationSummaries, cloneValidationSummaries(src.ValidationSummaries)...)
if len(src.normalizeReuseEligibility) > 0 {
if dst.normalizeReuseEligibility == nil {

View File

@@ -1,11 +1,7 @@
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"path"
"time"
@@ -15,74 +11,6 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func loadMerge(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.MergeForStep(stepID, laneID, moduleKey, deps)
}
return loader.Merge(laneID, moduleKey, deps)
}
func loadNormalize(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
if stepAware, ok := loader.(StepCheckpointLoader); ok {
return stepAware.NormalizeForStep(stepID, laneID, moduleKey, deps)
}
return loader.Normalize(laneID, moduleKey, deps)
}
func recordMerge(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return checkpointMergeSucceeded(recorder, stepID, laneID, moduleKey, deps, output)
}
func recordNormalize(recorder CheckpointRecorder, stepID, laneID, moduleKey string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return checkpointNormalizeSucceeded(recorder, stepID, laneID, moduleKey, deps, output)
}
func cloneCheckpointArtifact(output CheckpointArtifact) CheckpointArtifact {
output.Artifact = contracts.CloneSerializedArtifact(output.Artifact)
output.Diagnostics = cloneCheckpointDiagnostics(output.Diagnostics)
return output
}
func hydrateCheckpointArtifact(codec artifactCodecEntry, output CheckpointArtifact, value any) (CheckpointArtifact, error) {
output.Artifact.Schema = contracts.CloneArtifactSchema(codec.spec.Schema)
if codec.metadata != nil {
metadata, err := codec.metadata(value)
if err != nil {
return CheckpointArtifact{}, fmt.Errorf("clone artifact metadata: %w", err)
}
output.Artifact.Metadata = metadata
} else {
output.Artifact.Metadata = nil
}
return output, nil
}
func artifactCheckpointDigests(outputs []CheckpointArtifact) []CheckpointFingerprint {
values := make([]CheckpointFingerprint, 0, len(outputs))
for i, output := range outputs {
sum := sha256.Sum256(output.Artifact.Content)
values = append(values, CheckpointFingerprint{Name: fmt.Sprintf("artifact[%d]", i), Value: "sha256:" + hex.EncodeToString(sum[:])})
}
return normalizeCheckpointFingerprints(values)
}
func debugCheckpointArtifact(output CheckpointArtifact) map[string]any {
artifact := output.Artifact
schema := contracts.CloneArtifactSchema(artifact.Schema)
digest := output.SchemaDigest
if digest == "" {
digest = contracts.DigestArtifactSchema(schema)
}
schema.JSONSchema = nil
content := debugContentEnvelope(artifact.Content, artifact.MediaType, artifact.Metadata, nil)
content.ContentDigest = debugContentDigest(artifact.Content)
return map[string]any{"lane_id": output.LaneID, "module_key": output.ModuleKey, "source_id": output.SourceID, "chunk_id": output.ChunkID, "chunk_index": output.ChunkIndex, "chunk_ref": output.ChunkRef, "artifact_kind": artifact.Kind, "schema": schema, "schema_digest": digest, "content": content}
}
func debugCheckpointArtifacts(outputs []CheckpointArtifact) []map[string]any {
if len(outputs) == 0 {
return nil
}
out := make([]map[string]any, 0, len(outputs))
for _, output := range outputs {
out = append(out, debugCheckpointArtifact(output))
}
return out
}
func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (contracts.SerializedArtifact, error) {
encode := codec.encode
if candidate {
@@ -103,82 +31,6 @@ func serializeArtifact(codec artifactCodecEntry, value any, candidate bool) (con
return contracts.SerializedArtifact{Kind: codec.spec.Kind, Schema: contracts.CloneArtifactSchema(schema), MediaType: codec.spec.MediaType, Content: append([]byte(nil), content...), Metadata: metadata}, nil
}
type checkpointArtifactValidationError struct {
code CheckpointReasonCode
err error
}
func (e *checkpointArtifactValidationError) Error() string { return e.err.Error() }
func (e *checkpointArtifactValidationError) Unwrap() error { return e.err }
func checkpointArtifactValidationFailure(code CheckpointReasonCode, format string, args ...any) error {
return &checkpointArtifactValidationError{code: code, err: fmt.Errorf(format, args...)}
}
func checkpointArtifactReasonCode(err error) CheckpointReasonCode {
var validationErr *checkpointArtifactValidationError
if errors.As(err, &validationErr) {
return validationErr.code
}
return CheckpointReasonArtifactPayloadInvalid
}
func decodeCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, error) {
expectedDigest := contracts.DigestArtifactSchema(codec.spec.Schema)
if artifact.Artifact.Kind != codec.spec.Kind {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact kind %q does not match codec %q", artifact.Artifact.Kind, codec.spec.Kind)
}
if artifact.Artifact.Schema.ID != codec.spec.Schema.ID || artifact.Artifact.Schema.Name != codec.spec.Schema.Name || artifact.Artifact.Schema.Version != codec.spec.Schema.Version {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact schema %q version %q does not match codec schema %q version %q", artifact.Artifact.Schema.ID, artifact.Artifact.Schema.Version, codec.spec.Schema.ID, codec.spec.Schema.Version)
}
if artifact.SchemaDigest != expectedDigest {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact schema digest %q does not match codec schema digest %q", artifact.SchemaDigest, expectedDigest)
}
if artifact.Artifact.MediaType != codec.spec.MediaType {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactCodecIncompatible, "artifact media type %q does not match codec media type %q", artifact.Artifact.MediaType, codec.spec.MediaType)
}
value, err := codec.decode(append([]byte(nil), artifact.Artifact.Content...))
if err != nil {
return nil, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "decode artifact payload: %w", err)
}
return value, nil
}
func decodeCanonicalCheckpointArtifact(codec artifactCodecEntry, artifact CheckpointArtifact) (any, CheckpointArtifact, error) {
value, err := decodeCheckpointArtifact(codec, artifact)
if err != nil {
return nil, CheckpointArtifact{}, err
}
canonical, err := serializeArtifact(codec, value, false)
if err != nil {
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "encode canonical artifact: %w", err)
}
if canonical.Kind != artifact.Artifact.Kind || canonical.Schema.ID != artifact.Artifact.Schema.ID || canonical.Schema.Name != artifact.Artifact.Schema.Name || canonical.Schema.Version != artifact.Artifact.Schema.Version || canonical.MediaType != artifact.Artifact.MediaType || !bytes.Equal(canonical.Content, artifact.Artifact.Content) || checkpointContentDigest(canonical.Content) != checkpointContentDigest(artifact.Artifact.Content) {
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactNotCanonical, "stored artifact is not canonical")
}
hydrated, err := hydrateCheckpointArtifact(codec, cloneCheckpointArtifact(artifact), value)
if err != nil {
return nil, CheckpointArtifact{}, checkpointArtifactValidationFailure(CheckpointReasonArtifactPayloadInvalid, "hydrate checkpoint artifact: %w", err)
}
return value, hydrated, nil
}
func checkpointArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
serialized, err := serializeArtifact(codec, value, false)
if err != nil {
return CheckpointArtifact{}, err
}
return CheckpointArtifact{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
}
func serializeCandidateArtifact(codec artifactCodecEntry, laneID, moduleKey, sourceID string, value any) (CheckpointArtifact, error) {
serialized, err := serializeArtifact(codec, value, true)
if err != nil {
return CheckpointArtifact{}, err
}
return CheckpointArtifact{LaneID: laneID, ModuleKey: moduleKey, SourceID: sourceID, Artifact: serialized, SchemaDigest: contracts.DigestArtifactSchema(serialized.Schema)}, nil
}
type laneRunError struct {
stage ModuleStage
err error
@@ -189,17 +41,17 @@ func (e *laneRunError) Unwrap() error { return e.err }
type mergeStageResult struct {
artifact erasedMergeArtifact
serialized CheckpointArtifact
serialized contracts.SerializedArtifact
reuseEligible bool
}
type normalizeStageResult struct {
serialized CheckpointArtifact
serialized contracts.SerializedArtifact
accepted bool
reuseEligible bool
}
func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) error {
func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) error {
lane := prepared.resolved
if prepared.typed == nil {
return fmt.Errorf("typed lane %q executor is not prepared", lane.ID)
@@ -207,16 +59,16 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
if err := setTypedLaneManifestMetadata(output, lane.ID, prepared.typed.extractor, prepared.typed.merger, prepared.typed.normalizer); err != nil {
return err
}
merged, err := r.runMergeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, extracts, output)
merged, err := r.runMergeStage(ctx, input, doc, sourceInput, sessionID, prepared, extracts, output)
if err != nil {
return &laneRunError{stage: StageMerge, err: err}
}
normalized, err := r.runNormalizeStage(ctx, input, checkpoints, loader, doc, sourceInput, sessionID, prepared, merged.artifact, merged.serialized, merged.reuseEligible, output)
normalized, err := r.runNormalizeStage(ctx, input, doc, sourceInput, sessionID, prepared, merged.artifact, merged.reuseEligible, output)
if err != nil {
return &laneRunError{stage: StageNormalize, err: err}
}
if normalized.accepted {
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(normalized.serialized.Artifact)})
output.NormalizeOutputs = append(output.NormalizeOutputs, contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, NormalizerKey: lane.Normalize.Module, SourceID: doc.ID, Artifact: contracts.CloneSerializedArtifact(normalized.serialized)})
if output.normalizeReuseEligibility == nil {
output.normalizeReuseEligibility = make(map[generatedOutputKey]bool)
}
@@ -225,45 +77,18 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, checkpoi
return nil
}
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) {
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) {
state := mergeStageResult{}
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.MergeReferences)
state.reuseEligible = extracts.reuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.MergeReferences)
deps := normalizeCheckpointFingerprints(append(artifactCheckpointDigests(extracts.serialized), generatedReferenceDependencies(references)...))
var checkpoint MergeCheckpoint
decision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
if state.reuseEligible {
checkpoint, decision = loadMerge(loader, input.stepID, lane.ID, lane.Merge.Module, deps)
}
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageMerge, input.stepID, lane.ID, lane.Merge.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
if err != nil {
return state, err
}
if resolution.decision.Reused {
state.serialized = resolution.artifacts[0]
state.artifact = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: resolution.values[0]}
diagnostics, err := promoteCheckpointDiagnostics(state.serialized.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil)
if err != nil {
return state, err
}
appendDiagnosticGroups(output, diagnostics)
return state, nil
}
if err := ctx.Err(); err != nil {
return state, err
}
if state.reuseEligible {
if err := checkpointMergeRunning(checkpoints, input.stepID, lane.ID, lane.Merge.Module, deps); err != nil {
return state, err
}
}
inputs := make([]contracts.ExtractArtifact[any], len(extracts.accepted))
for i, value := range extracts.accepted {
inputs[i] = contracts.ExtractArtifact[any]{LaneID: value.LaneID, ExtractorKey: value.ExtractorKey, SourceID: value.SourceID, ChunkID: value.ChunkID, ChunkIndex: value.ChunkIndex, ChunkRef: value.ChunkRef, Value: value.Value}
}
if err := ctx.Err(); err != nil {
return state, err
}
metadata, err := cloneMetadata(input.Metadata)
if err != nil {
return state, err
@@ -282,16 +107,13 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
operation.Cache = artifactProducerCache(input, typed, lane.MergeExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, doc, nil, &cacheDiagnostics)
produced, err := typed.merge(ctx, typed.merger, contracts.TypedMergeRequest[any]{Operation: operation, Source: doc, LaneID: lane.ID, ExtractOutputs: inputs, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Merge.LLMProfile, Metadata: metadata})
if err != nil {
if state.reuseEligible {
_ = checkpointMergeFailed(checkpoints, input.stepID, lane.ID, lane.Merge.Module, deps, err)
}
return state, fmt.Errorf("merge lane %q: %w", lane.ID, err)
}
value, outcome, err := completeProducerOperation(ctx, lane.MergeExecutionClass, operation, produced.Value, produced.Outcome, produced.Fallback)
if err != nil {
return state, fmt.Errorf("accept merge lane %q: %w", lane.ID, err)
}
state.serialized, err = checkpointArtifact(typed.codec, lane.ID, lane.Merge.Module, doc.ID, value)
state.serialized, err = serializeArtifact(typed.codec, value, false)
if err != nil {
return state, fmt.Errorf("encode merge lane %q: %w", lane.ID, err)
}
@@ -305,51 +127,20 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, checkpoints
}
state.reuseEligible = outcome.ReuseEligible
state.artifact = erasedMergeArtifact{LaneID: lane.ID, MergerKey: lane.Merge.Module, SourceID: doc.ID, Value: value}
if state.reuseEligible {
if err := recordMerge(checkpoints, input.stepID, lane.ID, lane.Merge.Module, deps, state.serialized); err != nil {
return state, err
}
}
if err := writeDebugTimed(input.Debug, path.Join("merge", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugCheckpointArtifact(state.serialized)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("merge", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageMerge), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, SourceID: doc.ID, Artifact: state.serialized})}}); err != nil {
return state, err
}
return state, nil
}
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, serializedMerge CheckpointArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
state := normalizeStageResult{}
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.NormalizeReferences)
state.reuseEligible = upstreamReuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.NormalizeReferences)
deps := normalizeCheckpointFingerprints(append(artifactCheckpointDigests([]CheckpointArtifact{serializedMerge}), generatedReferenceDependencies(references)...))
var checkpoint NormalizeCheckpoint
decision := checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
if state.reuseEligible {
checkpoint, decision = loadNormalize(loader, input.stepID, lane.ID, lane.Normalize.Module, deps)
}
resolution, err := resolveCheckpointDecision(output, loader, input.CheckpointPolicy, StageNormalize, input.stepID, lane.ID, lane.Normalize.Module, decision, typed.codec, []CheckpointArtifact{checkpoint.Output})
if err != nil {
return state, err
}
if resolution.decision.Reused {
state.serialized = resolution.artifacts[0]
state.accepted = true
diagnostics, err := promoteCheckpointDiagnostics(state.serialized.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
if err != nil {
return state, err
}
appendDiagnosticGroups(output, diagnostics)
return state, nil
}
if err := ctx.Err(); err != nil {
return state, err
}
if state.reuseEligible {
if err := checkpointNormalizeRunning(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, deps); err != nil {
return state, err
}
}
metadata, err := cloneMetadata(input.Metadata)
if err != nil {
return state, err
@@ -368,16 +159,13 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
operation.Cache = artifactProducerCache(input, typed, lane.NormalizeExecutionClass, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, doc, nil, &cacheDiagnostics)
produced, err := typed.normalize(ctx, typed.normalizer, contracts.TypedNormalizeRequest[any]{Operation: operation, Source: doc, LaneID: lane.ID, MergeOutput: contracts.MergeArtifact[any]{LaneID: merged.LaneID, MergerKey: merged.MergerKey, SourceID: merged.SourceID, Value: merged.Value}, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Normalize.LLMProfile, Metadata: metadata})
if err != nil {
if state.reuseEligible {
_ = checkpointNormalizeFailed(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, deps, err)
}
return state, fmt.Errorf("normalize lane %q: %w", lane.ID, err)
}
value, outcome, err := completeProducerOperation(ctx, lane.NormalizeExecutionClass, operation, produced.Value, produced.Outcome, produced.Fallback)
if err != nil {
return state, fmt.Errorf("accept normalize lane %q: %w", lane.ID, err)
}
state.serialized, err = checkpointArtifact(typed.codec, lane.ID, lane.Normalize.Module, doc.ID, value)
state.serialized, err = serializeArtifact(typed.codec, value, false)
if err != nil {
return state, fmt.Errorf("encode normalize lane %q: %w", lane.ID, err)
}
@@ -391,12 +179,7 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, checkpoi
}
state.reuseEligible = outcome.ReuseEligible
state.accepted = true
if state.reuseEligible {
if err := recordNormalize(checkpoints, input.stepID, lane.ID, lane.Normalize.Module, deps, state.serialized); err != nil {
return state, err
}
}
if err := writeDebugTimed(input.Debug, path.Join("normalize", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugCheckpointArtifact(state.serialized)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("normalize", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageNormalize), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outcome": outcome, "output": debugSerializedOutputEnvelope(contracts.SerializedOutput{StepID: input.stepID, LaneID: lane.ID, SourceID: doc.ID, Artifact: state.serialized})}}); err != nil {
return state, err
}
return state, nil

View File

@@ -1,165 +0,0 @@
package pipeline
import (
"context"
"encoding/json"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type requiredCheckpointLoader struct {
CheckpointLoader
checkpoint ExtractCheckpoint
decision CheckpointDecision
}
func (l requiredCheckpointLoader) Enabled() bool { return true }
func (l requiredCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
return l.checkpoint, l.decision
}
func (l requiredCheckpointLoader) AcceptedNormalize(string, string, string) (NormalizeCheckpoint, CheckpointDecision) {
output := CheckpointArtifact{}
if len(l.checkpoint.Outputs) > 0 {
output = l.checkpoint.Outputs[0]
}
return NormalizeCheckpoint{Output: output}, l.decision
}
func TestRequiredCheckpointFailureRetainsDecision(t *testing.T) {
tests := []struct {
name string
decision CheckpointDecision
corrupt bool
wantCode CheckpointReasonCode
wantAction CheckpointDecisionCategory
}{
{"missing", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonMissing), false, CheckpointReasonMissing, CheckpointDecisionExecuted},
{"corrupt", NewCheckpointDecision(CheckpointDecisionReused, CheckpointReasonReused), true, CheckpointReasonArtifactNotCanonical, CheckpointDecisionExecuted},
{"incompatible", NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonArtifactCodecIncompatible), false, CheckpointReasonArtifactCodecIncompatible, CheckpointDecisionExecuted},
{"dependency invalidated", NewCheckpointDecision(CheckpointDecisionDependencyInvalidated, CheckpointReasonDependencyMismatch), false, CheckpointReasonDependencyMismatch, CheckpointDecisionDependencyInvalidated},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
prepared := preparedConcurrentPipeline(t, 1)
step := prepared.Steps[0]
lane := step.lanes[0]
doc := prepared.input.(*typedTestInput).doc
checkpoint := ExtractCheckpoint{}
if test.corrupt {
stored, err := checkpointArtifact(lane.typed.codec, lane.resolved.ID, lane.resolved.Normalize.Module, doc.ID, codecNotes{Items: []string{"stored"}})
if err != nil {
t.Fatal(err)
}
stored.Artifact.Content = []byte(`{"items": ["stored"]}`)
checkpoint.Outputs = []CheckpointArtifact{stored}
}
loader := requiredCheckpointLoader{CheckpointLoader: NoopCheckpointLoader(), checkpoint: checkpoint, decision: test.decision}
policy := CheckpointExecutionPolicy{RequireReusableLanes: map[string]struct{}{CheckpointLaneKey(step.ID, lane.resolved.ID): {}}}
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Checkpoint: loader, CheckpointPolicy: policy})
if err == nil || !strings.Contains(err.Error(), step.ID) || !strings.Contains(err.Error(), lane.resolved.ID) || !strings.Contains(err.Error(), string(test.wantCode)) {
t.Fatalf("Run() error = %v, want step, lane, and reason code %q", err, test.wantCode)
}
var found bool
for _, event := range output.CheckpointEvents {
if event.Stage == string(StageNormalize) && event.StepID == step.ID && event.LaneID == lane.resolved.ID {
found = true
if event.Action != test.wantAction || event.ReasonCode != test.wantCode {
t.Fatalf("checkpoint event = %#v, want action %q and reason %q", event, test.wantAction, test.wantCode)
}
if event.Detail == "" || event.Detail != event.Reason {
t.Fatalf("checkpoint event detail is not code-owned compatibility text: %#v", event)
}
}
}
if !found {
t.Fatalf("required checkpoint decision missing from %#v", output.CheckpointEvents)
}
var manifestFound bool
for _, decision := range output.Manifest.CheckpointDecisions {
if decision.Stage == string(StageNormalize) && decision.StepID == step.ID && decision.LaneID == lane.resolved.ID {
manifestFound = decision.Category == string(test.wantAction) && decision.ReasonCode == string(test.wantCode)
}
}
if !manifestFound {
t.Fatalf("manifest checkpoint decision missing category %q and code %q: %#v", test.wantAction, test.wantCode, output.Manifest.CheckpointDecisions)
}
encoded, marshalErr := json.Marshal(output.CheckpointEvents)
if marshalErr != nil || !strings.Contains(string(encoded), `"category":"`+string(test.wantAction)+`"`) || !strings.Contains(string(encoded), `"reason_code":"`+string(test.wantCode)+`"`) {
t.Fatalf("checkpoint event JSON = %s, error = %v", encoded, marshalErr)
}
})
}
}
func TestCheckpointDecisionDetailDoesNotCopyUnknownReasonCode(t *testing.T) {
const sentinel = "opaque-sensitive-value-78421"
decision := NewCheckpointDecision(CheckpointDecisionExecuted, CheckpointReasonCode(sentinel))
if decision.Detail == "" || decision.Detail != decision.Reason {
t.Fatalf("decision detail is not bounded compatibility text: %#v", decision)
}
if strings.Contains(decision.Detail, sentinel) || strings.Contains(decision.Reason, sentinel) {
t.Fatalf("decision detail copied caller-controlled reason code: %#v", decision)
}
}
func TestDecodeCheckpointArtifactRejectsIncompatibleCodecIdentityAndBytes(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec: %v", err)
}
codec, _, err := registry.entry("test/notes")
if err != nil {
t.Fatalf("entry: %v", err)
}
artifact, err := serializeArtifact(codec, codecNotes{Items: []string{"one"}}, false)
if err != nil {
t.Fatalf("serializeArtifact: %v", err)
}
base := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
tests := []struct {
name string
mutate func(*CheckpointArtifact)
want string
}{
{name: "missing kind", mutate: func(v *CheckpointArtifact) { v.Artifact.Kind = "" }, want: "artifact kind"},
{name: "schema version", mutate: func(v *CheckpointArtifact) { v.Artifact.Schema.Version = "v999" }, want: "does not match codec schema"},
{name: "schema digest", mutate: func(v *CheckpointArtifact) { v.SchemaDigest = "sha256:different" }, want: "schema digest"},
{name: "media type", mutate: func(v *CheckpointArtifact) { v.Artifact.MediaType = "text/plain" }, want: "media type"},
{name: "decode failure", mutate: func(v *CheckpointArtifact) { v.Artifact.Content = []byte(`{"items":[`) }, want: "unexpected EOF"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
stored := cloneCheckpointArtifact(base)
test.mutate(&stored)
if _, err := decodeCheckpointArtifact(codec, stored); err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("decode error = %v, want %q", err, test.want)
}
})
}
}
func TestDecodeCanonicalCheckpointArtifactRejectsNonCanonicalBytes(t *testing.T) {
registry := NewArtifactCodecRegistry()
if err := RegisterArtifactCodec(registry, notesCodec()); err != nil {
t.Fatalf("RegisterArtifactCodec: %v", err)
}
codec, _, err := registry.entry("test/notes")
if err != nil {
t.Fatalf("entry: %v", err)
}
artifact, err := serializeArtifact(codec, codecNotes{Items: []string{"one"}}, false)
if err != nil {
t.Fatalf("serializeArtifact: %v", err)
}
stored := CheckpointArtifact{Artifact: artifact, SchemaDigest: contracts.DigestArtifactSchema(artifact.Schema)}
if _, _, err := decodeCanonicalCheckpointArtifact(codec, stored); err != nil {
t.Fatalf("canonical artifact rejected: %v", err)
}
stored.Artifact.Content = []byte(`{"items": ["one"]}`)
if _, _, err := decodeCanonicalCheckpointArtifact(codec, stored); err == nil {
t.Fatal("non-canonical artifact was accepted")
}
}

View File

@@ -1,11 +1,6 @@
package pipeline
import (
"sync"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
import "sync"
type lockedDebugRecorder struct {
inner DebugRecorder
@@ -39,217 +34,3 @@ func (r *lockedDebugRecorder) WriteBytes(name string, data []byte) error {
defer r.mu.Unlock()
return r.inner.WriteBytes(name, data)
}
type lockedCheckpointLoader struct {
inner CheckpointLoader
mu sync.Mutex
}
func synchronizedCheckpointLoader(inner CheckpointLoader) CheckpointLoader {
if _, ok := inner.(*lockedCheckpointLoader); ok {
return inner
}
return &lockedCheckpointLoader{inner: inner}
}
func (l *lockedCheckpointLoader) Enabled() bool {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Enabled()
}
func (l *lockedCheckpointLoader) Source(key string) (SourceCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Source(key)
}
func (l *lockedCheckpointLoader) Extract(lane, key string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Extract(lane, key, deps)
}
func (l *lockedCheckpointLoader) Merge(lane, key string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Merge(lane, key, deps)
}
func (l *lockedCheckpointLoader) Normalize(lane, key string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.Normalize(lane, key, deps)
}
func (l *lockedCheckpointLoader) AcceptedNormalize(step, lane, key string) (NormalizeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
return l.inner.AcceptedNormalize(step, lane, key)
}
func (l *lockedCheckpointLoader) ExtractForStep(step, lane, key string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.ExtractForStep(step, lane, key, deps)
}
return l.inner.Extract(lane, key, deps)
}
func (l *lockedCheckpointLoader) MergeForStep(step, lane, key string, deps []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.MergeForStep(step, lane, key, deps)
}
return l.inner.Merge(lane, key, deps)
}
func (l *lockedCheckpointLoader) NormalizeForStep(step, lane, key string, deps []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
l.mu.Lock()
defer l.mu.Unlock()
if stepAware, ok := l.inner.(StepCheckpointLoader); ok {
return stepAware.NormalizeForStep(step, lane, key, deps)
}
return l.inner.Normalize(lane, key, deps)
}
type lockedCheckpointRecorder struct {
inner CheckpointRecorder
mu sync.Mutex
}
func synchronizedCheckpointRecorder(inner CheckpointRecorder) CheckpointRecorder {
if _, ok := inner.(*lockedCheckpointRecorder); ok {
return inner
}
return &lockedCheckpointRecorder{inner: inner}
}
func (r *lockedCheckpointRecorder) call(fn func() error) error {
r.mu.Lock()
defer r.mu.Unlock()
return fn()
}
func (r *lockedCheckpointRecorder) SourceRunning(key string) error {
return r.call(func() error { return r.inner.SourceRunning(key) })
}
func (r *lockedCheckpointRecorder) SourceSucceeded(key string, doc *source.SourceDocument) error {
return r.call(func() error { return r.inner.SourceSucceeded(key, doc) })
}
func (r *lockedCheckpointRecorder) SourceFailed(key string, err error) error {
return r.call(func() error { return r.inner.SourceFailed(key, err) })
}
func (r *lockedCheckpointRecorder) ExtractRunning(lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error { return r.inner.ExtractRunning(lane, key, deps) })
}
func (r *lockedCheckpointRecorder) ExtractSucceeded(lane, key string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
return r.call(func() error { return r.inner.ExtractSucceeded(lane, key, deps, outputs, rejected) })
}
func (r *lockedCheckpointRecorder) ExtractFailed(lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error { return r.inner.ExtractFailed(lane, key, deps, err) })
}
func (r *lockedCheckpointRecorder) MergeRunning(lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error { return r.inner.MergeRunning(lane, key, deps) })
}
func (r *lockedCheckpointRecorder) MergeSucceeded(lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error { return r.inner.MergeSucceeded(lane, key, deps, output) })
}
func (r *lockedCheckpointRecorder) MergeRejected(lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error { return r.inner.MergeRejected(lane, key, deps, rejected) })
}
func (r *lockedCheckpointRecorder) MergeFailed(lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error { return r.inner.MergeFailed(lane, key, deps, err) })
}
func (r *lockedCheckpointRecorder) NormalizeRunning(lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error { return r.inner.NormalizeRunning(lane, key, deps) })
}
func (r *lockedCheckpointRecorder) NormalizeSucceeded(lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error { return r.inner.NormalizeSucceeded(lane, key, deps, output) })
}
func (r *lockedCheckpointRecorder) NormalizeRejected(lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error { return r.inner.NormalizeRejected(lane, key, deps, rejected) })
}
func (r *lockedCheckpointRecorder) NormalizeFailed(lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error { return r.inner.NormalizeFailed(lane, key, deps, err) })
}
func (r *lockedCheckpointRecorder) ExtractRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractRunningForStep(step, lane, key, deps)
}
return r.inner.ExtractRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) ExtractSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, outputs []CheckpointArtifact, rejected []contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractSucceededForStep(step, lane, key, deps, outputs, rejected)
}
return r.inner.ExtractSucceeded(lane, key, deps, outputs, rejected)
})
}
func (r *lockedCheckpointRecorder) ExtractFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.ExtractFailedForStep(step, lane, key, deps, err)
}
return r.inner.ExtractFailed(lane, key, deps, err)
})
}
func (r *lockedCheckpointRecorder) MergeRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeRunningForStep(step, lane, key, deps)
}
return r.inner.MergeRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) MergeSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeSucceededForStep(step, lane, key, deps, output)
}
return r.inner.MergeSucceeded(lane, key, deps, output)
})
}
func (r *lockedCheckpointRecorder) MergeRejectedForStep(step, lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeRejectedForStep(step, lane, key, deps, rejected)
}
return r.inner.MergeRejected(lane, key, deps, rejected)
})
}
func (r *lockedCheckpointRecorder) MergeFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.MergeFailedForStep(step, lane, key, deps, err)
}
return r.inner.MergeFailed(lane, key, deps, err)
})
}
func (r *lockedCheckpointRecorder) NormalizeRunningForStep(step, lane, key string, deps []CheckpointFingerprint) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeRunningForStep(step, lane, key, deps)
}
return r.inner.NormalizeRunning(lane, key, deps)
})
}
func (r *lockedCheckpointRecorder) NormalizeSucceededForStep(step, lane, key string, deps []CheckpointFingerprint, output CheckpointArtifact) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeSucceededForStep(step, lane, key, deps, output)
}
return r.inner.NormalizeSucceeded(lane, key, deps, output)
})
}
func (r *lockedCheckpointRecorder) NormalizeRejectedForStep(step, lane, key string, deps []CheckpointFingerprint, rejected contracts.RejectedOutput) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeRejectedForStep(step, lane, key, deps, rejected)
}
return r.inner.NormalizeRejected(lane, key, deps, rejected)
})
}
func (r *lockedCheckpointRecorder) NormalizeFailedForStep(step, lane, key string, deps []CheckpointFingerprint, err error) error {
return r.call(func() error {
if v, ok := r.inner.(StepCheckpointRecorder); ok {
return v.NormalizeFailedForStep(step, lane, key, deps, err)
}
return r.inner.NormalizeFailed(lane, key, deps, err)
})
}

View File

@@ -14,6 +14,7 @@ type erasedExtractArtifact struct {
ChunkIndex int
ChunkRef source.SourceRef
Value any
artifact contracts.SerializedArtifact
}
type erasedMergeArtifact struct {

View File

@@ -181,7 +181,7 @@ func TestPrepareConstructsHeterogeneousTypedLanes(t *testing.T) {
}
}
func TestRunExecutesHeterogeneousTypedLanesWithCheckpointsDisabled(t *testing.T) {
func TestRunExecutesHeterogeneousTypedLanes(t *testing.T) {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
resolved, err := ResolvePipeline(typedResolutionProfile(), ResolveOptions{}, catalog)
if err != nil {
@@ -198,9 +198,6 @@ func TestRunExecutesHeterogeneousTypedLanesWithCheckpointsDisabled(t *testing.T)
if len(output.NormalizeOutputs) != 2 {
t.Fatalf("normalize outputs = %#v, want two typed lane results", output.NormalizeOutputs)
}
if len(output.CheckpointEvents) != 0 {
t.Fatalf("checkpoint events = %#v, want none with checkpoint loading disabled", output.CheckpointEvents)
}
}
func TestResolveTypedLaneRejectsIncompatibleComposition(t *testing.T) {