Bind runner assessment and propagate fallback reuse eligibility

This commit is contained in:
2026-11-21 23:48:51 +00:00
parent 44e4fb92c2
commit 923cbe594f
19 changed files with 852 additions and 902 deletions

View File

@@ -10,40 +10,36 @@ import (
"sync"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
type laneExtractState struct {
index int
prepared preparedLaneExecutor
deps []CheckpointFingerprint
decision CheckpointDecision
reuseEligible bool
values []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
incomplete []int
validationSummaries []artifacts.ValidationSummary
results map[int]extractJobResult
remaining int
failed bool
terminal bool
output RunOutput
fallbackCount int
index int
prepared preparedLaneExecutor
deps []CheckpointFingerprint
decision CheckpointDecision
reuseEligible bool
values []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
results map[int]extractJobResult
remaining int
failed bool
terminal bool
output RunOutput
}
type finalizedExtractResults struct {
accepted []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
incomplete []int
validationSummaries []artifacts.ValidationSummary
decision CheckpointDecision
reuseEligible bool
accepted []erasedExtractArtifact
serialized []CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected []contracts.RejectedOutput
decision CheckpointDecision
reuseEligible bool
}
func loadExtract(loader CheckpointLoader, stepID, laneID, moduleKey string, deps []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
@@ -63,21 +59,14 @@ type extractJob struct {
}
type extractJobResult struct {
laneIndex int
chunkIndex int
value erasedExtractArtifact
serialized CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected *contracts.RejectedOutput
validationIncomplete bool
validationSummary *artifacts.ValidationSummary
err error
}
type extractAttemptValue struct {
artifact erasedExtractArtifact
serialized CheckpointArtifact
terminal *attemptTerminalRecorder
laneIndex int
chunkIndex int
value erasedExtractArtifact
serialized CheckpointArtifact
diagnostics []contracts.DiagnosticGroup
rejected *contracts.RejectedOutput
outcome contracts.ProducerOutcome
err error
}
type laneCompletion struct {
@@ -125,7 +114,7 @@ func initializeLaneStates(input RunInput, step PreparedPipelineStep, checkpoints
return states, err
}
if input.CheckpointPolicy.requiresReusable(input.stepID, prepared.resolved.ID) && !input.CheckpointPolicy.forced(input.stepID, prepared.resolved.ID) {
if !laneReferencesReuseEligible(input, prepared.resolved) {
if input.reuseIneligible || !laneReferencesReuseEligible(input, prepared.resolved) {
return states, fmt.Errorf("required reusable checkpoint unavailable for step %q lane %q: generated input lineage is validation-incomplete", input.stepID, prepared.resolved.ID)
}
state, err := hydrateRequiredLane(input, loader, doc, i, prepared)
@@ -399,7 +388,7 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
}
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
deps := append(digestFingerprints("chunks", digest), generatedReferenceDependencies(extractReferences)...)
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), reuseEligible: referenceTargetReuseEligible(input, lane.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
state := &laneExtractState{index: index, prepared: prepared, deps: normalizeCheckpointFingerprints(deps), reuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences), remaining: len(chunks), results: make(map[int]extractJobResult, len(chunks))}
if !state.reuseEligible {
state.decision = checkpointDecision(CheckpointDecisionExecuted, CheckpointReasonValidationIncompleteLineage)
recordCheckpointEvent(output, loader, string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, state.decision)
@@ -436,99 +425,56 @@ func prepareLaneExtract(input RunInput, loader CheckpointLoader, doc *source.Sou
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
state := job.lane
chunk, cloneErr := cloneSourceChunk(job.chunk)
lane, typed := state.prepared.resolved, state.prepared.typed
result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
if cloneErr != nil {
result.err = fmt.Errorf("clone chunk %q for extraction: %w", job.chunk.ID, cloneErr)
if err := ctx.Err(); err != nil {
result.err = err
return result
}
terminalResult, err := runProducerAttempts(ctx, producerAttemptConfig{Retries: lane.Extract.Retries, Policy: lane.ExtractValidationPolicy, AllowStructuralRetry: lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked}, func(attemptCtx context.Context, request producerAttemptRequest) (producerAttemptOutput, error) {
attempt := request.Number
started := time.Now().UTC()
attemptPath := path.Join("extract", fileio.EncodePathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), fmt.Sprintf("attempt-%02d", attempt))
attemptCtx, llmScope := withDebugLLMScope(attemptCtx, attemptPath)
terminal := newAttemptTerminalRecorder(input.Debug, attemptPath, "extract", llmScope, debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, Attempt: attempt, AttemptKind: string(request.Kind), StartedAt: started})
requestMetadata, metadataErr := cloneMetadata(input.Metadata)
if metadataErr != nil {
return producerAttemptOutput{}, terminal.record(nil, fmt.Errorf("clone extract request metadata: %w", metadataErr))
}
extractReferences := operationReferenceSet(input, lane.ExtractReferences)
extracted, callErr := typed.extract(attemptCtx, typed.extractor, contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk, SourceInput: chunkInputMaterial(sourceInput, chunk), SessionID: sessionID, References: CloneReferenceSet(extractReferences), LLMProfile: lane.Extract.LLMProfile, StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(lane.Extract.StructuredOutputRepairAttempts), Correction: request.Correction, Metadata: requestMetadata})
if callErr != nil {
attemptErr := fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, lane.Extract.Module, callErr)
return producerAttemptOutput{}, terminal.record(nil, attemptErr)
}
artifact := erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: extracted.Value}
serializedCandidate, encodeErr := serializeCandidateArtifact(typed.codec, artifact.LaneID, artifact.ExtractorKey, artifact.SourceID, artifact.Value)
if encodeErr != nil {
attemptErr := fmt.Errorf("serialize extract candidate for lane %q chunk %q: %w", lane.ID, chunk.ID, encodeErr)
payload := map[string]any{}
return producerAttemptOutput{}, terminal.record(payload, attemptErr)
}
serializedCandidate.ChunkID, serializedCandidate.ChunkIndex, serializedCandidate.ChunkRef = artifact.ChunkID, artifact.ChunkIndex, artifact.ChunkRef
return producerAttemptOutput{Value: extractAttemptValue{artifact: artifact, serialized: serializedCandidate, terminal: &terminal}, Candidate: extracted.ModelCandidate, Diagnostics: contracts.CloneProducerDiagnostics(extracted.Diagnostics)}, nil
}, func(validationCtx context.Context, output producerAttemptOutput) (validationReport, error) {
candidate, ok := output.Value.(extractAttemptValue)
if !ok {
return validationReport{}, fmt.Errorf("extract attempt has incompatible value")
}
report, validationErr := r.validateTypedReport(validationCtx, typed.codec, typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: operationReferenceSet(input, lane.ExtractReferences), metadata: input.Metadata, chunk: &chunk, ref: chunk.Ref, value: candidate.artifact.Value, candidate: &candidate.serialized}, state.prepared.extractValidators, candidate.terminal.envelope.Attempt, input.Debug)
payload := map[string]any{
"output": debugCheckpointArtifact(candidate.serialized),
"rejection": debugRejectedOutputPtr(typedRejection(report, typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, chunk: &chunk}, candidate.terminal.envelope.Attempt)),
}
if validationErr != nil {
return report, candidate.terminal.record(payload, validationErr)
}
return report, candidate.terminal.record(payload, nil)
})
result.err = err
summary := validationSummary(terminalResult, StageExtract, input.stepID, lane.ID, lane.Extract.Module, chunk.ID, chunk.Index)
if debugErr := writeProducerTerminalDebug(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), fmt.Sprintf("chunk-%06d", chunk.Index+1), "terminal.json"), terminalResult, lane.ExtractValidationPolicy, summary); debugErr != nil {
result.err = errors.Join(result.err, debugErr)
chunk, err := cloneSourceChunk(job.chunk)
if err != nil {
result.err = err
return result
}
if err == nil {
diagnostics, diagnosticErr := terminalDiagnosticGroups(terminalResult, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if diagnosticErr != nil {
result.err = fmt.Errorf("promote extract diagnostics: %w", diagnosticErr)
return result
}
result.diagnostics = diagnostics
}
if err == nil && terminalResult.Action == producerTerminalRejected {
result.rejected = terminalResult.Rejection
if result.rejected != nil {
result.rejected.Stage, result.rejected.StepID, result.rejected.LaneID, result.rejected.ModuleKey, result.rejected.ChunkID, result.rejected.ChunkIndex = string(StageExtract), input.stepID, lane.ID, lane.Extract.Module, chunk.ID, chunk.Index
result.validationSummary = &summary
result.rejected.Validation = cloneValidationSummaryPtr(result.validationSummary)
}
metadata, err := cloneMetadata(input.Metadata)
if err != nil {
result.err = err
return result
}
if err == nil {
candidate, ok := terminalResult.Value.(extractAttemptValue)
if !ok {
result.err = fmt.Errorf("extract attempt terminal has incompatible value")
return result
}
stored, encodeErr := checkpointArtifact(typed.codec, candidate.artifact.LaneID, candidate.artifact.ExtractorKey, candidate.artifact.SourceID, candidate.artifact.Value)
payload := map[string]any{"output": debugCheckpointArtifact(candidate.serialized), "rejection": debugRejectedOutputPtr(nil)}
if encodeErr != nil {
attemptErr := fmt.Errorf("serialize accepted extract output for lane %q chunk %q: %w", lane.ID, chunk.ID, encodeErr)
result.err = candidate.terminal.record(payload, attemptErr)
return result
}
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = candidate.artifact.ChunkID, candidate.artifact.ChunkIndex, candidate.artifact.ChunkRef
stored.Diagnostics = terminalCheckpointDiagnostics(terminalResult)
if debugErr := candidate.terminal.record(payload, nil); debugErr != nil {
result.err = debugErr
return result
}
result.value, result.serialized = candidate.artifact, stored
result.validationIncomplete = terminalResult.ValidationIncomplete
result.validationSummary = &summary
references := operationReferenceSet(input, lane.ExtractReferences)
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution, chunk: &chunk, ref: chunk.Ref}
var assessment contracts.CandidateAssessment
assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators)
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
var err error
assessment, err = assess(ctx, value)
return assessment, err
},
}
extracted, err := typed.extract(ctx, typed.extractor, contracts.TypedExtractionRequest[any]{Operation: operation, Source: doc, Chunk: &chunk, SourceInput: target.sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: lane.Extract.LLMProfile, Metadata: metadata})
if err != nil {
result.err = fmt.Errorf("extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
value, outcome, err := completeProducerOperation(ctx, lane.ExtractExecutionClass, operation, extracted.Value, extracted.Outcome, extracted.Fallback)
if err != nil {
result.err = fmt.Errorf("accept extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
stored, err := checkpointArtifact(typed.codec, lane.ID, lane.Extract.Module, doc.ID, value)
if err != nil {
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
stored.ChunkID, stored.ChunkIndex, stored.ChunkRef = chunk.ID, chunk.Index, chunk.Ref
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if err != nil {
result.err = err
return result
}
result.value = erasedExtractArtifact{LaneID: lane.ID, ExtractorKey: lane.Extract.Module, SourceID: doc.ID, ChunkID: chunk.ID, ChunkIndex: chunk.Index, ChunkRef: chunk.Ref, Value: value}
result.serialized, result.outcome = stored, outcome
return result
}
@@ -541,9 +487,6 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
sort.Ints(indexes)
for _, index := range indexes {
result := state.results[index]
if result.validationSummary != nil {
state.validationSummaries = append(state.validationSummaries, artifacts.CloneValidationSummary(*result.validationSummary))
}
if result.rejected != nil {
state.rejected = append(state.rejected, *result.rejected)
state.diagnostics = append(state.diagnostics, result.diagnostics...)
@@ -552,17 +495,16 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
state.values = append(state.values, result.value)
state.serialized = append(state.serialized, result.serialized)
state.diagnostics = append(state.diagnostics, result.diagnostics...)
if result.validationIncomplete {
state.incomplete = append(state.incomplete, result.chunkIndex)
if !result.outcome.ReuseEligible {
state.reuseEligible = false
}
if result.outcome.Disposition == contracts.ProducerFallback {
state.fallbackCount++
}
}
sort.SliceStable(state.values, func(i, j int) bool { return state.values[i].ChunkIndex < state.values[j].ChunkIndex })
sort.SliceStable(state.serialized, func(i, j int) bool { return state.serialized[i].ChunkIndex < state.serialized[j].ChunkIndex })
sort.SliceStable(state.rejected, func(i, j int) bool { return state.rejected[i].ChunkIndex < state.rejected[j].ChunkIndex })
sort.Ints(state.incomplete)
if len(state.incomplete) > 0 {
state.reuseEligible = false
}
if !state.decision.Reused && state.reuseEligible {
if err := recordExtract(checkpoints, stepID, lane.ID, lane.Extract.Module, state.deps, state.serialized, state.rejected); err != nil {
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
@@ -573,24 +515,21 @@ func finalizeLaneExtract(checkpoints CheckpointRecorder, stepID string, state *l
func (r *Runner) continueLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, loader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []source.Chunk, state *laneExtractState) (RunOutput, error) {
lane := state.prepared.resolved
local := RunOutput{Manifest: manifestFromPipeline(input)}
local := RunOutput{Manifest: manifestFromPipeline(input), FallbackCount: state.fallbackCount}
results := finalizedExtractResults{
accepted: state.values,
serialized: state.serialized,
diagnostics: state.diagnostics,
rejected: state.rejected,
incomplete: state.incomplete,
validationSummaries: state.validationSummaries,
decision: state.decision,
reuseEligible: state.reuseEligible,
accepted: state.values,
serialized: state.serialized,
diagnostics: state.diagnostics,
rejected: state.rejected,
decision: state.decision,
reuseEligible: state.reuseEligible,
}
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
local.Rejected = append(local.Rejected, cloneRejectedOutputs(results.rejected)...)
local.ValidationSummaries = append(local.ValidationSummaries, cloneValidationSummaries(results.validationSummaries)...)
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "decision": results.decision, "source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "outputs": debugCheckpointArtifacts(results.serialized), "rejected": debugRejectedOutputEnvelopes(results.rejected), "validation_incomplete_chunks": append([]int(nil), results.incomplete...)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"reused": results.decision.Reused, "outputs": debugCheckpointArtifacts(results.serialized), "rejected": debugRejectedOutputEnvelopes(results.rejected)}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if len(results.accepted) == 0 {
@@ -653,6 +592,7 @@ func mergeLaneOutput(dst *RunOutput, src RunOutput) error {
if dst == nil {
return nil
}
dst.FallbackCount += src.FallbackCount
dst.NormalizeOutputs = append(dst.NormalizeOutputs, cloneSerializedOutputs(src.NormalizeOutputs)...)
dst.Rejected = append(dst.Rejected, cloneRejectedOutputs(src.Rejected)...)
appendDiagnosticGroups(dst, src.diagnosticGroups)