Bound concurrent section work and finalize proposal identities in order

This commit is contained in:
2026-11-16 14:53:14 +00:00
parent 1c43264f6f
commit b98d463958
11 changed files with 739 additions and 240 deletions

View File

@@ -56,17 +56,18 @@ type ModuleResult struct {
// WorkUnitOutcome keeps initial section identity while text evolves between
// module instances. Detailed candidate and decision records are debug evidence.
type WorkUnitOutcome struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleIndex int `json:"module_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
ProposalIndexOffset int `json:"-"`
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleIndex int `json:"module_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
}
type ValidatorDecisionRecord struct {
@@ -342,15 +343,21 @@ type sectionProposalResult struct {
}
type sectionValidationResult struct {
repairs []RepairOutcome
sectionPos int
enriched []proposals.EnrichedCorrectionProposal
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
failure *ExecutionFailure
interactions *sectionInteractions
repairs []RepairOutcome
sectionPos int
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
failure *ExecutionFailure
}
type sectionWork struct {
candidates []proposals.EnrichedCorrectionProposal
validation *sectionValidationResult
finished bool
}
type modulePipelineResult struct {
@@ -363,60 +370,42 @@ type modulePipelineResult struct {
ValidatorRejected []ValidatorRejectedChange
}
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
results := make(chan sectionProposalResult, len(input.Sections))
// moduleWorkWindow bounds active and downstream work across all roles in
// one module. PromptKit still owns admission to each provider backend.
const moduleWorkWindow = 8
go func() {
defer close(results)
var wg sync.WaitGroup
for sectionPos, section := range input.Sections {
sectionPos := sectionPos
section := section
wg.Add(1)
go func() {
defer wg.Done()
if ctx.Err() != nil {
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
return
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
InstanceName: input.Spec.InstanceName,
ReplacementPolicy: input.Policy,
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
},
LLMClient: input.ProposalClient,
})
var failure *ExecutionFailure
if err != nil {
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
fail(failure)
}
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
proposal: proposalResult,
started: true,
failure: failure,
}
}()
}
wg.Wait()
}()
return results
func proposeSection(ctx context.Context, input collectSectionProposalsInput, sectionPos int) sectionProposalResult {
section := input.Sections[sectionPos]
result := sectionProposalResult{sectionPos: sectionPos, section: section}
if ctx.Err() != nil {
return result
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
InstanceName: input.Spec.InstanceName,
ReplacementPolicy: input.Policy,
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
},
LLMClient: input.ProposalClient,
})
result.started = true
result.proposal = proposalResult
if err != nil {
result.failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
}
return result
}
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
@@ -453,23 +442,26 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
input.Spec.CaptureProposalExchange = true
}
pending := make(map[int]sectionProposalResult, len(input.Sections))
validationResults := make(chan sectionValidationResult, len(input.Sections))
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
pending := make(map[int]*sectionWork, moduleWorkWindow)
proposalResults := make(chan sectionProposalResult)
validationResults := make(chan sectionValidationResult)
validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators())
nextSectionToProcess := 0
nextSectionToRecord := 0
nextProposalIndex := 0
var (
firstFailure *ExecutionFailure
errMu sync.Mutex
vwg sync.WaitGroup
workers sync.WaitGroup
)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
setFailure := func(failure *ExecutionFailure) {
if failure == nil {
return
}
errMu.Lock()
defer errMu.Unlock()
if firstFailure == nil {
@@ -477,133 +469,93 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
cancel()
}
}
proposalResults := collectSectionProposals(runCtx, input, setFailure)
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
makeCandidates := func(sectionResult sectionProposalResult) []proposals.EnrichedCorrectionProposal {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := index
sectionIndex := sectionResult.section.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: nextProposalIndex + i,
ProposalIndex: i,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: &sectionIndex,
},
})
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range sectionEnriched {
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
}
}
}
return sectionEnriched
}
for result := range proposalResults {
unit := &out.WorkUnits[result.sectionPos]
if result.started {
unit.State = outcomes.WorkInterrupted
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
unit := &out.WorkUnits[sectionPos]
unit.ProposalIndexOffset = nextProposalIndex
for _, candidate := range enriched {
candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
}
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.failure != nil {
unit.Failure = result.failure
continue
}
if !result.started {
continue
}
pending[result.sectionPos] = result
for {
if runCtx.Err() != nil {
break
out.ProposalCount += len(enriched)
for operationIndex := range unit.Operations {
operation := &unit.Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range enriched {
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
}
}
sectionResult, ok := pending[nextSectionToProcess]
if !ok {
break
}
nextProposalIndex += len(enriched)
}
validate := func(sectionResult sectionProposalResult, enriched []proposals.EnrichedCorrectionProposal) sectionValidationResult {
sectionPos := sectionResult.sectionPos
sectionMetadata := contracts.SectionMetadataFromSection(sectionResult.section)
var interactions structuredoutput.InteractionSink
var capture *sectionInteractions
if input.Interactions != nil {
capture = &sectionInteractions{InteractionSink: input.Interactions}
interactions = capture
}
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
vwg.Add(1)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
setFailure(failure)
}
validationResults <- sectionValidationResult{
repairs: repairs,
sectionPos: sectionPos,
enriched: enriched,
approved: validated.approved,
decisions: validated.decisions,
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
failure: failure,
}
}(nextSectionToProcess, sectionEnriched, sectionMeta)
nextSectionToProcess++
}
return sectionValidationResult{
interactions: capture,
repairs: repairs,
sectionPos: sectionPos,
approved: validated.approved,
decisions: validated.decisions,
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
failure: failure,
}
}
vwg.Wait()
close(validationResults)
// A failed or canceled module can still have usable proposals from later
// sections. Assign their indices in order after preserving the dispatched
// prefix, without submitting new validation work.
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
if result, ok := pending[pos]; ok {
collectCandidates(result, result.section.Index)
}
}
for v := range validationResults {
validationBySection[v.sectionPos] = v
}
for i := range input.Sections {
res, ok := validationBySection[i]
if !ok {
continue
}
unit := &out.WorkUnits[i]
recordValidation := func(res sectionValidationResult) {
unit := &out.WorkUnits[res.sectionPos]
unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered)
@@ -617,6 +569,69 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
}
nextAdmission, active := 0, 0
for {
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
offset := nextProposalIndex
recordCandidates(nextSectionToRecord, work.candidates)
if work.validation != nil {
work.validation.assignProposalIndices(offset)
recordValidation(*work.validation)
}
delete(pending, nextSectionToRecord)
nextSectionToRecord++
}
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
sectionPos := nextAdmission
nextAdmission++
pending[sectionPos] = &sectionWork{}
active++
workers.Go(func() {
proposed := proposeSection(runCtx, input, sectionPos)
setFailure(proposed.failure)
proposalResults <- proposed
})
}
if active == 0 {
break
}
select {
case result := <-proposalResults:
active--
unit := &out.WorkUnits[result.sectionPos]
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Failure = result.failure
work := pending[result.sectionPos]
if result.started && result.failure == nil {
work.candidates = makeCandidates(result)
if runCtx.Err() == nil {
enriched := work.candidates
active++
workers.Go(func() {
validated := validate(result, enriched)
setFailure(validated.failure)
validationResults <- validated
})
continue
}
}
work.finished = true
case validated := <-validationResults:
active--
work := pending[validated.sectionPos]
work.validation = &validated
work.finished = true
}
}
workers.Wait()
sort.SliceStable(out.Approved, func(i, j int) bool {
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
})
for _, unit := range out.WorkUnits {
out.Operations = append(out.Operations, unit.Operations...)
}