|
|
|
|
@@ -63,6 +63,7 @@ type WorkUnitOutcome struct {
|
|
|
|
|
ModuleInstance string `json:"module_instance"`
|
|
|
|
|
Section contracts.SectionMetadata `json:"section"`
|
|
|
|
|
State outcomes.WorkState `json:"state"`
|
|
|
|
|
Failure *ExecutionFailure `json:"-"`
|
|
|
|
|
CorrectionsApplied int `json:"corrections_applied"`
|
|
|
|
|
Repairs []RepairOutcome `json:"-"`
|
|
|
|
|
Operations []outcomes.Operation `json:"operations,omitempty"`
|
|
|
|
|
@@ -116,15 +117,18 @@ func New(factory ModuleFactory) *Runner {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return RunOutput{FinalTranscript: input.Transcript.Clone()}, &ExecutionFailure{Phase: "processing", Cause: err}
|
|
|
|
|
}
|
|
|
|
|
if r == nil || r.factory == nil {
|
|
|
|
|
return RunOutput{}, fmt.Errorf("runner module factory is required")
|
|
|
|
|
return RunOutput{}, &ExecutionFailure{Phase: "setup", Cause: errors.New("runner module factory is required")}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
working := input.Transcript.Clone()
|
|
|
|
|
results := make([]ModuleResult, 0, len(input.ModuleSpecs))
|
|
|
|
|
sections, err := chunkWorkingTranscript(input.Config, working)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return RunOutput{FinalTranscript: working}, fmt.Errorf("chunk normalized transcript: %w", err)
|
|
|
|
|
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
|
|
|
|
|
}
|
|
|
|
|
inventory := make([]contracts.SectionMetadata, len(sections))
|
|
|
|
|
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
|
|
|
|
|
@@ -149,135 +153,124 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for moduleIndex, spec := range input.ModuleSpecs {
|
|
|
|
|
module, err := r.factory.ModuleForSpec(spec)
|
|
|
|
|
if err != nil {
|
|
|
|
|
failed := ModuleResult{
|
|
|
|
|
StageIndex: spec.Index, StageID: spec.ID,
|
|
|
|
|
ModuleKey: spec.ModuleKey,
|
|
|
|
|
ModuleInstance: spec.InstanceName,
|
|
|
|
|
Status: ModuleStatusFailed,
|
|
|
|
|
ErrorMessage: err.Error(),
|
|
|
|
|
}
|
|
|
|
|
results = append(results, failed)
|
|
|
|
|
return snapshot(), fmt.Errorf("module %q setup failed: %w", spec.InstanceName, err)
|
|
|
|
|
execution, err := r.executeModule(ctx, spec, sections, input.Interactions)
|
|
|
|
|
units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
|
|
|
|
|
copy(units, execution.pipeline.WorkUnits)
|
|
|
|
|
if err == nil && ctx.Err() != nil {
|
|
|
|
|
err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil)
|
|
|
|
|
}
|
|
|
|
|
if err == nil {
|
|
|
|
|
previous := sections
|
|
|
|
|
sections = execution.updated
|
|
|
|
|
attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
policy := module.ReplacementPolicy()
|
|
|
|
|
if err := spec.PrepareGlossary(); err != nil {
|
|
|
|
|
return snapshot(), fmt.Errorf("module %q glossary preparation: %w", spec.InstanceName, err)
|
|
|
|
|
var failure *ExecutionFailure
|
|
|
|
|
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
|
|
|
|
|
appliedIndices := make(map[int]struct{}, len(execution.applied))
|
|
|
|
|
for _, change := range execution.applied {
|
|
|
|
|
appliedIndices[change.ProposalIndex] = struct{}{}
|
|
|
|
|
}
|
|
|
|
|
pipelineResult, pipelineErr := runModulePipeline(ctx, collectSectionProposalsInput{
|
|
|
|
|
Module: module,
|
|
|
|
|
Spec: spec,
|
|
|
|
|
Policy: policy,
|
|
|
|
|
Sections: sections,
|
|
|
|
|
ProposalClient: spec.ProposalClient,
|
|
|
|
|
Interactions: input.Interactions,
|
|
|
|
|
ValidationClient: spec.ValidationClient,
|
|
|
|
|
GlossaryVariables: spec.GlossaryVariables,
|
|
|
|
|
GlossaryView: spec.GlossaryView,
|
|
|
|
|
})
|
|
|
|
|
if input.CaptureCandidates {
|
|
|
|
|
candidates = append(candidates, pipelineResult.Candidates...)
|
|
|
|
|
}
|
|
|
|
|
if pipelineErr == nil && ctx.Err() != nil {
|
|
|
|
|
pipelineErr = ctx.Err()
|
|
|
|
|
for i := range pipelineResult.WorkUnits {
|
|
|
|
|
for j := range pipelineResult.WorkUnits[i].Repairs {
|
|
|
|
|
if pipelineResult.WorkUnits[i].Repairs[j].Outcome == "approved" {
|
|
|
|
|
pipelineResult.WorkUnits[i].Repairs[j].Outcome = "interrupted"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if pipelineErr != nil {
|
|
|
|
|
failed := ModuleResult{
|
|
|
|
|
StageIndex: spec.Index, StageID: spec.ID,
|
|
|
|
|
ModuleKey: spec.ModuleKey,
|
|
|
|
|
ModuleInstance: spec.InstanceName,
|
|
|
|
|
ReplacementPolicy: policy,
|
|
|
|
|
Status: ModuleStatusFailed,
|
|
|
|
|
ProposalCount: pipelineResult.ProposalCount,
|
|
|
|
|
Operations: pipelineResult.Operations,
|
|
|
|
|
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
|
|
|
|
ValidatorRejected: pipelineResult.ValidatorRejected,
|
|
|
|
|
ErrorMessage: pipelineErr.Error(),
|
|
|
|
|
}
|
|
|
|
|
results = append(results, failed)
|
|
|
|
|
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
|
|
|
|
|
for sectionPos := range sections {
|
|
|
|
|
workUnits[moduleIndex*len(sections)+sectionPos].ModuleIndex = moduleIndex
|
|
|
|
|
}
|
|
|
|
|
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord)
|
|
|
|
|
if applyErr != nil {
|
|
|
|
|
for i := range pipelineResult.WorkUnits {
|
|
|
|
|
unit := &pipelineResult.WorkUnits[i]
|
|
|
|
|
unit.ModuleIndex = moduleIndex
|
|
|
|
|
unit.State = outcomes.WorkInterrupted
|
|
|
|
|
for j := range unit.Repairs {
|
|
|
|
|
if unit.Repairs[j].Outcome == "approved" {
|
|
|
|
|
unit.Repairs[j].Outcome = "interrupted"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
|
|
|
|
|
results = append(results, ModuleResult{StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, ModuleInstance: spec.InstanceName, ReplacementPolicy: policy, Status: ModuleStatusFailed, ProposalCount: pipelineResult.ProposalCount, Operations: pipelineResult.Operations, ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorRejected: pipelineResult.ValidatorRejected, ErrorMessage: applyErr.Error()})
|
|
|
|
|
return snapshot(), fmt.Errorf("module %q application failed: %w", spec.InstanceName, applyErr)
|
|
|
|
|
}
|
|
|
|
|
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
|
|
|
|
|
for sectionPos := range sections {
|
|
|
|
|
unit := &workUnits[moduleIndex*len(sections)+sectionPos]
|
|
|
|
|
var operations []outcomes.Operation
|
|
|
|
|
for i := range units {
|
|
|
|
|
unit := &units[i]
|
|
|
|
|
unit.ModuleIndex = moduleIndex
|
|
|
|
|
unit.State = outcomes.WorkCompleted
|
|
|
|
|
unit.CorrectionsApplied = appliedBySection[sectionPos]
|
|
|
|
|
}
|
|
|
|
|
var appliedIndices map[int]struct{}
|
|
|
|
|
for i := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] {
|
|
|
|
|
unit := &workUnits[moduleIndex*len(sections)+i]
|
|
|
|
|
if err == nil {
|
|
|
|
|
unit.State = outcomes.WorkCompleted
|
|
|
|
|
unit.CorrectionsApplied = execution.appliedBySection[i]
|
|
|
|
|
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
|
|
|
|
|
unit.State = outcomes.WorkInterrupted
|
|
|
|
|
}
|
|
|
|
|
for j := range unit.Repairs {
|
|
|
|
|
branch := &unit.Repairs[j]
|
|
|
|
|
if branch.Outcome != "approved" {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if appliedIndices == nil {
|
|
|
|
|
appliedIndices = make(map[int]struct{}, len(applied))
|
|
|
|
|
for _, change := range applied {
|
|
|
|
|
appliedIndices[change.ProposalIndex] = struct{}{}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
branch.Outcome = "approved_skipped"
|
|
|
|
|
if _, ok := appliedIndices[branch.ProposalIndex]; ok {
|
|
|
|
|
if err != nil {
|
|
|
|
|
branch.Outcome = "interrupted"
|
|
|
|
|
} else if _, ok := appliedIndices[branch.ProposalIndex]; ok {
|
|
|
|
|
branch.Outcome = "applied"
|
|
|
|
|
} else {
|
|
|
|
|
branch.Outcome = "approved_skipped"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
attributeApplyFiltering(sections, pipelineResult.Approved, skipped, policy, workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)])
|
|
|
|
|
sections = updated
|
|
|
|
|
moduleOperations := make([]outcomes.Operation, 0)
|
|
|
|
|
for _, unit := range workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] {
|
|
|
|
|
moduleOperations = append(moduleOperations, unit.Operations...)
|
|
|
|
|
operations = append(operations, unit.Operations...)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
results = append(results, ModuleResult{
|
|
|
|
|
StageIndex: spec.Index, StageID: spec.ID,
|
|
|
|
|
ModuleKey: spec.ModuleKey,
|
|
|
|
|
ModuleInstance: spec.InstanceName,
|
|
|
|
|
ReplacementPolicy: policy,
|
|
|
|
|
Status: ModuleStatusSuccess,
|
|
|
|
|
ProposalCount: pipelineResult.ProposalCount,
|
|
|
|
|
Operations: moduleOperations,
|
|
|
|
|
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
|
|
|
|
ValidatorRejected: pipelineResult.ValidatorRejected,
|
|
|
|
|
AppliedChanges: applied,
|
|
|
|
|
SkippedChanges: skipped,
|
|
|
|
|
})
|
|
|
|
|
if input.CaptureCandidates {
|
|
|
|
|
candidates = append(candidates, execution.pipeline.Candidates...)
|
|
|
|
|
}
|
|
|
|
|
result := ModuleResult{
|
|
|
|
|
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
|
|
|
|
|
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
|
|
|
|
|
ProposalCount: execution.pipeline.ProposalCount, Operations: operations,
|
|
|
|
|
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
|
|
|
|
|
}
|
|
|
|
|
if err != nil {
|
|
|
|
|
result.Status = ModuleStatusFailed
|
|
|
|
|
result.ErrorMessage = err.Error()
|
|
|
|
|
} else {
|
|
|
|
|
result.Status = ModuleStatusSuccess
|
|
|
|
|
result.AppliedChanges = execution.applied
|
|
|
|
|
result.SkippedChanges = execution.skipped
|
|
|
|
|
}
|
|
|
|
|
results = append(results, result)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
output := snapshot()
|
|
|
|
|
return output, nil
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return snapshot(), &ExecutionFailure{Phase: "processing", Cause: err}
|
|
|
|
|
}
|
|
|
|
|
return snapshot(), nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type moduleExecution struct {
|
|
|
|
|
policy proposals.ReplacementPolicy
|
|
|
|
|
pipeline modulePipelineResult
|
|
|
|
|
updated []chunking.Section
|
|
|
|
|
applied []proposals.AppliedChange
|
|
|
|
|
skipped []proposals.SkippedChange
|
|
|
|
|
appliedBySection []int
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) {
|
|
|
|
|
var execution moduleExecution
|
|
|
|
|
origin := failureReference(spec, nil, "")
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return execution, locateFailure(err, "processing", origin, false, nil)
|
|
|
|
|
}
|
|
|
|
|
module, err := r.factory.ModuleForSpec(spec)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return execution, locateFailure(fmt.Errorf("setup failed: %w", err), "setup", origin, false, nil)
|
|
|
|
|
}
|
|
|
|
|
execution.policy = module.ReplacementPolicy()
|
|
|
|
|
if err := spec.PrepareGlossary(); err != nil {
|
|
|
|
|
return execution, locateFailure(fmt.Errorf("glossary preparation failed: %w", err), "setup", origin, false, nil)
|
|
|
|
|
}
|
|
|
|
|
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
|
|
|
|
|
Module: module,
|
|
|
|
|
Spec: spec,
|
|
|
|
|
Policy: execution.policy,
|
|
|
|
|
Sections: sections,
|
|
|
|
|
ProposalClient: spec.ProposalClient,
|
|
|
|
|
Interactions: interactions,
|
|
|
|
|
ValidationClient: spec.ValidationClient,
|
|
|
|
|
GlossaryVariables: spec.GlossaryVariables,
|
|
|
|
|
GlossaryView: spec.GlossaryView,
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
return execution, err
|
|
|
|
|
}
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return execution, locateFailure(err, "application", origin, false, nil)
|
|
|
|
|
}
|
|
|
|
|
execution.updated, execution.applied, execution.skipped, execution.appliedBySection, err = applyWithinSections(sections, execution.pipeline.Approved, execution.policy, spec.GlossaryView.Protection, spec.ModuleKey == modulecatalog.KeySpokenWord)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return execution, locateFailure(fmt.Errorf("application failed: %w", err), "application", origin, false, nil)
|
|
|
|
|
}
|
|
|
|
|
return execution, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
|
|
|
|
|
@@ -345,7 +338,7 @@ type sectionProposalResult struct {
|
|
|
|
|
section chunking.Section
|
|
|
|
|
proposal contracts.ProposalResult
|
|
|
|
|
started bool
|
|
|
|
|
err error
|
|
|
|
|
failure *ExecutionFailure
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type sectionValidationResult struct {
|
|
|
|
|
@@ -357,7 +350,7 @@ type sectionValidationResult struct {
|
|
|
|
|
rejected []ValidatorRejectedChange
|
|
|
|
|
operations []outcomes.Operation
|
|
|
|
|
filtered []outcomes.FilteredProposal
|
|
|
|
|
err error
|
|
|
|
|
failure *ExecutionFailure
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type modulePipelineResult struct {
|
|
|
|
|
@@ -370,9 +363,8 @@ type modulePipelineResult struct {
|
|
|
|
|
ValidatorRejected []ValidatorRejectedChange
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput) (context.Context, <-chan sectionProposalResult, context.CancelFunc) {
|
|
|
|
|
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
|
|
|
|
|
results := make(chan sectionProposalResult, len(input.Sections))
|
|
|
|
|
runCtx, cancel := context.WithCancel(ctx)
|
|
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
|
defer close(results)
|
|
|
|
|
@@ -383,12 +375,12 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
|
|
|
|
wg.Add(1)
|
|
|
|
|
go func() {
|
|
|
|
|
defer wg.Done()
|
|
|
|
|
if runCtx.Err() != nil {
|
|
|
|
|
if ctx.Err() != nil {
|
|
|
|
|
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
meta := contracts.SectionMetadataFromSection(section)
|
|
|
|
|
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
|
|
|
|
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
|
|
|
|
|
ExecutionContext: contracts.ExecutionContext{
|
|
|
|
|
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
|
|
|
|
Settings: input.Spec.Settings,
|
|
|
|
|
@@ -407,19 +399,24 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
|
|
|
|
},
|
|
|
|
|
LLMClient: input.ProposalClient,
|
|
|
|
|
})
|
|
|
|
|
var failure *ExecutionFailure
|
|
|
|
|
if err != nil {
|
|
|
|
|
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
|
|
|
|
|
fail(failure)
|
|
|
|
|
}
|
|
|
|
|
results <- sectionProposalResult{
|
|
|
|
|
sectionPos: sectionPos,
|
|
|
|
|
section: section,
|
|
|
|
|
proposal: proposalResult,
|
|
|
|
|
started: true,
|
|
|
|
|
err: err,
|
|
|
|
|
failure: failure,
|
|
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
}
|
|
|
|
|
wg.Wait()
|
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
return runCtx, results, cancel
|
|
|
|
|
return results
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
|
|
|
|
@@ -445,18 +442,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 {
|
|
|
|
|
provider, ok := input.Module.(interface{ RepairPromptID() string })
|
|
|
|
|
if !ok || input.Spec.RepairCoordinator == nil {
|
|
|
|
|
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"}
|
|
|
|
|
cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair coordinator or prompt provider is unavailable"}
|
|
|
|
|
return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil)
|
|
|
|
|
}
|
|
|
|
|
repairPromptID = provider.RepairPromptID()
|
|
|
|
|
if repairPromptID == "" {
|
|
|
|
|
return out, &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"}
|
|
|
|
|
cause := &structuredoutput.Failure{Class: outcomes.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "correction repair prompt is unavailable"}
|
|
|
|
|
return out, locateFailure(cause, "setup", failureReference(input.Spec, nil, ""), false, nil)
|
|
|
|
|
}
|
|
|
|
|
input.Spec.CaptureProposalExchange = true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
runCtx, proposalResults, cancel := collectSectionProposals(ctx, input)
|
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
|
|
pending := make(map[int]sectionProposalResult, len(input.Sections))
|
|
|
|
|
validationResults := make(chan sectionValidationResult, len(input.Sections))
|
|
|
|
|
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
|
|
|
|
|
@@ -466,26 +462,49 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
nextProposalIndex := 0
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
|
firstErr error
|
|
|
|
|
errMu sync.Mutex
|
|
|
|
|
vwg sync.WaitGroup
|
|
|
|
|
firstFailure *ExecutionFailure
|
|
|
|
|
errMu sync.Mutex
|
|
|
|
|
vwg sync.WaitGroup
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
setErr := func(err error) {
|
|
|
|
|
if err == nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
runCtx, cancel := context.WithCancel(ctx)
|
|
|
|
|
defer cancel()
|
|
|
|
|
setFailure := func(failure *ExecutionFailure) {
|
|
|
|
|
errMu.Lock()
|
|
|
|
|
defer errMu.Unlock()
|
|
|
|
|
if firstErr == nil {
|
|
|
|
|
firstErr = err
|
|
|
|
|
if firstFailure == nil {
|
|
|
|
|
firstFailure = failure
|
|
|
|
|
cancel()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
getErr := func() error {
|
|
|
|
|
errMu.Lock()
|
|
|
|
|
defer errMu.Unlock()
|
|
|
|
|
return firstErr
|
|
|
|
|
proposalResults := collectSectionProposals(runCtx, input, setFailure)
|
|
|
|
|
|
|
|
|
|
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
|
|
|
|
|
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
|
|
|
|
|
for i, corrected := range sectionResult.proposal.Proposals {
|
|
|
|
|
sectionIndex := index
|
|
|
|
|
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
|
|
|
|
CorrectionProposal: corrected,
|
|
|
|
|
ProposalMetadata: proposals.ProposalMetadata{
|
|
|
|
|
ProposalIndex: nextProposalIndex + i,
|
|
|
|
|
ModuleKey: input.Spec.ModuleKey,
|
|
|
|
|
ModuleInstance: input.Spec.InstanceName,
|
|
|
|
|
SectionIndex: §ionIndex,
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
nextProposalIndex += len(sectionEnriched)
|
|
|
|
|
out.ProposalCount += len(sectionEnriched)
|
|
|
|
|
out.Candidates = append(out.Candidates, sectionEnriched...)
|
|
|
|
|
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
|
|
|
|
|
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
|
|
|
|
|
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
|
|
|
|
for _, proposal := range sectionEnriched {
|
|
|
|
|
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return sectionEnriched
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for result := range proposalResults {
|
|
|
|
|
@@ -494,11 +513,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
unit.State = outcomes.WorkInterrupted
|
|
|
|
|
}
|
|
|
|
|
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
|
|
|
|
if result.err != nil {
|
|
|
|
|
setErr(result.err)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if getErr() != nil {
|
|
|
|
|
if result.failure != nil {
|
|
|
|
|
unit.Failure = result.failure
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if !result.started {
|
|
|
|
|
@@ -507,7 +523,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
pending[result.sectionPos] = result
|
|
|
|
|
|
|
|
|
|
for {
|
|
|
|
|
if getErr() != nil {
|
|
|
|
|
if runCtx.Err() != nil {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
sectionResult, ok := pending[nextSectionToProcess]
|
|
|
|
|
@@ -516,30 +532,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
}
|
|
|
|
|
delete(pending, nextSectionToProcess)
|
|
|
|
|
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
|
|
|
|
|
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
|
|
|
|
|
for i, corrected := range sectionResult.proposal.Proposals {
|
|
|
|
|
sectionIndex := sectionMeta.Index
|
|
|
|
|
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
|
|
|
|
CorrectionProposal: corrected,
|
|
|
|
|
ProposalMetadata: proposals.ProposalMetadata{
|
|
|
|
|
ProposalIndex: nextProposalIndex + i,
|
|
|
|
|
ModuleKey: input.Spec.ModuleKey,
|
|
|
|
|
ModuleInstance: input.Spec.InstanceName,
|
|
|
|
|
SectionIndex: §ionIndex,
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
nextProposalIndex += len(sectionEnriched)
|
|
|
|
|
out.ProposalCount += len(sectionEnriched)
|
|
|
|
|
out.Candidates = append(out.Candidates, sectionEnriched...)
|
|
|
|
|
for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations {
|
|
|
|
|
operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex]
|
|
|
|
|
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
|
|
|
|
for _, proposal := range sectionEnriched {
|
|
|
|
|
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
|
|
|
|
|
|
|
|
|
|
vwg.Add(1)
|
|
|
|
|
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
|
|
|
|
|
@@ -561,10 +554,18 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
}
|
|
|
|
|
validated, err := validateSectionCandidates(runCtx, validationInput)
|
|
|
|
|
var repairs []RepairOutcome
|
|
|
|
|
phase, kind := "validation", outcomes.OperationValidation
|
|
|
|
|
if err == nil && repairPromptID != "" {
|
|
|
|
|
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
|
|
|
|
|
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
|
|
|
|
|
}
|
|
|
|
|
setErr(err)
|
|
|
|
|
var failure *ExecutionFailure
|
|
|
|
|
if err != nil {
|
|
|
|
|
if !errors.As(err, &failure) {
|
|
|
|
|
failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations)
|
|
|
|
|
}
|
|
|
|
|
setFailure(failure)
|
|
|
|
|
}
|
|
|
|
|
validationResults <- sectionValidationResult{
|
|
|
|
|
repairs: repairs,
|
|
|
|
|
sectionPos: sectionPos,
|
|
|
|
|
@@ -574,7 +575,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
rejected: validated.rejected,
|
|
|
|
|
operations: validated.operations,
|
|
|
|
|
filtered: validated.filtered,
|
|
|
|
|
err: err,
|
|
|
|
|
failure: failure,
|
|
|
|
|
}
|
|
|
|
|
}(nextSectionToProcess, sectionEnriched, sectionMeta)
|
|
|
|
|
nextSectionToProcess++
|
|
|
|
|
@@ -584,11 +585,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
vwg.Wait()
|
|
|
|
|
close(validationResults)
|
|
|
|
|
|
|
|
|
|
// A failed or canceled module can still have usable proposals from later
|
|
|
|
|
// sections. Assign their indices in order after preserving the dispatched
|
|
|
|
|
// prefix, without submitting new validation work.
|
|
|
|
|
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
|
|
|
|
|
if result, ok := pending[pos]; ok {
|
|
|
|
|
collectCandidates(result, result.section.Index)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for v := range validationResults {
|
|
|
|
|
validationBySection[v.sectionPos] = v
|
|
|
|
|
if v.err != nil {
|
|
|
|
|
setErr(v.err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i := range input.Sections {
|
|
|
|
|
@@ -600,7 +607,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
unit.Operations = append(unit.Operations, res.operations...)
|
|
|
|
|
unit.Repairs = res.repairs
|
|
|
|
|
mergeFilteredProposals(unit, res.filtered)
|
|
|
|
|
if res.err == nil {
|
|
|
|
|
unit.Failure = res.failure
|
|
|
|
|
if res.failure == nil {
|
|
|
|
|
unit.State = outcomes.WorkCompleted
|
|
|
|
|
} else {
|
|
|
|
|
unit.State = outcomes.WorkInterrupted
|
|
|
|
|
@@ -625,11 +633,11 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
}
|
|
|
|
|
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
|
|
|
|
|
})
|
|
|
|
|
if ctx.Err() != nil {
|
|
|
|
|
setErr(ctx.Err())
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
setFailure(locateFailure(err, "processing", failureReference(input.Spec, nil, ""), false, nil))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if err := getErr(); err != nil {
|
|
|
|
|
if firstFailure != nil {
|
|
|
|
|
for i := range out.WorkUnits {
|
|
|
|
|
for j := range out.WorkUnits[i].Repairs {
|
|
|
|
|
if out.WorkUnits[i].Repairs[j].Outcome == "approved" {
|
|
|
|
|
@@ -637,7 +645,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return out, err
|
|
|
|
|
return out, firstFailure
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return out, nil
|
|
|
|
|
@@ -713,6 +721,13 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|
|
|
|
})
|
|
|
|
|
operations = append(operations, vResult.Operations...)
|
|
|
|
|
|
|
|
|
|
locate := func(cause error) *ExecutionFailure {
|
|
|
|
|
ref := failureReference(input.Spec, input.SectionIndex, outcomes.OperationValidation)
|
|
|
|
|
ref.ValidatorName, ref.ValidatorOrder = validator.Name(), &validatorOrder
|
|
|
|
|
ref.ProposalIndex, ref.RepairAttempt = input.RepairProposalIndex, input.RepairAttempt
|
|
|
|
|
return locateFailure(cause, "validation", ref, input.SectionIndex != nil, vResult.Operations)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var decisionErr error
|
|
|
|
|
if err != nil {
|
|
|
|
|
decisionErr = validators.EnforcePartialDecisionCardinality(submitted, vResult.Decisions)
|
|
|
|
|
@@ -722,7 +737,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|
|
|
|
if decisionErr != nil {
|
|
|
|
|
return validateSectionCandidatesResult{
|
|
|
|
|
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
|
|
|
|
|
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr))
|
|
|
|
|
}, locate(fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), errors.Join(err, decisionErr)))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(submitted))
|
|
|
|
|
@@ -737,7 +752,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|
|
|
|
if contractErr != nil {
|
|
|
|
|
return validateSectionCandidatesResult{
|
|
|
|
|
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
|
|
|
|
|
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
|
|
|
|
|
}, locate(fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr)))
|
|
|
|
|
}
|
|
|
|
|
dispositions[i] = disposition
|
|
|
|
|
}
|
|
|
|
|
@@ -789,7 +804,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|
|
|
|
// approval to candidates that have not finished the full chain.
|
|
|
|
|
return validateSectionCandidatesResult{
|
|
|
|
|
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
|
|
|
|
|
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
|
|
|
|
|
}, locate(fmt.Errorf("validator %q failed: %w", validator.Name(), err))
|
|
|
|
|
}
|
|
|
|
|
eligible = nextEligible
|
|
|
|
|
}
|
|
|
|
|
|