Bound concurrent section work and finalize proposal identities in order

This commit is contained in:
2026-11-16 14:53:14 +00:00
parent 1c43264f6f
commit b98d463958
11 changed files with 739 additions and 240 deletions

View File

@@ -15,8 +15,11 @@ formats; [configuration](../config.md) owns settings; the
estimates stay fixed while text evolves.
4. Module instances execute in configured order. Each section hands its decoded
proposals and original local segment references to the runner along with
operation facts and any captured exchange. Sections may propose concurrently.
The runner assigns indices in section order and response order.
operation facts and any captured exchange. A bounded window admits sections
in order and lets validation and repair proceed as proposals become ready.
The runner assigns final proposal indices in section and response order
when it records completed work. It then writes diagnostics that use those
indices. PromptKit controls provider capacity across all three roles.
5. Deterministic validators precede LLM-backed validators. Every submitted
candidate receives one decision per validator on a normal return. Fatal returns
may preserve a checked subset of completed decisions as diagnostic evidence.
@@ -43,8 +46,8 @@ complete with model errors and applied corrections. Recoverable model failures
never trigger shared cancellation. Fatal failures and parent cancellation stop
further dispatch and join active workers. The initiating error carries its
processing phase and available origin. Work-unit evidence preserves each peer's
interruption or error. Usable proposal results from later sections are indexed
in order after joining, even when the runner could not validate them. The
interruption or error. The runner drains active outcomes and assigns indices
to usable proposals from later sections even when it could not validate them. The
working transcript advances only at the module's successful commit boundary.
Earlier committed text remains available to an interrupted run, and transcript
delivery requires successful processing.

View File

@@ -589,6 +589,21 @@ pending, and identical index ordering for the same usable results under
different completion orders. Report representative pending-work measurements
without pinning a timing or worker-count threshold in the default suite.
**Complete.** The runner admits sections through a bounded rolling window,
drains active outcomes on cancellation, and lets validation and repair proceed
while earlier proposals are pending. It assigns final indices in section order
and publishes identity-sensitive artifacts through the interaction writer.
Tests checked native contention and overlap, stable ordering, and repair and
revalidation evidence after completion and cancellation. The full offline Go
suite and targeted race checks passed.
In three runs of a 36-section fixture with two shared provider slots and the
first proposal held for 250 ms, ordered dispatch completed seven later proposals
and no validations during the hold. With section-local identities, seven later
validations also reached the provider before release, with eight sections
charged to the window in both variants. Both variants completed 36 proposals
and 36 validations overall. The window of eight was retained for this workload.
## Stage 17 — Project typed report and ledger facts
Audit #22, #27 (ledger), #39, #41 (projection).

View File

@@ -59,7 +59,8 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
for _, warning := range writer.Warnings() {
warn(warning)
}
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot())))
observations := processreport.IndexLLMObservations(observer.Snapshot(), output.WorkUnits)
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observations)))
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
}
var failures []processreport.RunFailure

View File

@@ -5,6 +5,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
@@ -94,3 +95,30 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
Events []executionEvent `json:"events"`
}{llm.Summarize(snapshot), calls, events}
}
// IndexLLMObservations assigns the same proposal identities used by the ledger.
func IndexLLMObservations(snapshot llm.ObservationSnapshot, units []runner.WorkUnitOutcome) llm.ObservationSnapshot {
type sectionKey struct {
module string
section int
}
offsets := make(map[sectionKey]int)
for _, unit := range units {
if unit.ProposalIndexOffset != 0 {
offsets[sectionKey{unit.ModuleInstance, unit.Section.Index}] = unit.ProposalIndexOffset
}
}
index := func(context *structuredoutput.DiagnosticContext) {
if context.SectionIndex != nil && context.ProposalIndex != nil {
offset := offsets[sectionKey{context.ModuleInstance, *context.SectionIndex}]
context.ProposalIndex = structuredoutput.OffsetProposalIndex(context.ProposalIndex, offset)
}
}
for i := range snapshot.Calls {
index(&snapshot.Calls[i].Context)
}
for i := range snapshot.Events {
index(&snapshot.Events[i].Context)
}
return snapshot
}

View File

@@ -176,27 +176,26 @@ func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
origin := errors.New("earlier proposal failed")
laterCompleted := make(chan struct{})
laterValidationStarted := make(chan struct{})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var validations atomic.Int32
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) {
validations.Add(1)
return validators.Result{}, errors.New("validation should not be submitted")
contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
close(laterValidationStarted)
<-ctx.Done()
return validators.Result{}, ctx.Err()
}},
}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index == 0 {
select {
case <-laterCompleted:
case <-laterValidationStarted:
return contracts.ProposalResult{}, origin
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
close(laterCompleted)
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
SegmentReferences: []int{1},
@@ -212,8 +211,8 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
})
if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err)
if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err)
}
var failure *ExecutionFailure
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
@@ -222,7 +221,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
}
candidate := out.Candidates[0]
later := out.WorkUnits[1]
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted ||
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) ||
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
out.FinalTranscript.Segments[1].Text != "other text" {
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)

View File

@@ -190,18 +190,37 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
snapshot := *revision
record.Revision = &snapshot
}
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
var failure any
if err != nil {
failure = map[string]string{"error": err.Error()}
}
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
Name: name,
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
RequestPayload: completion.Prepared,
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
ErrorPayload: failure,
feedback := findings
record.Artifacts, _ = structuredoutput.WriteIndexedInteraction(input.Interactions, func(offset int, resolvePath func(string) string) structuredoutput.Interaction {
index := proposalIndex + offset
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, index, attemptNumber)
var failure any
if err != nil {
failure = map[string]string{"error": err.Error()}
}
var correction *proposals.EnrichedCorrectionProposal
if record.Revision != nil {
snapshot := *record.Revision
snapshot.ProposalIndex = index
correction = &snapshot
}
decisions := slices.Clone(validated.decisions)
for i := range decisions {
decisions[i].ProposalIndex += offset
decisions[i].DiagnosticArtifactPath = resolvePath(decisions[i].DiagnosticArtifactPath)
}
rejections := slices.Clone(validated.rejected)
for i := range rejections {
rejections[i].ProposalIndex += offset
}
return structuredoutput.Interaction{
Name: name,
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &index, RepairAttempt: &attemptNumber},
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": index, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": feedback},
RequestPayload: completion.Prepared,
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": correction, "decisions": decisions, "rejections": rejections},
ErrorPayload: failure,
}
})
branch.Attempts = append(branch.Attempts, record)
}

View File

@@ -56,17 +56,18 @@ type ModuleResult struct {
// WorkUnitOutcome keeps initial section identity while text evolves between
// module instances. Detailed candidate and decision records are debug evidence.
type WorkUnitOutcome struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleIndex int `json:"module_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
ProposalIndexOffset int `json:"-"`
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleIndex int `json:"module_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
}
type ValidatorDecisionRecord struct {
@@ -342,15 +343,21 @@ type sectionProposalResult struct {
}
type sectionValidationResult struct {
repairs []RepairOutcome
sectionPos int
enriched []proposals.EnrichedCorrectionProposal
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
failure *ExecutionFailure
interactions *sectionInteractions
repairs []RepairOutcome
sectionPos int
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
failure *ExecutionFailure
}
type sectionWork struct {
candidates []proposals.EnrichedCorrectionProposal
validation *sectionValidationResult
finished bool
}
type modulePipelineResult struct {
@@ -363,60 +370,42 @@ type modulePipelineResult struct {
ValidatorRejected []ValidatorRejectedChange
}
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
results := make(chan sectionProposalResult, len(input.Sections))
// moduleWorkWindow bounds active and downstream work across all roles in
// one module. PromptKit still owns admission to each provider backend.
const moduleWorkWindow = 8
go func() {
defer close(results)
var wg sync.WaitGroup
for sectionPos, section := range input.Sections {
sectionPos := sectionPos
section := section
wg.Add(1)
go func() {
defer wg.Done()
if ctx.Err() != nil {
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
return
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
InstanceName: input.Spec.InstanceName,
ReplacementPolicy: input.Policy,
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
},
LLMClient: input.ProposalClient,
})
var failure *ExecutionFailure
if err != nil {
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
fail(failure)
}
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
proposal: proposalResult,
started: true,
failure: failure,
}
}()
}
wg.Wait()
}()
return results
func proposeSection(ctx context.Context, input collectSectionProposalsInput, sectionPos int) sectionProposalResult {
section := input.Sections[sectionPos]
result := sectionProposalResult{sectionPos: sectionPos, section: section}
if ctx.Err() != nil {
return result
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
InstanceName: input.Spec.InstanceName,
ReplacementPolicy: input.Policy,
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
},
LLMClient: input.ProposalClient,
})
result.started = true
result.proposal = proposalResult
if err != nil {
result.failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
}
return result
}
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
@@ -453,23 +442,26 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
input.Spec.CaptureProposalExchange = true
}
pending := make(map[int]sectionProposalResult, len(input.Sections))
validationResults := make(chan sectionValidationResult, len(input.Sections))
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
pending := make(map[int]*sectionWork, moduleWorkWindow)
proposalResults := make(chan sectionProposalResult)
validationResults := make(chan sectionValidationResult)
validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators())
nextSectionToProcess := 0
nextSectionToRecord := 0
nextProposalIndex := 0
var (
firstFailure *ExecutionFailure
errMu sync.Mutex
vwg sync.WaitGroup
workers sync.WaitGroup
)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
setFailure := func(failure *ExecutionFailure) {
if failure == nil {
return
}
errMu.Lock()
defer errMu.Unlock()
if firstFailure == nil {
@@ -477,133 +469,93 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
cancel()
}
}
proposalResults := collectSectionProposals(runCtx, input, setFailure)
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
makeCandidates := func(sectionResult sectionProposalResult) []proposals.EnrichedCorrectionProposal {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := index
sectionIndex := sectionResult.section.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: nextProposalIndex + i,
ProposalIndex: i,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: &sectionIndex,
},
})
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range sectionEnriched {
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
}
}
}
return sectionEnriched
}
for result := range proposalResults {
unit := &out.WorkUnits[result.sectionPos]
if result.started {
unit.State = outcomes.WorkInterrupted
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
unit := &out.WorkUnits[sectionPos]
unit.ProposalIndexOffset = nextProposalIndex
for _, candidate := range enriched {
candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
}
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.failure != nil {
unit.Failure = result.failure
continue
}
if !result.started {
continue
}
pending[result.sectionPos] = result
for {
if runCtx.Err() != nil {
break
out.ProposalCount += len(enriched)
for operationIndex := range unit.Operations {
operation := &unit.Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range enriched {
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
}
}
sectionResult, ok := pending[nextSectionToProcess]
if !ok {
break
}
nextProposalIndex += len(enriched)
}
validate := func(sectionResult sectionProposalResult, enriched []proposals.EnrichedCorrectionProposal) sectionValidationResult {
sectionPos := sectionResult.sectionPos
sectionMetadata := contracts.SectionMetadataFromSection(sectionResult.section)
var interactions structuredoutput.InteractionSink
var capture *sectionInteractions
if input.Interactions != nil {
capture = &sectionInteractions{InteractionSink: input.Interactions}
interactions = capture
}
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
vwg.Add(1)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
setFailure(failure)
}
validationResults <- sectionValidationResult{
repairs: repairs,
sectionPos: sectionPos,
enriched: enriched,
approved: validated.approved,
decisions: validated.decisions,
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
failure: failure,
}
}(nextSectionToProcess, sectionEnriched, sectionMeta)
nextSectionToProcess++
}
return sectionValidationResult{
interactions: capture,
repairs: repairs,
sectionPos: sectionPos,
approved: validated.approved,
decisions: validated.decisions,
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
failure: failure,
}
}
vwg.Wait()
close(validationResults)
// A failed or canceled module can still have usable proposals from later
// sections. Assign their indices in order after preserving the dispatched
// prefix, without submitting new validation work.
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
if result, ok := pending[pos]; ok {
collectCandidates(result, result.section.Index)
}
}
for v := range validationResults {
validationBySection[v.sectionPos] = v
}
for i := range input.Sections {
res, ok := validationBySection[i]
if !ok {
continue
}
unit := &out.WorkUnits[i]
recordValidation := func(res sectionValidationResult) {
unit := &out.WorkUnits[res.sectionPos]
unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered)
@@ -617,6 +569,69 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
}
nextAdmission, active := 0, 0
for {
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
offset := nextProposalIndex
recordCandidates(nextSectionToRecord, work.candidates)
if work.validation != nil {
work.validation.assignProposalIndices(offset)
recordValidation(*work.validation)
}
delete(pending, nextSectionToRecord)
nextSectionToRecord++
}
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
sectionPos := nextAdmission
nextAdmission++
pending[sectionPos] = &sectionWork{}
active++
workers.Go(func() {
proposed := proposeSection(runCtx, input, sectionPos)
setFailure(proposed.failure)
proposalResults <- proposed
})
}
if active == 0 {
break
}
select {
case result := <-proposalResults:
active--
unit := &out.WorkUnits[result.sectionPos]
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Failure = result.failure
work := pending[result.sectionPos]
if result.started && result.failure == nil {
work.candidates = makeCandidates(result)
if runCtx.Err() == nil {
enriched := work.candidates
active++
workers.Go(func() {
validated := validate(result, enriched)
setFailure(validated.failure)
validationResults <- validated
})
continue
}
}
work.finished = true
case validated := <-validationResults:
active--
work := pending[validated.sectionPos]
work.validation = &validated
work.finished = true
}
}
workers.Wait()
sort.SliceStable(out.Approved, func(i, j int) bool {
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
})
for _, unit := range out.WorkUnits {
out.Operations = append(out.Operations, unit.Operations...)
}

View File

@@ -0,0 +1,111 @@
package runner
import (
"slices"
"strconv"
"strings"
"sync"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
// Section validation can finish before preceding proposal counts are known.
type sectionInteractions struct {
structuredoutput.InteractionSink
mu sync.Mutex
builders []structuredoutput.IndexedInteraction
written []structuredoutput.InteractionArtifacts
}
const pendingInteractionPrefix = "\x00interaction:"
func (s *sectionInteractions) WriteIndexedInteraction(build structuredoutput.IndexedInteraction) (structuredoutput.InteractionArtifacts, error) {
s.mu.Lock()
defer s.mu.Unlock()
receipt := pendingInteractionPrefix + strconv.Itoa(len(s.builders))
s.builders = append(s.builders, build)
return structuredoutput.InteractionArtifacts{ResponsePayloadPath: receipt}, nil
}
func (s *sectionInteractions) write(offset int) {
for _, build := range s.builders {
paths, _ := s.InteractionSink.WriteInteraction(build(offset, s.responsePath))
s.written = append(s.written, paths)
}
s.builders = nil
}
func (s *sectionInteractions) artifacts(paths structuredoutput.InteractionArtifacts) structuredoutput.InteractionArtifacts {
value, pending := strings.CutPrefix(paths.ResponsePayloadPath, pendingInteractionPrefix)
if !pending {
return paths
}
index, err := strconv.Atoi(value)
if err != nil || index < 0 || index >= len(s.written) {
return structuredoutput.InteractionArtifacts{}
}
return s.written[index]
}
func (s *sectionInteractions) responsePath(path string) string {
return s.artifacts(structuredoutput.InteractionArtifacts{ResponsePayloadPath: path}).ResponsePayloadPath
}
func (r *sectionValidationResult) assignProposalIndices(offset int) {
capture := r.interactions != nil
if capture {
r.interactions.write(offset)
}
for i := range r.approved {
r.approved[i].ProposalIndex += offset
}
for i := range r.rejected {
r.rejected[i].ProposalIndex += offset
}
assignDecisions := func(decisions []ValidatorDecisionRecord) {
for i := range decisions {
decisions[i].ProposalIndex += offset
if capture {
decisions[i].DiagnosticArtifactPath = r.interactions.responsePath(decisions[i].DiagnosticArtifactPath)
}
}
}
assignDecisions(r.decisions)
for i := range r.operations {
op := &r.operations[i]
op.Reference.ProposalIndex = structuredoutput.OffsetProposalIndex(op.Reference.ProposalIndex, offset)
op.ProposalIndices = slices.Clone(op.ProposalIndices)
for j := range op.ProposalIndices {
op.ProposalIndices[j] += offset
}
op.Filtered = slices.Clone(op.Filtered)
for j := range op.Filtered {
op.Filtered[j].ProposalIndex += offset
}
}
for i := range r.filtered {
r.filtered[i].ProposalIndex += offset
}
for i := range r.repairs {
repair := &r.repairs[i]
repair.ProposalIndex += offset
if capture {
repair.Original.ProposalIndex += offset
repair.Final.ProposalIndex += offset
}
for j := range repair.Attempts {
attempt := &repair.Attempts[j]
if attempt.Revision != nil {
attempt.Revision.ProposalIndex += offset
}
assignDecisions(attempt.Decisions)
if capture {
attempt.Artifacts = r.interactions.artifacts(attempt.Artifacts)
}
}
}
if r.failure != nil && r.failure.Origin != nil {
r.failure.Origin.ProposalIndex = structuredoutput.OffsetProposalIndex(r.failure.Origin.ProposalIndex, offset)
}
r.interactions = nil
}

View File

@@ -0,0 +1,275 @@
package runner
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
ck "go.maximumdirect.net/configkit"
)
type sharedBackendModule struct {
*grammar.Module
validator validators.Validator
headEntered <-chan struct{}
}
func (m sharedBackendModule) Validators() []validators.Validator {
return []validators.Validator{m.validator}
}
func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index > 0 {
select {
case <-m.headEntered:
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
return m.Module.Propose(ctx, req)
}
func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
dir := t.TempDir()
path := filepath.Join(dir, "config.yml")
if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n"
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "")
if err != nil {
t.Fatal(err)
}
release := make(chan struct{})
entered := make(chan struct{})
validationStarted := make(chan struct{}, 1)
var other, validating, active, peak atomic.Int32
client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) {
body, err := io.ReadAll(r.Body)
if err != nil {
return nil, err
}
n := active.Add(1)
for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
}
defer active.Add(-1)
output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}`
switch {
case bytes.Contains(body, []byte(`"validations"`)):
validating.Add(1)
select {
case validationStarted <- struct{}{}:
default:
}
output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
case bytes.Contains(body, []byte("marker-000")):
close(entered)
select {
case <-release:
case <-r.Context().Done():
return nil, r.Context().Err()
}
default:
other.Add(1)
}
content, _ := json.Marshal(output)
wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil
})}
runtime, err := llm.NewRuntime(ctx, loaded, client)
if err != nil {
t.Fatal(err)
}
role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{}
for i := 0; i < 36; i++ {
transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)})
}
type result struct {
out RunOutput
err error
}
finished := make(chan result, 1)
go func() {
out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}},
})
finished <- result{out, err}
}()
select {
case <-entered:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
select {
case <-validationStarted:
case <-ctx.Done():
t.Fatal("later validation did not reach the backend while the earlier proposal held a slot")
}
close(release)
completed := <-finished
if completed.err != nil {
t.Fatal(completed.err)
}
if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 {
t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load())
}
changes := completed.out.ModuleResults[0].AppliedChanges
if len(changes) != len(transcript.Segments) {
t.Fatalf("got %d applied changes", len(changes))
}
for i, change := range changes {
if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID {
t.Fatalf("change %d: %+v", i, change)
}
}
}
func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
for _, canceled := range []bool{false, true} {
name := "completed"
if canceled {
name = "canceled"
}
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
laterRevalidated := make(chan struct{})
proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
section := *req.Context.SectionIndex
if section == 0 {
select {
case <-laterRevalidated:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
text := []string{"alpha", "bravo"}[section]
dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{
SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text),
}}
return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil
})
validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
section := *req.Context.SectionIndex
first := section == 0
revision := req.Context.RepairAttempt != nil
if first && canceled {
cancel()
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
CorrectionIndex: "1", Approved: first || revision, Reason: "checked",
}}
if revision {
close(laterRevalidated)
}
return contracts.StructuredCompletionResponse{}, nil
})
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator}
// The module's transport gates are supplied by the scripted calls above.
started := make(chan struct{})
close(started)
module.headEntered = started
repair := &repairClient{t: t, outputs: []string{
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
}}
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}},
}},
})
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
t.Fatal(err)
}
later := out.WorkUnits[1]
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 {
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
}
attempt := later.Repairs[0].Attempts[0]
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
t.Fatalf("lost ordered repair evidence: %+v", attempt)
}
var repaired struct {
Decoded proposal_generation.RepairResponse
Revision *proposals.EnrichedCorrectionProposal
Decisions []ValidatorDecisionRecord
}
readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired)
if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 ||
len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 ||
repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath ||
!strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") {
t.Fatalf("repair artifact uses different identities: %+v", repaired)
}
var checked struct {
Decoded validators.LLMValidationResponse
Reconciliation struct{ Decisions []validators.Decision }
}
readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked)
if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" ||
len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 {
t.Fatalf("validation evidence uses different identities: %+v", checked)
}
})
}
}
func readInteraction(t *testing.T, path string, result any) {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(raw, result); err != nil {
t.Fatal(err)
}
}

View File

@@ -22,3 +22,24 @@ type InteractionArtifacts struct {
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
}
// IndexedInteraction builds an artifact after the section's ordered proposal
// indices and paths from earlier interactions are available.
type IndexedInteraction func(offset int, resolvePath func(string) string) Interaction
func WriteIndexedInteraction(sink InteractionSink, build IndexedInteraction) (InteractionArtifacts, error) {
if deferred, ok := sink.(interface {
WriteIndexedInteraction(IndexedInteraction) (InteractionArtifacts, error)
}); ok {
return deferred.WriteIndexedInteraction(build)
}
return sink.WriteInteraction(build(0, func(path string) string { return path }))
}
func OffsetProposalIndex(index *int, offset int) *int {
if index == nil || offset == 0 {
return index
}
next := *index + offset
return &next
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
@@ -141,33 +142,44 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if req.Interactions == nil {
return structuredoutput.InteractionArtifacts{}
}
stageInstance := req.ModuleInstance
if req.SectionIndex != nil {
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
}
if req.RepairProposalIndex != nil && req.RepairAttempt != nil {
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt)
}
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
if reconciled != nil {
responsePayload["reconciliation"] = *reconciled
}
artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{
Name: stage, Context: coordinates, RequestMetadata: map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt,
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
RequestPayload: completion.Prepared,
ResponsePayload: responsePayload,
ErrorPayload: errPayload(callErr),
artifacts, _ := structuredoutput.WriteIndexedInteraction(req.Interactions, func(offset int, _ func(string) string) structuredoutput.Interaction {
coordinates := coordinates
coordinates.ProposalIndex = structuredoutput.OffsetProposalIndex(req.RepairProposalIndex, offset)
stageInstance := req.ModuleInstance
if req.SectionIndex != nil {
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
}
if coordinates.ProposalIndex != nil && req.RepairAttempt != nil {
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *coordinates.ProposalIndex, *req.RepairAttempt)
}
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
if reconciled != nil {
view := *reconciled
view.Decisions = slices.Clone(view.Decisions)
for i := range view.Decisions {
view.Decisions[i].ProposalIndex += offset
view.Decisions[i].DiagnosticArtifactPath = ""
}
view.UnavailableIndices = slices.Clone(view.UnavailableIndices)
for i := range view.UnavailableIndices {
view.UnavailableIndices[i] += offset
}
responsePayload["reconciliation"] = view
}
indices := batchProposalIndices(batch.Items)
for i := range indices {
indices[i] += offset
}
return structuredoutput.Interaction{
Name: stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex), Context: coordinates, RequestMetadata: map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
"validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex,
"proposal_indices": indices, "repair_proposal_index": coordinates.ProposalIndex, "repair_attempt": req.RepairAttempt,
"prompt": completion.Prompt, "response_schema": responseSchema.DiagnosticsMap(),
},
RequestPayload: completion.Prepared, ResponsePayload: responsePayload, ErrorPayload: errPayload(callErr),
}
})
return artifacts
}