Bound concurrent section work and finalize proposal identities in order

This commit is contained in:
2026-11-16 14:53:14 +00:00
parent 1c43264f6f
commit b98d463958
11 changed files with 739 additions and 240 deletions

View File

@@ -176,27 +176,26 @@ func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
origin := errors.New("earlier proposal failed")
laterCompleted := make(chan struct{})
laterValidationStarted := make(chan struct{})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var validations atomic.Int32
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) {
validations.Add(1)
return validators.Result{}, errors.New("validation should not be submitted")
contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
close(laterValidationStarted)
<-ctx.Done()
return validators.Result{}, ctx.Err()
}},
}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index == 0 {
select {
case <-laterCompleted:
case <-laterValidationStarted:
return contracts.ProposalResult{}, origin
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
close(laterCompleted)
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
SegmentReferences: []int{1},
@@ -212,8 +211,8 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
})
if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err)
if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err)
}
var failure *ExecutionFailure
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
@@ -222,7 +221,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
}
candidate := out.Candidates[0]
later := out.WorkUnits[1]
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted ||
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) ||
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
out.FinalTranscript.Segments[1].Text != "other text" {
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)

View File

@@ -190,18 +190,37 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
snapshot := *revision
record.Revision = &snapshot
}
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
var failure any
if err != nil {
failure = map[string]string{"error": err.Error()}
}
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
Name: name,
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
RequestPayload: completion.Prepared,
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
ErrorPayload: failure,
feedback := findings
record.Artifacts, _ = structuredoutput.WriteIndexedInteraction(input.Interactions, func(offset int, resolvePath func(string) string) structuredoutput.Interaction {
index := proposalIndex + offset
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, index, attemptNumber)
var failure any
if err != nil {
failure = map[string]string{"error": err.Error()}
}
var correction *proposals.EnrichedCorrectionProposal
if record.Revision != nil {
snapshot := *record.Revision
snapshot.ProposalIndex = index
correction = &snapshot
}
decisions := slices.Clone(validated.decisions)
for i := range decisions {
decisions[i].ProposalIndex += offset
decisions[i].DiagnosticArtifactPath = resolvePath(decisions[i].DiagnosticArtifactPath)
}
rejections := slices.Clone(validated.rejected)
for i := range rejections {
rejections[i].ProposalIndex += offset
}
return structuredoutput.Interaction{
Name: name,
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &index, RepairAttempt: &attemptNumber},
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": index, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": feedback},
RequestPayload: completion.Prepared,
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": correction, "decisions": decisions, "rejections": rejections},
ErrorPayload: failure,
}
})
branch.Attempts = append(branch.Attempts, record)
}

View File

@@ -56,17 +56,18 @@ type ModuleResult struct {
// WorkUnitOutcome keeps initial section identity while text evolves between
// module instances. Detailed candidate and decision records are debug evidence.
type WorkUnitOutcome struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleIndex int `json:"module_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
ProposalIndexOffset int `json:"-"`
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id,omitempty"`
ModuleIndex int `json:"module_index"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
Section contracts.SectionMetadata `json:"section"`
State outcomes.WorkState `json:"state"`
Failure *ExecutionFailure `json:"-"`
CorrectionsApplied int `json:"corrections_applied"`
Repairs []RepairOutcome `json:"-"`
Operations []outcomes.Operation `json:"operations,omitempty"`
}
type ValidatorDecisionRecord struct {
@@ -342,15 +343,21 @@ type sectionProposalResult struct {
}
type sectionValidationResult struct {
repairs []RepairOutcome
sectionPos int
enriched []proposals.EnrichedCorrectionProposal
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
failure *ExecutionFailure
interactions *sectionInteractions
repairs []RepairOutcome
sectionPos int
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
operations []outcomes.Operation
filtered []outcomes.FilteredProposal
failure *ExecutionFailure
}
type sectionWork struct {
candidates []proposals.EnrichedCorrectionProposal
validation *sectionValidationResult
finished bool
}
type modulePipelineResult struct {
@@ -363,60 +370,42 @@ type modulePipelineResult struct {
ValidatorRejected []ValidatorRejectedChange
}
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
results := make(chan sectionProposalResult, len(input.Sections))
// moduleWorkWindow bounds active and downstream work across all roles in
// one module. PromptKit still owns admission to each provider backend.
const moduleWorkWindow = 8
go func() {
defer close(results)
var wg sync.WaitGroup
for sectionPos, section := range input.Sections {
sectionPos := sectionPos
section := section
wg.Add(1)
go func() {
defer wg.Done()
if ctx.Err() != nil {
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
return
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
InstanceName: input.Spec.InstanceName,
ReplacementPolicy: input.Policy,
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
},
LLMClient: input.ProposalClient,
})
var failure *ExecutionFailure
if err != nil {
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
fail(failure)
}
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
proposal: proposalResult,
started: true,
failure: failure,
}
}()
}
wg.Wait()
}()
return results
func proposeSection(ctx context.Context, input collectSectionProposalsInput, sectionPos int) sectionProposalResult {
section := input.Sections[sectionPos]
result := sectionProposalResult{sectionPos: sectionPos, section: section}
if ctx.Err() != nil {
return result
}
meta := contracts.SectionMetadataFromSection(section)
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
Settings: input.Spec.Settings,
WorkingTranscript: transcriptFromSection(section),
Glossary: input.Spec.Glossary,
ProtectedTerms: input.Spec.ProtectedTerms,
GlossaryVariables: input.GlossaryVariables,
Section: &meta,
Interactions: input.Interactions,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: input.Spec.ModuleKey,
InstanceName: input.Spec.InstanceName,
ReplacementPolicy: input.Policy,
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
},
LLMClient: input.ProposalClient,
})
result.started = true
result.proposal = proposalResult
if err != nil {
result.failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
}
return result
}
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
@@ -453,23 +442,26 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
input.Spec.CaptureProposalExchange = true
}
pending := make(map[int]sectionProposalResult, len(input.Sections))
validationResults := make(chan sectionValidationResult, len(input.Sections))
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
pending := make(map[int]*sectionWork, moduleWorkWindow)
proposalResults := make(chan sectionProposalResult)
validationResults := make(chan sectionValidationResult)
validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators())
nextSectionToProcess := 0
nextSectionToRecord := 0
nextProposalIndex := 0
var (
firstFailure *ExecutionFailure
errMu sync.Mutex
vwg sync.WaitGroup
workers sync.WaitGroup
)
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
setFailure := func(failure *ExecutionFailure) {
if failure == nil {
return
}
errMu.Lock()
defer errMu.Unlock()
if firstFailure == nil {
@@ -477,133 +469,93 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
cancel()
}
}
proposalResults := collectSectionProposals(runCtx, input, setFailure)
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
makeCandidates := func(sectionResult sectionProposalResult) []proposals.EnrichedCorrectionProposal {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := index
sectionIndex := sectionResult.section.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: nextProposalIndex + i,
ProposalIndex: i,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: &sectionIndex,
},
})
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range sectionEnriched {
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
}
}
}
return sectionEnriched
}
for result := range proposalResults {
unit := &out.WorkUnits[result.sectionPos]
if result.started {
unit.State = outcomes.WorkInterrupted
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
unit := &out.WorkUnits[sectionPos]
unit.ProposalIndexOffset = nextProposalIndex
for _, candidate := range enriched {
candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
}
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.failure != nil {
unit.Failure = result.failure
continue
}
if !result.started {
continue
}
pending[result.sectionPos] = result
for {
if runCtx.Err() != nil {
break
out.ProposalCount += len(enriched)
for operationIndex := range unit.Operations {
operation := &unit.Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range enriched {
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
}
}
sectionResult, ok := pending[nextSectionToProcess]
if !ok {
break
}
nextProposalIndex += len(enriched)
}
validate := func(sectionResult sectionProposalResult, enriched []proposals.EnrichedCorrectionProposal) sectionValidationResult {
sectionPos := sectionResult.sectionPos
sectionMetadata := contracts.SectionMetadataFromSection(sectionResult.section)
var interactions structuredoutput.InteractionSink
var capture *sectionInteractions
if input.Interactions != nil {
capture = &sectionInteractions{InteractionSink: input.Interactions}
interactions = capture
}
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
vwg.Add(1)
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
defer vwg.Done()
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
}
validated, err := validateSectionCandidates(runCtx, validationInput)
var repairs []RepairOutcome
phase, kind := "validation", outcomes.OperationValidation
if err == nil && repairPromptID != "" {
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
}
var failure *ExecutionFailure
if err != nil {
if !errors.As(err, &failure) {
failure = locateFailure(err, phase, failureReference(input.Spec, &sectionMetadata.Index, kind), true, validated.operations)
}
setFailure(failure)
}
validationResults <- sectionValidationResult{
repairs: repairs,
sectionPos: sectionPos,
enriched: enriched,
approved: validated.approved,
decisions: validated.decisions,
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
failure: failure,
}
}(nextSectionToProcess, sectionEnriched, sectionMeta)
nextSectionToProcess++
}
return sectionValidationResult{
interactions: capture,
repairs: repairs,
sectionPos: sectionPos,
approved: validated.approved,
decisions: validated.decisions,
rejected: validated.rejected,
operations: validated.operations,
filtered: validated.filtered,
failure: failure,
}
}
vwg.Wait()
close(validationResults)
// A failed or canceled module can still have usable proposals from later
// sections. Assign their indices in order after preserving the dispatched
// prefix, without submitting new validation work.
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
if result, ok := pending[pos]; ok {
collectCandidates(result, result.section.Index)
}
}
for v := range validationResults {
validationBySection[v.sectionPos] = v
}
for i := range input.Sections {
res, ok := validationBySection[i]
if !ok {
continue
}
unit := &out.WorkUnits[i]
recordValidation := func(res sectionValidationResult) {
unit := &out.WorkUnits[res.sectionPos]
unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered)
@@ -617,6 +569,69 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
}
nextAdmission, active := 0, 0
for {
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
offset := nextProposalIndex
recordCandidates(nextSectionToRecord, work.candidates)
if work.validation != nil {
work.validation.assignProposalIndices(offset)
recordValidation(*work.validation)
}
delete(pending, nextSectionToRecord)
nextSectionToRecord++
}
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
sectionPos := nextAdmission
nextAdmission++
pending[sectionPos] = &sectionWork{}
active++
workers.Go(func() {
proposed := proposeSection(runCtx, input, sectionPos)
setFailure(proposed.failure)
proposalResults <- proposed
})
}
if active == 0 {
break
}
select {
case result := <-proposalResults:
active--
unit := &out.WorkUnits[result.sectionPos]
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Failure = result.failure
work := pending[result.sectionPos]
if result.started && result.failure == nil {
work.candidates = makeCandidates(result)
if runCtx.Err() == nil {
enriched := work.candidates
active++
workers.Go(func() {
validated := validate(result, enriched)
setFailure(validated.failure)
validationResults <- validated
})
continue
}
}
work.finished = true
case validated := <-validationResults:
active--
work := pending[validated.sectionPos]
work.validation = &validated
work.finished = true
}
}
workers.Wait()
sort.SliceStable(out.Approved, func(i, j int) bool {
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
})
for _, unit := range out.WorkUnits {
out.Operations = append(out.Operations, unit.Operations...)
}

View File

@@ -0,0 +1,111 @@
package runner
import (
"slices"
"strconv"
"strings"
"sync"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
// Section validation can finish before preceding proposal counts are known.
type sectionInteractions struct {
structuredoutput.InteractionSink
mu sync.Mutex
builders []structuredoutput.IndexedInteraction
written []structuredoutput.InteractionArtifacts
}
const pendingInteractionPrefix = "\x00interaction:"
func (s *sectionInteractions) WriteIndexedInteraction(build structuredoutput.IndexedInteraction) (structuredoutput.InteractionArtifacts, error) {
s.mu.Lock()
defer s.mu.Unlock()
receipt := pendingInteractionPrefix + strconv.Itoa(len(s.builders))
s.builders = append(s.builders, build)
return structuredoutput.InteractionArtifacts{ResponsePayloadPath: receipt}, nil
}
func (s *sectionInteractions) write(offset int) {
for _, build := range s.builders {
paths, _ := s.InteractionSink.WriteInteraction(build(offset, s.responsePath))
s.written = append(s.written, paths)
}
s.builders = nil
}
func (s *sectionInteractions) artifacts(paths structuredoutput.InteractionArtifacts) structuredoutput.InteractionArtifacts {
value, pending := strings.CutPrefix(paths.ResponsePayloadPath, pendingInteractionPrefix)
if !pending {
return paths
}
index, err := strconv.Atoi(value)
if err != nil || index < 0 || index >= len(s.written) {
return structuredoutput.InteractionArtifacts{}
}
return s.written[index]
}
func (s *sectionInteractions) responsePath(path string) string {
return s.artifacts(structuredoutput.InteractionArtifacts{ResponsePayloadPath: path}).ResponsePayloadPath
}
func (r *sectionValidationResult) assignProposalIndices(offset int) {
capture := r.interactions != nil
if capture {
r.interactions.write(offset)
}
for i := range r.approved {
r.approved[i].ProposalIndex += offset
}
for i := range r.rejected {
r.rejected[i].ProposalIndex += offset
}
assignDecisions := func(decisions []ValidatorDecisionRecord) {
for i := range decisions {
decisions[i].ProposalIndex += offset
if capture {
decisions[i].DiagnosticArtifactPath = r.interactions.responsePath(decisions[i].DiagnosticArtifactPath)
}
}
}
assignDecisions(r.decisions)
for i := range r.operations {
op := &r.operations[i]
op.Reference.ProposalIndex = structuredoutput.OffsetProposalIndex(op.Reference.ProposalIndex, offset)
op.ProposalIndices = slices.Clone(op.ProposalIndices)
for j := range op.ProposalIndices {
op.ProposalIndices[j] += offset
}
op.Filtered = slices.Clone(op.Filtered)
for j := range op.Filtered {
op.Filtered[j].ProposalIndex += offset
}
}
for i := range r.filtered {
r.filtered[i].ProposalIndex += offset
}
for i := range r.repairs {
repair := &r.repairs[i]
repair.ProposalIndex += offset
if capture {
repair.Original.ProposalIndex += offset
repair.Final.ProposalIndex += offset
}
for j := range repair.Attempts {
attempt := &repair.Attempts[j]
if attempt.Revision != nil {
attempt.Revision.ProposalIndex += offset
}
assignDecisions(attempt.Decisions)
if capture {
attempt.Artifacts = r.interactions.artifacts(attempt.Artifacts)
}
}
}
if r.failure != nil && r.failure.Origin != nil {
r.failure.Origin.ProposalIndex = structuredoutput.OffsetProposalIndex(r.failure.Origin.ProposalIndex, offset)
}
r.interactions = nil
}

View File

@@ -0,0 +1,275 @@
package runner
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
ck "go.maximumdirect.net/configkit"
)
type sharedBackendModule struct {
*grammar.Module
validator validators.Validator
headEntered <-chan struct{}
}
func (m sharedBackendModule) Validators() []validators.Validator {
return []validators.Validator{m.validator}
}
func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if req.Section.Index > 0 {
select {
case <-m.headEntered:
case <-ctx.Done():
return contracts.ProposalResult{}, ctx.Err()
}
}
return m.Module.Propose(ctx, req)
}
func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
dir := t.TempDir()
path := filepath.Join(dir, "config.yml")
if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n"
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "")
if err != nil {
t.Fatal(err)
}
release := make(chan struct{})
entered := make(chan struct{})
validationStarted := make(chan struct{}, 1)
var other, validating, active, peak atomic.Int32
client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) {
body, err := io.ReadAll(r.Body)
if err != nil {
return nil, err
}
n := active.Add(1)
for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
}
defer active.Add(-1)
output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}`
switch {
case bytes.Contains(body, []byte(`"validations"`)):
validating.Add(1)
select {
case validationStarted <- struct{}{}:
default:
}
output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
case bytes.Contains(body, []byte("marker-000")):
close(entered)
select {
case <-release:
case <-r.Context().Done():
return nil, r.Context().Err()
}
default:
other.Add(1)
}
content, _ := json.Marshal(output)
wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil
})}
runtime, err := llm.NewRuntime(ctx, loaded, client)
if err != nil {
t.Fatal(err)
}
role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{}
for i := 0; i < 36; i++ {
transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)})
}
type result struct {
out RunOutput
err error
}
finished := make(chan result, 1)
go func() {
out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}},
})
finished <- result{out, err}
}()
select {
case <-entered:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
select {
case <-validationStarted:
case <-ctx.Done():
t.Fatal("later validation did not reach the backend while the earlier proposal held a slot")
}
close(release)
completed := <-finished
if completed.err != nil {
t.Fatal(completed.err)
}
if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 {
t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load())
}
changes := completed.out.ModuleResults[0].AppliedChanges
if len(changes) != len(transcript.Segments) {
t.Fatalf("got %d applied changes", len(changes))
}
for i, change := range changes {
if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID {
t.Fatalf("change %d: %+v", i, change)
}
}
}
func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
for _, canceled := range []bool{false, true} {
name := "completed"
if canceled {
name = "canceled"
}
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
laterRevalidated := make(chan struct{})
proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
section := *req.Context.SectionIndex
if section == 0 {
select {
case <-laterRevalidated:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
text := []string{"alpha", "bravo"}[section]
dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{
SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text),
}}
return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil
})
validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
section := *req.Context.SectionIndex
first := section == 0
revision := req.Context.RepairAttempt != nil
if first && canceled {
cancel()
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
CorrectionIndex: "1", Approved: first || revision, Reason: "checked",
}}
if revision {
close(laterRevalidated)
}
return contracts.StructuredCompletionResponse{}, nil
})
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator}
// The module's transport gates are supplied by the scripted calls above.
started := make(chan struct{})
close(started)
module.headEntered = started
repair := &repairClient{t: t, outputs: []string{
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
}}
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}},
}},
})
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
t.Fatal(err)
}
later := out.WorkUnits[1]
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 {
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
}
attempt := later.Repairs[0].Attempts[0]
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
t.Fatalf("lost ordered repair evidence: %+v", attempt)
}
var repaired struct {
Decoded proposal_generation.RepairResponse
Revision *proposals.EnrichedCorrectionProposal
Decisions []ValidatorDecisionRecord
}
readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired)
if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 ||
len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 ||
repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath ||
!strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") {
t.Fatalf("repair artifact uses different identities: %+v", repaired)
}
var checked struct {
Decoded validators.LLMValidationResponse
Reconciliation struct{ Decisions []validators.Decision }
}
readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked)
if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" ||
len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 {
t.Fatalf("validation evidence uses different identities: %+v", checked)
}
})
}
}
func readInteraction(t *testing.T, path string, result any) {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(raw, result); err != nil {
t.Fatal(err)
}
}