Bound concurrent section work and finalize proposal identities in order
This commit is contained in:
@@ -15,8 +15,11 @@ formats; [configuration](../config.md) owns settings; the
|
|||||||
estimates stay fixed while text evolves.
|
estimates stay fixed while text evolves.
|
||||||
4. Module instances execute in configured order. Each section hands its decoded
|
4. Module instances execute in configured order. Each section hands its decoded
|
||||||
proposals and original local segment references to the runner along with
|
proposals and original local segment references to the runner along with
|
||||||
operation facts and any captured exchange. Sections may propose concurrently.
|
operation facts and any captured exchange. A bounded window admits sections
|
||||||
The runner assigns indices in section order and response order.
|
in order and lets validation and repair proceed as proposals become ready.
|
||||||
|
The runner assigns final proposal indices in section and response order
|
||||||
|
when it records completed work. It then writes diagnostics that use those
|
||||||
|
indices. PromptKit controls provider capacity across all three roles.
|
||||||
5. Deterministic validators precede LLM-backed validators. Every submitted
|
5. Deterministic validators precede LLM-backed validators. Every submitted
|
||||||
candidate receives one decision per validator on a normal return. Fatal returns
|
candidate receives one decision per validator on a normal return. Fatal returns
|
||||||
may preserve a checked subset of completed decisions as diagnostic evidence.
|
may preserve a checked subset of completed decisions as diagnostic evidence.
|
||||||
@@ -43,8 +46,8 @@ complete with model errors and applied corrections. Recoverable model failures
|
|||||||
never trigger shared cancellation. Fatal failures and parent cancellation stop
|
never trigger shared cancellation. Fatal failures and parent cancellation stop
|
||||||
further dispatch and join active workers. The initiating error carries its
|
further dispatch and join active workers. The initiating error carries its
|
||||||
processing phase and available origin. Work-unit evidence preserves each peer's
|
processing phase and available origin. Work-unit evidence preserves each peer's
|
||||||
interruption or error. Usable proposal results from later sections are indexed
|
interruption or error. The runner drains active outcomes and assigns indices
|
||||||
in order after joining, even when the runner could not validate them. The
|
to usable proposals from later sections even when it could not validate them. The
|
||||||
working transcript advances only at the module's successful commit boundary.
|
working transcript advances only at the module's successful commit boundary.
|
||||||
Earlier committed text remains available to an interrupted run, and transcript
|
Earlier committed text remains available to an interrupted run, and transcript
|
||||||
delivery requires successful processing.
|
delivery requires successful processing.
|
||||||
|
|||||||
@@ -589,6 +589,21 @@ pending, and identical index ordering for the same usable results under
|
|||||||
different completion orders. Report representative pending-work measurements
|
different completion orders. Report representative pending-work measurements
|
||||||
without pinning a timing or worker-count threshold in the default suite.
|
without pinning a timing or worker-count threshold in the default suite.
|
||||||
|
|
||||||
|
**Complete.** The runner admits sections through a bounded rolling window,
|
||||||
|
drains active outcomes on cancellation, and lets validation and repair proceed
|
||||||
|
while earlier proposals are pending. It assigns final indices in section order
|
||||||
|
and publishes identity-sensitive artifacts through the interaction writer.
|
||||||
|
Tests checked native contention and overlap, stable ordering, and repair and
|
||||||
|
revalidation evidence after completion and cancellation. The full offline Go
|
||||||
|
suite and targeted race checks passed.
|
||||||
|
|
||||||
|
In three runs of a 36-section fixture with two shared provider slots and the
|
||||||
|
first proposal held for 250 ms, ordered dispatch completed seven later proposals
|
||||||
|
and no validations during the hold. With section-local identities, seven later
|
||||||
|
validations also reached the provider before release, with eight sections
|
||||||
|
charged to the window in both variants. Both variants completed 36 proposals
|
||||||
|
and 36 validations overall. The window of eight was retained for this workload.
|
||||||
|
|
||||||
## Stage 17 — Project typed report and ledger facts
|
## Stage 17 — Project typed report and ledger facts
|
||||||
|
|
||||||
Audit #22, #27 (ledger), #39, #41 (projection).
|
Audit #22, #27 (ledger), #39, #41 (projection).
|
||||||
|
|||||||
@@ -59,7 +59,8 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
|||||||
for _, warning := range writer.Warnings() {
|
for _, warning := range writer.Warnings() {
|
||||||
warn(warning)
|
warn(warning)
|
||||||
}
|
}
|
||||||
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot())))
|
observations := processreport.IndexLLMObservations(observer.Snapshot(), output.WorkUnits)
|
||||||
|
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observations)))
|
||||||
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
||||||
}
|
}
|
||||||
var failures []processreport.RunFailure
|
var failures []processreport.RunFailure
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -94,3 +95,30 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
|||||||
Events []executionEvent `json:"events"`
|
Events []executionEvent `json:"events"`
|
||||||
}{llm.Summarize(snapshot), calls, events}
|
}{llm.Summarize(snapshot), calls, events}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IndexLLMObservations assigns the same proposal identities used by the ledger.
|
||||||
|
func IndexLLMObservations(snapshot llm.ObservationSnapshot, units []runner.WorkUnitOutcome) llm.ObservationSnapshot {
|
||||||
|
type sectionKey struct {
|
||||||
|
module string
|
||||||
|
section int
|
||||||
|
}
|
||||||
|
offsets := make(map[sectionKey]int)
|
||||||
|
for _, unit := range units {
|
||||||
|
if unit.ProposalIndexOffset != 0 {
|
||||||
|
offsets[sectionKey{unit.ModuleInstance, unit.Section.Index}] = unit.ProposalIndexOffset
|
||||||
|
}
|
||||||
|
}
|
||||||
|
index := func(context *structuredoutput.DiagnosticContext) {
|
||||||
|
if context.SectionIndex != nil && context.ProposalIndex != nil {
|
||||||
|
offset := offsets[sectionKey{context.ModuleInstance, *context.SectionIndex}]
|
||||||
|
context.ProposalIndex = structuredoutput.OffsetProposalIndex(context.ProposalIndex, offset)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := range snapshot.Calls {
|
||||||
|
index(&snapshot.Calls[i].Context)
|
||||||
|
}
|
||||||
|
for i := range snapshot.Events {
|
||||||
|
index(&snapshot.Events[i].Context)
|
||||||
|
}
|
||||||
|
return snapshot
|
||||||
|
}
|
||||||
|
|||||||
@@ -176,27 +176,26 @@ func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
|
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
|
||||||
origin := errors.New("earlier proposal failed")
|
origin := errors.New("earlier proposal failed")
|
||||||
laterCompleted := make(chan struct{})
|
laterValidationStarted := make(chan struct{})
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
var validations atomic.Int32
|
|
||||||
module := outcomeModule{
|
module := outcomeModule{
|
||||||
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
|
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
|
||||||
contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) {
|
contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
|
||||||
validations.Add(1)
|
close(laterValidationStarted)
|
||||||
return validators.Result{}, errors.New("validation should not be submitted")
|
<-ctx.Done()
|
||||||
|
return validators.Result{}, ctx.Err()
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
if req.Section.Index == 0 {
|
if req.Section.Index == 0 {
|
||||||
select {
|
select {
|
||||||
case <-laterCompleted:
|
case <-laterValidationStarted:
|
||||||
return contracts.ProposalResult{}, origin
|
return contracts.ProposalResult{}, origin
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return contracts.ProposalResult{}, ctx.Err()
|
return contracts.ProposalResult{}, ctx.Err()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
close(laterCompleted)
|
|
||||||
return contracts.ProposalResult{
|
return contracts.ProposalResult{
|
||||||
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
|
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
|
||||||
SegmentReferences: []int{1},
|
SegmentReferences: []int{1},
|
||||||
@@ -212,8 +211,8 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
|
|||||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
|
if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
|
||||||
t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err)
|
t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err)
|
||||||
}
|
}
|
||||||
var failure *ExecutionFailure
|
var failure *ExecutionFailure
|
||||||
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
|
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
|
||||||
@@ -222,7 +221,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
|
|||||||
}
|
}
|
||||||
candidate := out.Candidates[0]
|
candidate := out.Candidates[0]
|
||||||
later := out.WorkUnits[1]
|
later := out.WorkUnits[1]
|
||||||
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted ||
|
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) ||
|
||||||
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
|
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
|
||||||
out.FinalTranscript.Segments[1].Text != "other text" {
|
out.FinalTranscript.Segments[1].Text != "other text" {
|
||||||
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)
|
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)
|
||||||
|
|||||||
@@ -190,18 +190,37 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
|
|||||||
snapshot := *revision
|
snapshot := *revision
|
||||||
record.Revision = &snapshot
|
record.Revision = &snapshot
|
||||||
}
|
}
|
||||||
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
|
feedback := findings
|
||||||
var failure any
|
record.Artifacts, _ = structuredoutput.WriteIndexedInteraction(input.Interactions, func(offset int, resolvePath func(string) string) structuredoutput.Interaction {
|
||||||
if err != nil {
|
index := proposalIndex + offset
|
||||||
failure = map[string]string{"error": err.Error()}
|
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, index, attemptNumber)
|
||||||
}
|
var failure any
|
||||||
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
|
if err != nil {
|
||||||
Name: name,
|
failure = map[string]string{"error": err.Error()}
|
||||||
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
|
}
|
||||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
|
var correction *proposals.EnrichedCorrectionProposal
|
||||||
RequestPayload: completion.Prepared,
|
if record.Revision != nil {
|
||||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
|
snapshot := *record.Revision
|
||||||
ErrorPayload: failure,
|
snapshot.ProposalIndex = index
|
||||||
|
correction = &snapshot
|
||||||
|
}
|
||||||
|
decisions := slices.Clone(validated.decisions)
|
||||||
|
for i := range decisions {
|
||||||
|
decisions[i].ProposalIndex += offset
|
||||||
|
decisions[i].DiagnosticArtifactPath = resolvePath(decisions[i].DiagnosticArtifactPath)
|
||||||
|
}
|
||||||
|
rejections := slices.Clone(validated.rejected)
|
||||||
|
for i := range rejections {
|
||||||
|
rejections[i].ProposalIndex += offset
|
||||||
|
}
|
||||||
|
return structuredoutput.Interaction{
|
||||||
|
Name: name,
|
||||||
|
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &index, RepairAttempt: &attemptNumber},
|
||||||
|
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": index, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": feedback},
|
||||||
|
RequestPayload: completion.Prepared,
|
||||||
|
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": correction, "decisions": decisions, "rejections": rejections},
|
||||||
|
ErrorPayload: failure,
|
||||||
|
}
|
||||||
})
|
})
|
||||||
branch.Attempts = append(branch.Attempts, record)
|
branch.Attempts = append(branch.Attempts, record)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,17 +56,18 @@ type ModuleResult struct {
|
|||||||
// WorkUnitOutcome keeps initial section identity while text evolves between
|
// WorkUnitOutcome keeps initial section identity while text evolves between
|
||||||
// module instances. Detailed candidate and decision records are debug evidence.
|
// module instances. Detailed candidate and decision records are debug evidence.
|
||||||
type WorkUnitOutcome struct {
|
type WorkUnitOutcome struct {
|
||||||
StageIndex int `json:"stage_index"`
|
ProposalIndexOffset int `json:"-"`
|
||||||
StageID *string `json:"stage_id,omitempty"`
|
StageIndex int `json:"stage_index"`
|
||||||
ModuleIndex int `json:"module_index"`
|
StageID *string `json:"stage_id,omitempty"`
|
||||||
ModuleKey string `json:"module_key"`
|
ModuleIndex int `json:"module_index"`
|
||||||
ModuleInstance string `json:"module_instance"`
|
ModuleKey string `json:"module_key"`
|
||||||
Section contracts.SectionMetadata `json:"section"`
|
ModuleInstance string `json:"module_instance"`
|
||||||
State outcomes.WorkState `json:"state"`
|
Section contracts.SectionMetadata `json:"section"`
|
||||||
Failure *ExecutionFailure `json:"-"`
|
State outcomes.WorkState `json:"state"`
|
||||||
CorrectionsApplied int `json:"corrections_applied"`
|
Failure *ExecutionFailure `json:"-"`
|
||||||
Repairs []RepairOutcome `json:"-"`
|
CorrectionsApplied int `json:"corrections_applied"`
|
||||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
Repairs []RepairOutcome `json:"-"`
|
||||||
|
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ValidatorDecisionRecord struct {
|
type ValidatorDecisionRecord struct {
|
||||||
@@ -342,15 +343,21 @@ type sectionProposalResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type sectionValidationResult struct {
|
type sectionValidationResult struct {
|
||||||
repairs []RepairOutcome
|
interactions *sectionInteractions
|
||||||
sectionPos int
|
repairs []RepairOutcome
|
||||||
enriched []proposals.EnrichedCorrectionProposal
|
sectionPos int
|
||||||
approved []proposals.EnrichedCorrectionProposal
|
approved []proposals.EnrichedCorrectionProposal
|
||||||
decisions []ValidatorDecisionRecord
|
decisions []ValidatorDecisionRecord
|
||||||
rejected []ValidatorRejectedChange
|
rejected []ValidatorRejectedChange
|
||||||
operations []outcomes.Operation
|
operations []outcomes.Operation
|
||||||
filtered []outcomes.FilteredProposal
|
filtered []outcomes.FilteredProposal
|
||||||
failure *ExecutionFailure
|
failure *ExecutionFailure
|
||||||
|
}
|
||||||
|
|
||||||
|
type sectionWork struct {
|
||||||
|
candidates []proposals.EnrichedCorrectionProposal
|
||||||
|
validation *sectionValidationResult
|
||||||
|
finished bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type modulePipelineResult struct {
|
type modulePipelineResult struct {
|
||||||
@@ -363,60 +370,42 @@ type modulePipelineResult struct {
|
|||||||
ValidatorRejected []ValidatorRejectedChange
|
ValidatorRejected []ValidatorRejectedChange
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
|
// moduleWorkWindow bounds active and downstream work across all roles in
|
||||||
results := make(chan sectionProposalResult, len(input.Sections))
|
// one module. PromptKit still owns admission to each provider backend.
|
||||||
|
const moduleWorkWindow = 8
|
||||||
|
|
||||||
go func() {
|
func proposeSection(ctx context.Context, input collectSectionProposalsInput, sectionPos int) sectionProposalResult {
|
||||||
defer close(results)
|
section := input.Sections[sectionPos]
|
||||||
var wg sync.WaitGroup
|
result := sectionProposalResult{sectionPos: sectionPos, section: section}
|
||||||
for sectionPos, section := range input.Sections {
|
if ctx.Err() != nil {
|
||||||
sectionPos := sectionPos
|
return result
|
||||||
section := section
|
}
|
||||||
wg.Add(1)
|
meta := contracts.SectionMetadataFromSection(section)
|
||||||
go func() {
|
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
|
||||||
defer wg.Done()
|
ExecutionContext: contracts.ExecutionContext{
|
||||||
if ctx.Err() != nil {
|
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||||
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
|
Settings: input.Spec.Settings,
|
||||||
return
|
WorkingTranscript: transcriptFromSection(section),
|
||||||
}
|
Glossary: input.Spec.Glossary,
|
||||||
meta := contracts.SectionMetadataFromSection(section)
|
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||||
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
|
GlossaryVariables: input.GlossaryVariables,
|
||||||
ExecutionContext: contracts.ExecutionContext{
|
Section: &meta,
|
||||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
Interactions: input.Interactions,
|
||||||
Settings: input.Spec.Settings,
|
},
|
||||||
WorkingTranscript: transcriptFromSection(section),
|
RunSpec: contracts.ModuleRunSpec{
|
||||||
Glossary: input.Spec.Glossary,
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
InstanceName: input.Spec.InstanceName,
|
||||||
GlossaryVariables: input.GlossaryVariables,
|
ReplacementPolicy: input.Policy,
|
||||||
Section: &meta,
|
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
|
||||||
Interactions: input.Interactions,
|
},
|
||||||
},
|
LLMClient: input.ProposalClient,
|
||||||
RunSpec: contracts.ModuleRunSpec{
|
})
|
||||||
ModuleKey: input.Spec.ModuleKey,
|
result.started = true
|
||||||
InstanceName: input.Spec.InstanceName,
|
result.proposal = proposalResult
|
||||||
ReplacementPolicy: input.Policy,
|
if err != nil {
|
||||||
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
|
result.failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
|
||||||
},
|
}
|
||||||
LLMClient: input.ProposalClient,
|
return result
|
||||||
})
|
|
||||||
var failure *ExecutionFailure
|
|
||||||
if err != nil {
|
|
||||||
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
|
|
||||||
fail(failure)
|
|
||||||
}
|
|
||||||
results <- sectionProposalResult{
|
|
||||||
sectionPos: sectionPos,
|
|
||||||
section: section,
|
|
||||||
proposal: proposalResult,
|
|
||||||
started: true,
|
|
||||||
failure: failure,
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
}()
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
||||||
@@ -453,23 +442,26 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
input.Spec.CaptureProposalExchange = true
|
input.Spec.CaptureProposalExchange = true
|
||||||
}
|
}
|
||||||
|
|
||||||
pending := make(map[int]sectionProposalResult, len(input.Sections))
|
pending := make(map[int]*sectionWork, moduleWorkWindow)
|
||||||
validationResults := make(chan sectionValidationResult, len(input.Sections))
|
proposalResults := make(chan sectionProposalResult)
|
||||||
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
|
validationResults := make(chan sectionValidationResult)
|
||||||
validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators())
|
validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators())
|
||||||
|
|
||||||
nextSectionToProcess := 0
|
nextSectionToRecord := 0
|
||||||
nextProposalIndex := 0
|
nextProposalIndex := 0
|
||||||
|
|
||||||
var (
|
var (
|
||||||
firstFailure *ExecutionFailure
|
firstFailure *ExecutionFailure
|
||||||
errMu sync.Mutex
|
errMu sync.Mutex
|
||||||
vwg sync.WaitGroup
|
workers sync.WaitGroup
|
||||||
)
|
)
|
||||||
|
|
||||||
runCtx, cancel := context.WithCancel(ctx)
|
runCtx, cancel := context.WithCancel(ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
setFailure := func(failure *ExecutionFailure) {
|
setFailure := func(failure *ExecutionFailure) {
|
||||||
|
if failure == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
errMu.Lock()
|
errMu.Lock()
|
||||||
defer errMu.Unlock()
|
defer errMu.Unlock()
|
||||||
if firstFailure == nil {
|
if firstFailure == nil {
|
||||||
@@ -477,133 +469,93 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
cancel()
|
cancel()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
proposalResults := collectSectionProposals(runCtx, input, setFailure)
|
makeCandidates := func(sectionResult sectionProposalResult) []proposals.EnrichedCorrectionProposal {
|
||||||
|
|
||||||
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
|
|
||||||
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
|
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
|
||||||
for i, corrected := range sectionResult.proposal.Proposals {
|
for i, corrected := range sectionResult.proposal.Proposals {
|
||||||
sectionIndex := index
|
sectionIndex := sectionResult.section.Index
|
||||||
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
||||||
CorrectionProposal: corrected,
|
CorrectionProposal: corrected,
|
||||||
ProposalMetadata: proposals.ProposalMetadata{
|
ProposalMetadata: proposals.ProposalMetadata{
|
||||||
ProposalIndex: nextProposalIndex + i,
|
ProposalIndex: i,
|
||||||
ModuleKey: input.Spec.ModuleKey,
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
ModuleInstance: input.Spec.InstanceName,
|
ModuleInstance: input.Spec.InstanceName,
|
||||||
SectionIndex: §ionIndex,
|
SectionIndex: §ionIndex,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
nextProposalIndex += len(sectionEnriched)
|
|
||||||
out.ProposalCount += len(sectionEnriched)
|
|
||||||
out.Candidates = append(out.Candidates, sectionEnriched...)
|
|
||||||
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
|
|
||||||
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
|
|
||||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
|
||||||
for _, proposal := range sectionEnriched {
|
|
||||||
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sectionEnriched
|
return sectionEnriched
|
||||||
}
|
}
|
||||||
|
|
||||||
for result := range proposalResults {
|
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
|
||||||
unit := &out.WorkUnits[result.sectionPos]
|
unit := &out.WorkUnits[sectionPos]
|
||||||
if result.started {
|
unit.ProposalIndexOffset = nextProposalIndex
|
||||||
unit.State = outcomes.WorkInterrupted
|
for _, candidate := range enriched {
|
||||||
|
candidate.ProposalIndex += nextProposalIndex
|
||||||
|
out.Candidates = append(out.Candidates, candidate)
|
||||||
}
|
}
|
||||||
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
out.ProposalCount += len(enriched)
|
||||||
if result.failure != nil {
|
for operationIndex := range unit.Operations {
|
||||||
unit.Failure = result.failure
|
operation := &unit.Operations[operationIndex]
|
||||||
continue
|
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||||
}
|
for _, proposal := range enriched {
|
||||||
if !result.started {
|
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
|
||||||
continue
|
}
|
||||||
}
|
|
||||||
pending[result.sectionPos] = result
|
|
||||||
|
|
||||||
for {
|
|
||||||
if runCtx.Err() != nil {
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
sectionResult, ok := pending[nextSectionToProcess]
|
}
|
||||||
if !ok {
|
nextProposalIndex += len(enriched)
|
||||||
break
|
}
|
||||||
|
|
||||||
|
validate := func(sectionResult sectionProposalResult, enriched []proposals.EnrichedCorrectionProposal) sectionValidationResult {
|
||||||
|
sectionPos := sectionResult.sectionPos
|
||||||
|
sectionMetadata := contracts.SectionMetadataFromSection(sectionResult.section)
|
||||||
|
var interactions structuredoutput.InteractionSink
|
||||||
|
var capture *sectionInteractions
|
||||||
|
if input.Interactions != nil {
|
||||||
|
capture = §ionInteractions{InteractionSink: input.Interactions}
|
||||||
|
interactions = capture
|
||||||
|
}
|
||||||
|
validationInput := validateSectionCandidatesInput{
|
||||||
|
Spec: input.Spec,
|
||||||
|
Policy: input.Policy,
|
||||||
|
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||||
|
SectionIndex: §ionMetadata.Index,
|
||||||
|
Validators: validatorsOrdered,
|
||||||
|
SectionEnriched: enriched,
|
||||||
|
Exchange: sectionResult.proposal.Exchange,
|
||||||
|
ValidationLLMClient: input.ValidationClient,
|
||||||
|
GlossaryView: input.GlossaryView,
|
||||||
|
Interactions: interactions,
|
||||||
|
}
|
||||||
|
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
||||||
|
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
||||||
|
}
|
||||||
|
validated, err := validateSectionCandidates(runCtx, validationInput)
|
||||||
|
var repairs []RepairOutcome
|
||||||
|
phase, kind := "validation", outcomes.OperationValidation
|
||||||
|
if err == nil && repairPromptID != "" {
|
||||||
|
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
|
||||||
|
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
|
||||||
|
}
|
||||||
|
var failure *ExecutionFailure
|
||||||
|
if err != nil {
|
||||||
|
if !errors.As(err, &failure) {
|
||||||
|
failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations)
|
||||||
}
|
}
|
||||||
delete(pending, nextSectionToProcess)
|
}
|
||||||
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
|
return sectionValidationResult{
|
||||||
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
|
interactions: capture,
|
||||||
|
repairs: repairs,
|
||||||
vwg.Add(1)
|
sectionPos: sectionPos,
|
||||||
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
|
approved: validated.approved,
|
||||||
defer vwg.Done()
|
decisions: validated.decisions,
|
||||||
validationInput := validateSectionCandidatesInput{
|
rejected: validated.rejected,
|
||||||
Spec: input.Spec,
|
operations: validated.operations,
|
||||||
Policy: input.Policy,
|
filtered: validated.filtered,
|
||||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
failure: failure,
|
||||||
SectionIndex: §ionMetadata.Index,
|
|
||||||
Validators: validatorsOrdered,
|
|
||||||
SectionEnriched: enriched,
|
|
||||||
Exchange: sectionResult.proposal.Exchange,
|
|
||||||
ValidationLLMClient: input.ValidationClient,
|
|
||||||
GlossaryView: input.GlossaryView,
|
|
||||||
Interactions: input.Interactions,
|
|
||||||
}
|
|
||||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
|
||||||
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
|
||||||
}
|
|
||||||
validated, err := validateSectionCandidates(runCtx, validationInput)
|
|
||||||
var repairs []RepairOutcome
|
|
||||||
phase, kind := "validation", outcomes.OperationValidation
|
|
||||||
if err == nil && repairPromptID != "" {
|
|
||||||
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
|
|
||||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
|
|
||||||
}
|
|
||||||
var failure *ExecutionFailure
|
|
||||||
if err != nil {
|
|
||||||
if !errors.As(err, &failure) {
|
|
||||||
failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations)
|
|
||||||
}
|
|
||||||
setFailure(failure)
|
|
||||||
}
|
|
||||||
validationResults <- sectionValidationResult{
|
|
||||||
repairs: repairs,
|
|
||||||
sectionPos: sectionPos,
|
|
||||||
enriched: enriched,
|
|
||||||
approved: validated.approved,
|
|
||||||
decisions: validated.decisions,
|
|
||||||
rejected: validated.rejected,
|
|
||||||
operations: validated.operations,
|
|
||||||
filtered: validated.filtered,
|
|
||||||
failure: failure,
|
|
||||||
}
|
|
||||||
}(nextSectionToProcess, sectionEnriched, sectionMeta)
|
|
||||||
nextSectionToProcess++
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
recordValidation := func(res sectionValidationResult) {
|
||||||
vwg.Wait()
|
unit := &out.WorkUnits[res.sectionPos]
|
||||||
close(validationResults)
|
|
||||||
|
|
||||||
// A failed or canceled module can still have usable proposals from later
|
|
||||||
// sections. Assign their indices in order after preserving the dispatched
|
|
||||||
// prefix, without submitting new validation work.
|
|
||||||
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
|
|
||||||
if result, ok := pending[pos]; ok {
|
|
||||||
collectCandidates(result, result.section.Index)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for v := range validationResults {
|
|
||||||
validationBySection[v.sectionPos] = v
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range input.Sections {
|
|
||||||
res, ok := validationBySection[i]
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
unit := &out.WorkUnits[i]
|
|
||||||
unit.Operations = append(unit.Operations, res.operations...)
|
unit.Operations = append(unit.Operations, res.operations...)
|
||||||
unit.Repairs = res.repairs
|
unit.Repairs = res.repairs
|
||||||
mergeFilteredProposals(unit, res.filtered)
|
mergeFilteredProposals(unit, res.filtered)
|
||||||
@@ -617,6 +569,69 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
||||||
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
nextAdmission, active := 0, 0
|
||||||
|
for {
|
||||||
|
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
|
||||||
|
offset := nextProposalIndex
|
||||||
|
recordCandidates(nextSectionToRecord, work.candidates)
|
||||||
|
if work.validation != nil {
|
||||||
|
work.validation.assignProposalIndices(offset)
|
||||||
|
recordValidation(*work.validation)
|
||||||
|
}
|
||||||
|
delete(pending, nextSectionToRecord)
|
||||||
|
nextSectionToRecord++
|
||||||
|
}
|
||||||
|
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
|
||||||
|
sectionPos := nextAdmission
|
||||||
|
nextAdmission++
|
||||||
|
pending[sectionPos] = §ionWork{}
|
||||||
|
active++
|
||||||
|
workers.Go(func() {
|
||||||
|
proposed := proposeSection(runCtx, input, sectionPos)
|
||||||
|
setFailure(proposed.failure)
|
||||||
|
proposalResults <- proposed
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if active == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case result := <-proposalResults:
|
||||||
|
active--
|
||||||
|
unit := &out.WorkUnits[result.sectionPos]
|
||||||
|
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
||||||
|
if result.started {
|
||||||
|
unit.State = outcomes.WorkInterrupted
|
||||||
|
}
|
||||||
|
unit.Failure = result.failure
|
||||||
|
work := pending[result.sectionPos]
|
||||||
|
if result.started && result.failure == nil {
|
||||||
|
work.candidates = makeCandidates(result)
|
||||||
|
if runCtx.Err() == nil {
|
||||||
|
enriched := work.candidates
|
||||||
|
active++
|
||||||
|
workers.Go(func() {
|
||||||
|
validated := validate(result, enriched)
|
||||||
|
setFailure(validated.failure)
|
||||||
|
validationResults <- validated
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
work.finished = true
|
||||||
|
case validated := <-validationResults:
|
||||||
|
active--
|
||||||
|
work := pending[validated.sectionPos]
|
||||||
|
work.validation = &validated
|
||||||
|
work.finished = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
workers.Wait()
|
||||||
|
sort.SliceStable(out.Approved, func(i, j int) bool {
|
||||||
|
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
|
||||||
|
})
|
||||||
|
|
||||||
for _, unit := range out.WorkUnits {
|
for _, unit := range out.WorkUnits {
|
||||||
out.Operations = append(out.Operations, unit.Operations...)
|
out.Operations = append(out.Operations, unit.Operations...)
|
||||||
}
|
}
|
||||||
|
|||||||
111
internal/framework/runner/section_evidence.go
Normal file
111
internal/framework/runner/section_evidence.go
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Section validation can finish before preceding proposal counts are known.
|
||||||
|
type sectionInteractions struct {
|
||||||
|
structuredoutput.InteractionSink
|
||||||
|
mu sync.Mutex
|
||||||
|
builders []structuredoutput.IndexedInteraction
|
||||||
|
written []structuredoutput.InteractionArtifacts
|
||||||
|
}
|
||||||
|
|
||||||
|
const pendingInteractionPrefix = "\x00interaction:"
|
||||||
|
|
||||||
|
func (s *sectionInteractions) WriteIndexedInteraction(build structuredoutput.IndexedInteraction) (structuredoutput.InteractionArtifacts, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
receipt := pendingInteractionPrefix + strconv.Itoa(len(s.builders))
|
||||||
|
s.builders = append(s.builders, build)
|
||||||
|
return structuredoutput.InteractionArtifacts{ResponsePayloadPath: receipt}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sectionInteractions) write(offset int) {
|
||||||
|
for _, build := range s.builders {
|
||||||
|
paths, _ := s.InteractionSink.WriteInteraction(build(offset, s.responsePath))
|
||||||
|
s.written = append(s.written, paths)
|
||||||
|
}
|
||||||
|
s.builders = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sectionInteractions) artifacts(paths structuredoutput.InteractionArtifacts) structuredoutput.InteractionArtifacts {
|
||||||
|
value, pending := strings.CutPrefix(paths.ResponsePayloadPath, pendingInteractionPrefix)
|
||||||
|
if !pending {
|
||||||
|
return paths
|
||||||
|
}
|
||||||
|
index, err := strconv.Atoi(value)
|
||||||
|
if err != nil || index < 0 || index >= len(s.written) {
|
||||||
|
return structuredoutput.InteractionArtifacts{}
|
||||||
|
}
|
||||||
|
return s.written[index]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sectionInteractions) responsePath(path string) string {
|
||||||
|
return s.artifacts(structuredoutput.InteractionArtifacts{ResponsePayloadPath: path}).ResponsePayloadPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *sectionValidationResult) assignProposalIndices(offset int) {
|
||||||
|
capture := r.interactions != nil
|
||||||
|
if capture {
|
||||||
|
r.interactions.write(offset)
|
||||||
|
}
|
||||||
|
for i := range r.approved {
|
||||||
|
r.approved[i].ProposalIndex += offset
|
||||||
|
}
|
||||||
|
for i := range r.rejected {
|
||||||
|
r.rejected[i].ProposalIndex += offset
|
||||||
|
}
|
||||||
|
assignDecisions := func(decisions []ValidatorDecisionRecord) {
|
||||||
|
for i := range decisions {
|
||||||
|
decisions[i].ProposalIndex += offset
|
||||||
|
if capture {
|
||||||
|
decisions[i].DiagnosticArtifactPath = r.interactions.responsePath(decisions[i].DiagnosticArtifactPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assignDecisions(r.decisions)
|
||||||
|
for i := range r.operations {
|
||||||
|
op := &r.operations[i]
|
||||||
|
op.Reference.ProposalIndex = structuredoutput.OffsetProposalIndex(op.Reference.ProposalIndex, offset)
|
||||||
|
op.ProposalIndices = slices.Clone(op.ProposalIndices)
|
||||||
|
for j := range op.ProposalIndices {
|
||||||
|
op.ProposalIndices[j] += offset
|
||||||
|
}
|
||||||
|
op.Filtered = slices.Clone(op.Filtered)
|
||||||
|
for j := range op.Filtered {
|
||||||
|
op.Filtered[j].ProposalIndex += offset
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := range r.filtered {
|
||||||
|
r.filtered[i].ProposalIndex += offset
|
||||||
|
}
|
||||||
|
for i := range r.repairs {
|
||||||
|
repair := &r.repairs[i]
|
||||||
|
repair.ProposalIndex += offset
|
||||||
|
if capture {
|
||||||
|
repair.Original.ProposalIndex += offset
|
||||||
|
repair.Final.ProposalIndex += offset
|
||||||
|
}
|
||||||
|
for j := range repair.Attempts {
|
||||||
|
attempt := &repair.Attempts[j]
|
||||||
|
if attempt.Revision != nil {
|
||||||
|
attempt.Revision.ProposalIndex += offset
|
||||||
|
}
|
||||||
|
assignDecisions(attempt.Decisions)
|
||||||
|
if capture {
|
||||||
|
attempt.Artifacts = r.interactions.artifacts(attempt.Artifacts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.failure != nil && r.failure.Origin != nil {
|
||||||
|
r.failure.Origin.ProposalIndex = structuredoutput.OffsetProposalIndex(r.failure.Origin.ProposalIndex, offset)
|
||||||
|
}
|
||||||
|
r.interactions = nil
|
||||||
|
}
|
||||||
275
internal/framework/runner/window_test.go
Normal file
275
internal/framework/runner/window_test.go
Normal file
@@ -0,0 +1,275 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type sharedBackendModule struct {
|
||||||
|
*grammar.Module
|
||||||
|
validator validators.Validator
|
||||||
|
headEntered <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m sharedBackendModule) Validators() []validators.Validator {
|
||||||
|
return []validators.Validator{m.validator}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
|
if req.Section.Index > 0 {
|
||||||
|
select {
|
||||||
|
case <-m.headEntered:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return contracts.ProposalResult{}, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m.Module.Propose(ctx, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
release := make(chan struct{})
|
||||||
|
entered := make(chan struct{})
|
||||||
|
validationStarted := make(chan struct{}, 1)
|
||||||
|
var other, validating, active, peak atomic.Int32
|
||||||
|
client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
body, err := io.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
n := active.Add(1)
|
||||||
|
for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
|
||||||
|
}
|
||||||
|
defer active.Add(-1)
|
||||||
|
output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}`
|
||||||
|
switch {
|
||||||
|
case bytes.Contains(body, []byte(`"validations"`)):
|
||||||
|
validating.Add(1)
|
||||||
|
select {
|
||||||
|
case validationStarted <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||||
|
case bytes.Contains(body, []byte("marker-000")):
|
||||||
|
close(entered)
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
case <-r.Context().Done():
|
||||||
|
return nil, r.Context().Err()
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
other.Add(1)
|
||||||
|
}
|
||||||
|
content, _ := json.Marshal(output)
|
||||||
|
wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil
|
||||||
|
})}
|
||||||
|
runtime, err := llm.NewRuntime(ctx, loaded, client)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered}
|
||||||
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||||
|
cfg := runnerTestConfig()
|
||||||
|
cfg.Chunking.MaxSectionTokens = 1
|
||||||
|
cfg.Chunking.MinSectionTokens = 0
|
||||||
|
transcript := &schema.Transcript{}
|
||||||
|
for i := 0; i < 36; i++ {
|
||||||
|
transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)})
|
||||||
|
}
|
||||||
|
type result struct {
|
||||||
|
out RunOutput
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
finished := make(chan result, 1)
|
||||||
|
go func() {
|
||||||
|
out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript,
|
||||||
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}},
|
||||||
|
})
|
||||||
|
finished <- result{out, err}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-entered:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal(ctx.Err())
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-validationStarted:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("later validation did not reach the backend while the earlier proposal held a slot")
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
completed := <-finished
|
||||||
|
if completed.err != nil {
|
||||||
|
t.Fatal(completed.err)
|
||||||
|
}
|
||||||
|
if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 {
|
||||||
|
t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load())
|
||||||
|
}
|
||||||
|
changes := completed.out.ModuleResults[0].AppliedChanges
|
||||||
|
if len(changes) != len(transcript.Segments) {
|
||||||
|
t.Fatalf("got %d applied changes", len(changes))
|
||||||
|
}
|
||||||
|
for i, change := range changes {
|
||||||
|
if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID {
|
||||||
|
t.Fatalf("change %d: %+v", i, change)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||||
|
for _, canceled := range []bool{false, true} {
|
||||||
|
name := "completed"
|
||||||
|
if canceled {
|
||||||
|
name = "canceled"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
laterRevalidated := make(chan struct{})
|
||||||
|
proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
section := *req.Context.SectionIndex
|
||||||
|
if section == 0 {
|
||||||
|
select {
|
||||||
|
case <-laterRevalidated:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
text := []string{"alpha", "bravo"}[section]
|
||||||
|
dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{
|
||||||
|
SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text),
|
||||||
|
}}
|
||||||
|
return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil
|
||||||
|
})
|
||||||
|
validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
section := *req.Context.SectionIndex
|
||||||
|
first := section == 0
|
||||||
|
revision := req.Context.RepairAttempt != nil
|
||||||
|
if first && canceled {
|
||||||
|
cancel()
|
||||||
|
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||||
|
}
|
||||||
|
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
|
||||||
|
CorrectionIndex: "1", Approved: first || revision, Reason: "checked",
|
||||||
|
}}
|
||||||
|
if revision {
|
||||||
|
close(laterRevalidated)
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{}, nil
|
||||||
|
})
|
||||||
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator}
|
||||||
|
// The module's transport gates are supplied by the scripted calls above.
|
||||||
|
started := make(chan struct{})
|
||||||
|
close(started)
|
||||||
|
module.headEntered = started
|
||||||
|
repair := &repairClient{t: t, outputs: []string{
|
||||||
|
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
|
||||||
|
}}
|
||||||
|
cfg := runnerTestConfig()
|
||||||
|
cfg.Chunking.MaxSectionTokens = 1
|
||||||
|
cfg.Chunking.MinSectionTokens = 0
|
||||||
|
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
||||||
|
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true,
|
||||||
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
|
||||||
|
ModuleSpecs: []contracts.ModuleRunSpec{{
|
||||||
|
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
|
||||||
|
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}},
|
||||||
|
}},
|
||||||
|
})
|
||||||
|
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
later := out.WorkUnits[1]
|
||||||
|
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
|
||||||
|
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 {
|
||||||
|
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
|
||||||
|
}
|
||||||
|
attempt := later.Repairs[0].Attempts[0]
|
||||||
|
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
|
||||||
|
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
|
||||||
|
t.Fatalf("lost ordered repair evidence: %+v", attempt)
|
||||||
|
}
|
||||||
|
var repaired struct {
|
||||||
|
Decoded proposal_generation.RepairResponse
|
||||||
|
Revision *proposals.EnrichedCorrectionProposal
|
||||||
|
Decisions []ValidatorDecisionRecord
|
||||||
|
}
|
||||||
|
readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired)
|
||||||
|
if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 ||
|
||||||
|
len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 ||
|
||||||
|
repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath ||
|
||||||
|
!strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") {
|
||||||
|
t.Fatalf("repair artifact uses different identities: %+v", repaired)
|
||||||
|
}
|
||||||
|
var checked struct {
|
||||||
|
Decoded validators.LLMValidationResponse
|
||||||
|
Reconciliation struct{ Decisions []validators.Decision }
|
||||||
|
}
|
||||||
|
readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked)
|
||||||
|
if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" ||
|
||||||
|
len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 {
|
||||||
|
t.Fatalf("validation evidence uses different identities: %+v", checked)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readInteraction(t *testing.T, path string, result any) {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,3 +22,24 @@ type InteractionArtifacts struct {
|
|||||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IndexedInteraction builds an artifact after the section's ordered proposal
|
||||||
|
// indices and paths from earlier interactions are available.
|
||||||
|
type IndexedInteraction func(offset int, resolvePath func(string) string) Interaction
|
||||||
|
|
||||||
|
func WriteIndexedInteraction(sink InteractionSink, build IndexedInteraction) (InteractionArtifacts, error) {
|
||||||
|
if deferred, ok := sink.(interface {
|
||||||
|
WriteIndexedInteraction(IndexedInteraction) (InteractionArtifacts, error)
|
||||||
|
}); ok {
|
||||||
|
return deferred.WriteIndexedInteraction(build)
|
||||||
|
}
|
||||||
|
return sink.WriteInteraction(build(0, func(path string) string { return path }))
|
||||||
|
}
|
||||||
|
|
||||||
|
func OffsetProposalIndex(index *int, offset int) *int {
|
||||||
|
if index == nil || offset == 0 {
|
||||||
|
return index
|
||||||
|
}
|
||||||
|
next := *index + offset
|
||||||
|
return &next
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -141,33 +142,44 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
if req.Interactions == nil {
|
if req.Interactions == nil {
|
||||||
return structuredoutput.InteractionArtifacts{}
|
return structuredoutput.InteractionArtifacts{}
|
||||||
}
|
}
|
||||||
stageInstance := req.ModuleInstance
|
artifacts, _ := structuredoutput.WriteIndexedInteraction(req.Interactions, func(offset int, _ func(string) string) structuredoutput.Interaction {
|
||||||
if req.SectionIndex != nil {
|
coordinates := coordinates
|
||||||
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
|
coordinates.ProposalIndex = structuredoutput.OffsetProposalIndex(req.RepairProposalIndex, offset)
|
||||||
}
|
stageInstance := req.ModuleInstance
|
||||||
if req.RepairProposalIndex != nil && req.RepairAttempt != nil {
|
if req.SectionIndex != nil {
|
||||||
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt)
|
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
|
||||||
}
|
}
|
||||||
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
|
if coordinates.ProposalIndex != nil && req.RepairAttempt != nil {
|
||||||
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *coordinates.ProposalIndex, *req.RepairAttempt)
|
||||||
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
|
}
|
||||||
if reconciled != nil {
|
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||||
responsePayload["reconciliation"] = *reconciled
|
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
|
||||||
}
|
if reconciled != nil {
|
||||||
artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{
|
view := *reconciled
|
||||||
Name: stage, Context: coordinates, RequestMetadata: map[string]any{
|
view.Decisions = slices.Clone(view.Decisions)
|
||||||
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
for i := range view.Decisions {
|
||||||
"validator_name": v.name,
|
view.Decisions[i].ProposalIndex += offset
|
||||||
"validator_type": v.validatorType,
|
view.Decisions[i].DiagnosticArtifactPath = ""
|
||||||
"batch_index": batch.BatchIndex,
|
}
|
||||||
"proposal_indices": batchProposalIndices(batch.Items),
|
view.UnavailableIndices = slices.Clone(view.UnavailableIndices)
|
||||||
"repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt,
|
for i := range view.UnavailableIndices {
|
||||||
"prompt": completion.Prompt,
|
view.UnavailableIndices[i] += offset
|
||||||
"response_schema": responseSchema.DiagnosticsMap(),
|
}
|
||||||
},
|
responsePayload["reconciliation"] = view
|
||||||
RequestPayload: completion.Prepared,
|
}
|
||||||
ResponsePayload: responsePayload,
|
indices := batchProposalIndices(batch.Items)
|
||||||
ErrorPayload: errPayload(callErr),
|
for i := range indices {
|
||||||
|
indices[i] += offset
|
||||||
|
}
|
||||||
|
return structuredoutput.Interaction{
|
||||||
|
Name: stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex), Context: coordinates, RequestMetadata: map[string]any{
|
||||||
|
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
||||||
|
"validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex,
|
||||||
|
"proposal_indices": indices, "repair_proposal_index": coordinates.ProposalIndex, "repair_attempt": req.RepairAttempt,
|
||||||
|
"prompt": completion.Prompt, "response_schema": responseSchema.DiagnosticsMap(),
|
||||||
|
},
|
||||||
|
RequestPayload: completion.Prepared, ResponsePayload: responsePayload, ErrorPayload: errPayload(callErr),
|
||||||
|
}
|
||||||
})
|
})
|
||||||
return artifacts
|
return artifacts
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user