Bound concurrent section work and finalize proposal identities in order
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
@@ -94,3 +95,30 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
Events []executionEvent `json:"events"`
|
||||
}{llm.Summarize(snapshot), calls, events}
|
||||
}
|
||||
|
||||
// IndexLLMObservations assigns the same proposal identities used by the ledger.
|
||||
func IndexLLMObservations(snapshot llm.ObservationSnapshot, units []runner.WorkUnitOutcome) llm.ObservationSnapshot {
|
||||
type sectionKey struct {
|
||||
module string
|
||||
section int
|
||||
}
|
||||
offsets := make(map[sectionKey]int)
|
||||
for _, unit := range units {
|
||||
if unit.ProposalIndexOffset != 0 {
|
||||
offsets[sectionKey{unit.ModuleInstance, unit.Section.Index}] = unit.ProposalIndexOffset
|
||||
}
|
||||
}
|
||||
index := func(context *structuredoutput.DiagnosticContext) {
|
||||
if context.SectionIndex != nil && context.ProposalIndex != nil {
|
||||
offset := offsets[sectionKey{context.ModuleInstance, *context.SectionIndex}]
|
||||
context.ProposalIndex = structuredoutput.OffsetProposalIndex(context.ProposalIndex, offset)
|
||||
}
|
||||
}
|
||||
for i := range snapshot.Calls {
|
||||
index(&snapshot.Calls[i].Context)
|
||||
}
|
||||
for i := range snapshot.Events {
|
||||
index(&snapshot.Events[i].Context)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
@@ -176,27 +176,26 @@ func TestCanceledEmptyPipelineReturnsCancellation(t *testing.T) {
|
||||
|
||||
func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T) {
|
||||
origin := errors.New("earlier proposal failed")
|
||||
laterCompleted := make(chan struct{})
|
||||
laterValidationStarted := make(chan struct{})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
var validations atomic.Int32
|
||||
module := outcomeModule{
|
||||
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{
|
||||
contextValidator{validate: func(context.Context, validators.Request) (validators.Result, error) {
|
||||
validations.Add(1)
|
||||
return validators.Result{}, errors.New("validation should not be submitted")
|
||||
contextValidator{validate: func(ctx context.Context, req validators.Request) (validators.Result, error) {
|
||||
close(laterValidationStarted)
|
||||
<-ctx.Done()
|
||||
return validators.Result{}, ctx.Err()
|
||||
}},
|
||||
}},
|
||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if req.Section.Index == 0 {
|
||||
select {
|
||||
case <-laterCompleted:
|
||||
case <-laterValidationStarted:
|
||||
return contracts.ProposalResult{}, origin
|
||||
case <-ctx.Done():
|
||||
return contracts.ProposalResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
close(laterCompleted)
|
||||
return contracts.ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 93, OriginalText: "other", CorrectedText: "Other"}},
|
||||
SegmentReferences: []int{1},
|
||||
@@ -212,8 +211,8 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
||||
})
|
||||
if !errors.Is(err, origin) || validations.Load() != 0 || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
|
||||
t.Fatalf("interrupted evidence: candidates=%+v work=%+v validations=%d error=%v", out.Candidates, out.WorkUnits, validations.Load(), err)
|
||||
if !errors.Is(err, origin) || len(out.Candidates) != 1 || len(out.WorkUnits) != 2 {
|
||||
t.Fatalf("interrupted evidence: candidates=%+v work=%+v error=%v", out.Candidates, out.WorkUnits, err)
|
||||
}
|
||||
var failure *ExecutionFailure
|
||||
if !errors.As(err, &failure) || failure.Phase != "proposal" || !failure.HasSection ||
|
||||
@@ -222,7 +221,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
|
||||
}
|
||||
candidate := out.Candidates[0]
|
||||
later := out.WorkUnits[1]
|
||||
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted ||
|
||||
if candidate.ProposalIndex != 0 || candidate.TargetSegmentID != 93 || later.State != outcomes.WorkInterrupted || later.Failure == nil || !errors.Is(later.Failure, context.Canceled) ||
|
||||
len(later.Operations) != 1 || len(later.Operations[0].ProposalIndices) != 1 || later.Operations[0].ProposalIndices[0] != candidate.ProposalIndex ||
|
||||
out.FinalTranscript.Segments[1].Text != "other text" {
|
||||
t.Fatalf("later candidate or association lost: candidate=%+v work=%+v", candidate, later)
|
||||
|
||||
@@ -190,18 +190,37 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
|
||||
snapshot := *revision
|
||||
record.Revision = &snapshot
|
||||
}
|
||||
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
|
||||
var failure any
|
||||
if err != nil {
|
||||
failure = map[string]string{"error": err.Error()}
|
||||
}
|
||||
record.Artifacts, _ = input.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: name,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber},
|
||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings},
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected},
|
||||
ErrorPayload: failure,
|
||||
feedback := findings
|
||||
record.Artifacts, _ = structuredoutput.WriteIndexedInteraction(input.Interactions, func(offset int, resolvePath func(string) string) structuredoutput.Interaction {
|
||||
index := proposalIndex + offset
|
||||
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, index, attemptNumber)
|
||||
var failure any
|
||||
if err != nil {
|
||||
failure = map[string]string{"error": err.Error()}
|
||||
}
|
||||
var correction *proposals.EnrichedCorrectionProposal
|
||||
if record.Revision != nil {
|
||||
snapshot := *record.Revision
|
||||
snapshot.ProposalIndex = index
|
||||
correction = &snapshot
|
||||
}
|
||||
decisions := slices.Clone(validated.decisions)
|
||||
for i := range decisions {
|
||||
decisions[i].ProposalIndex += offset
|
||||
decisions[i].DiagnosticArtifactPath = resolvePath(decisions[i].DiagnosticArtifactPath)
|
||||
}
|
||||
rejections := slices.Clone(validated.rejected)
|
||||
for i := range rejections {
|
||||
rejections[i].ProposalIndex += offset
|
||||
}
|
||||
return structuredoutput.Interaction{
|
||||
Name: name,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &index, RepairAttempt: &attemptNumber},
|
||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": index, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": feedback},
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": correction, "decisions": decisions, "rejections": rejections},
|
||||
ErrorPayload: failure,
|
||||
}
|
||||
})
|
||||
branch.Attempts = append(branch.Attempts, record)
|
||||
}
|
||||
|
||||
@@ -56,17 +56,18 @@ type ModuleResult struct {
|
||||
// WorkUnitOutcome keeps initial section identity while text evolves between
|
||||
// module instances. Detailed candidate and decision records are debug evidence.
|
||||
type WorkUnitOutcome struct {
|
||||
StageIndex int `json:"stage_index"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
ModuleIndex int `json:"module_index"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
Section contracts.SectionMetadata `json:"section"`
|
||||
State outcomes.WorkState `json:"state"`
|
||||
Failure *ExecutionFailure `json:"-"`
|
||||
CorrectionsApplied int `json:"corrections_applied"`
|
||||
Repairs []RepairOutcome `json:"-"`
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
ProposalIndexOffset int `json:"-"`
|
||||
StageIndex int `json:"stage_index"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
ModuleIndex int `json:"module_index"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
Section contracts.SectionMetadata `json:"section"`
|
||||
State outcomes.WorkState `json:"state"`
|
||||
Failure *ExecutionFailure `json:"-"`
|
||||
CorrectionsApplied int `json:"corrections_applied"`
|
||||
Repairs []RepairOutcome `json:"-"`
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorDecisionRecord struct {
|
||||
@@ -342,15 +343,21 @@ type sectionProposalResult struct {
|
||||
}
|
||||
|
||||
type sectionValidationResult struct {
|
||||
repairs []RepairOutcome
|
||||
sectionPos int
|
||||
enriched []proposals.EnrichedCorrectionProposal
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
operations []outcomes.Operation
|
||||
filtered []outcomes.FilteredProposal
|
||||
failure *ExecutionFailure
|
||||
interactions *sectionInteractions
|
||||
repairs []RepairOutcome
|
||||
sectionPos int
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
operations []outcomes.Operation
|
||||
filtered []outcomes.FilteredProposal
|
||||
failure *ExecutionFailure
|
||||
}
|
||||
|
||||
type sectionWork struct {
|
||||
candidates []proposals.EnrichedCorrectionProposal
|
||||
validation *sectionValidationResult
|
||||
finished bool
|
||||
}
|
||||
|
||||
type modulePipelineResult struct {
|
||||
@@ -363,60 +370,42 @@ type modulePipelineResult struct {
|
||||
ValidatorRejected []ValidatorRejectedChange
|
||||
}
|
||||
|
||||
func collectSectionProposals(ctx context.Context, input collectSectionProposalsInput, fail func(*ExecutionFailure)) <-chan sectionProposalResult {
|
||||
results := make(chan sectionProposalResult, len(input.Sections))
|
||||
// moduleWorkWindow bounds active and downstream work across all roles in
|
||||
// one module. PromptKit still owns admission to each provider backend.
|
||||
const moduleWorkWindow = 8
|
||||
|
||||
go func() {
|
||||
defer close(results)
|
||||
var wg sync.WaitGroup
|
||||
for sectionPos, section := range input.Sections {
|
||||
sectionPos := sectionPos
|
||||
section := section
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if ctx.Err() != nil {
|
||||
results <- sectionProposalResult{sectionPos: sectionPos, section: section}
|
||||
return
|
||||
}
|
||||
meta := contracts.SectionMetadataFromSection(section)
|
||||
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
Settings: input.Spec.Settings,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryVariables: input.GlossaryVariables,
|
||||
Section: &meta,
|
||||
Interactions: input.Interactions,
|
||||
},
|
||||
RunSpec: contracts.ModuleRunSpec{
|
||||
ModuleKey: input.Spec.ModuleKey,
|
||||
InstanceName: input.Spec.InstanceName,
|
||||
ReplacementPolicy: input.Policy,
|
||||
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
|
||||
},
|
||||
LLMClient: input.ProposalClient,
|
||||
})
|
||||
var failure *ExecutionFailure
|
||||
if err != nil {
|
||||
failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
|
||||
fail(failure)
|
||||
}
|
||||
results <- sectionProposalResult{
|
||||
sectionPos: sectionPos,
|
||||
section: section,
|
||||
proposal: proposalResult,
|
||||
started: true,
|
||||
failure: failure,
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
return results
|
||||
func proposeSection(ctx context.Context, input collectSectionProposalsInput, sectionPos int) sectionProposalResult {
|
||||
section := input.Sections[sectionPos]
|
||||
result := sectionProposalResult{sectionPos: sectionPos, section: section}
|
||||
if ctx.Err() != nil {
|
||||
return result
|
||||
}
|
||||
meta := contracts.SectionMetadataFromSection(section)
|
||||
proposalResult, err := input.Module.Propose(ctx, contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
Settings: input.Spec.Settings,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
GlossaryVariables: input.GlossaryVariables,
|
||||
Section: &meta,
|
||||
Interactions: input.Interactions,
|
||||
},
|
||||
RunSpec: contracts.ModuleRunSpec{
|
||||
ModuleKey: input.Spec.ModuleKey,
|
||||
InstanceName: input.Spec.InstanceName,
|
||||
ReplacementPolicy: input.Policy,
|
||||
CaptureProposalExchange: input.Spec.CaptureProposalExchange,
|
||||
},
|
||||
LLMClient: input.ProposalClient,
|
||||
})
|
||||
result.started = true
|
||||
result.proposal = proposalResult
|
||||
if err != nil {
|
||||
result.failure = locateFailure(err, "proposal", failureReference(input.Spec, &meta.Index, outcomes.OperationProposal), true, proposalResult.Operations)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
||||
@@ -453,23 +442,26 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
input.Spec.CaptureProposalExchange = true
|
||||
}
|
||||
|
||||
pending := make(map[int]sectionProposalResult, len(input.Sections))
|
||||
validationResults := make(chan sectionValidationResult, len(input.Sections))
|
||||
validationBySection := make(map[int]sectionValidationResult, len(input.Sections))
|
||||
pending := make(map[int]*sectionWork, moduleWorkWindow)
|
||||
proposalResults := make(chan sectionProposalResult)
|
||||
validationResults := make(chan sectionValidationResult)
|
||||
validatorsOrdered, validatorOrder := reorderValidatorsForPipeline(input.Module.Validators())
|
||||
|
||||
nextSectionToProcess := 0
|
||||
nextSectionToRecord := 0
|
||||
nextProposalIndex := 0
|
||||
|
||||
var (
|
||||
firstFailure *ExecutionFailure
|
||||
errMu sync.Mutex
|
||||
vwg sync.WaitGroup
|
||||
workers sync.WaitGroup
|
||||
)
|
||||
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
setFailure := func(failure *ExecutionFailure) {
|
||||
if failure == nil {
|
||||
return
|
||||
}
|
||||
errMu.Lock()
|
||||
defer errMu.Unlock()
|
||||
if firstFailure == nil {
|
||||
@@ -477,133 +469,93 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
proposalResults := collectSectionProposals(runCtx, input, setFailure)
|
||||
|
||||
collectCandidates := func(sectionResult sectionProposalResult, index int) []proposals.EnrichedCorrectionProposal {
|
||||
makeCandidates := func(sectionResult sectionProposalResult) []proposals.EnrichedCorrectionProposal {
|
||||
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
|
||||
for i, corrected := range sectionResult.proposal.Proposals {
|
||||
sectionIndex := index
|
||||
sectionIndex := sectionResult.section.Index
|
||||
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: corrected,
|
||||
ProposalMetadata: proposals.ProposalMetadata{
|
||||
ProposalIndex: nextProposalIndex + i,
|
||||
ProposalIndex: i,
|
||||
ModuleKey: input.Spec.ModuleKey,
|
||||
ModuleInstance: input.Spec.InstanceName,
|
||||
SectionIndex: §ionIndex,
|
||||
},
|
||||
})
|
||||
}
|
||||
nextProposalIndex += len(sectionEnriched)
|
||||
out.ProposalCount += len(sectionEnriched)
|
||||
out.Candidates = append(out.Candidates, sectionEnriched...)
|
||||
for operationIndex := range out.WorkUnits[sectionResult.sectionPos].Operations {
|
||||
operation := &out.WorkUnits[sectionResult.sectionPos].Operations[operationIndex]
|
||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||
for _, proposal := range sectionEnriched {
|
||||
operation.ProposalIndices = append(operation.ProposalIndices, proposal.ProposalIndex)
|
||||
}
|
||||
}
|
||||
}
|
||||
return sectionEnriched
|
||||
}
|
||||
|
||||
for result := range proposalResults {
|
||||
unit := &out.WorkUnits[result.sectionPos]
|
||||
if result.started {
|
||||
unit.State = outcomes.WorkInterrupted
|
||||
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
|
||||
unit := &out.WorkUnits[sectionPos]
|
||||
unit.ProposalIndexOffset = nextProposalIndex
|
||||
for _, candidate := range enriched {
|
||||
candidate.ProposalIndex += nextProposalIndex
|
||||
out.Candidates = append(out.Candidates, candidate)
|
||||
}
|
||||
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
||||
if result.failure != nil {
|
||||
unit.Failure = result.failure
|
||||
continue
|
||||
}
|
||||
if !result.started {
|
||||
continue
|
||||
}
|
||||
pending[result.sectionPos] = result
|
||||
|
||||
for {
|
||||
if runCtx.Err() != nil {
|
||||
break
|
||||
out.ProposalCount += len(enriched)
|
||||
for operationIndex := range unit.Operations {
|
||||
operation := &unit.Operations[operationIndex]
|
||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||
for _, proposal := range enriched {
|
||||
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
|
||||
}
|
||||
}
|
||||
sectionResult, ok := pending[nextSectionToProcess]
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
nextProposalIndex += len(enriched)
|
||||
}
|
||||
|
||||
validate := func(sectionResult sectionProposalResult, enriched []proposals.EnrichedCorrectionProposal) sectionValidationResult {
|
||||
sectionPos := sectionResult.sectionPos
|
||||
sectionMetadata := contracts.SectionMetadataFromSection(sectionResult.section)
|
||||
var interactions structuredoutput.InteractionSink
|
||||
var capture *sectionInteractions
|
||||
if input.Interactions != nil {
|
||||
capture = §ionInteractions{InteractionSink: input.Interactions}
|
||||
interactions = capture
|
||||
}
|
||||
validationInput := validateSectionCandidatesInput{
|
||||
Spec: input.Spec,
|
||||
Policy: input.Policy,
|
||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.proposal.Exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Interactions: interactions,
|
||||
}
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
||||
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
||||
}
|
||||
validated, err := validateSectionCandidates(runCtx, validationInput)
|
||||
var repairs []RepairOutcome
|
||||
phase, kind := "validation", outcomes.OperationValidation
|
||||
if err == nil && repairPromptID != "" {
|
||||
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
|
||||
}
|
||||
var failure *ExecutionFailure
|
||||
if err != nil {
|
||||
if !errors.As(err, &failure) {
|
||||
failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations)
|
||||
}
|
||||
delete(pending, nextSectionToProcess)
|
||||
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
|
||||
sectionEnriched := collectCandidates(sectionResult, sectionMeta.Index)
|
||||
|
||||
vwg.Add(1)
|
||||
go func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal, sectionMetadata contracts.SectionMetadata) {
|
||||
defer vwg.Done()
|
||||
validationInput := validateSectionCandidatesInput{
|
||||
Spec: input.Spec,
|
||||
Policy: input.Policy,
|
||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.proposal.Exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Interactions: input.Interactions,
|
||||
}
|
||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
||||
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
||||
}
|
||||
validated, err := validateSectionCandidates(runCtx, validationInput)
|
||||
var repairs []RepairOutcome
|
||||
phase, kind := "validation", outcomes.OperationValidation
|
||||
if err == nil && repairPromptID != "" {
|
||||
phase, kind = "correction_repair", outcomes.OperationCorrectionRepair
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, repairPromptID)
|
||||
}
|
||||
var failure *ExecutionFailure
|
||||
if err != nil {
|
||||
if !errors.As(err, &failure) {
|
||||
failure = locateFailure(err, phase, failureReference(input.Spec, §ionMetadata.Index, kind), true, validated.operations)
|
||||
}
|
||||
setFailure(failure)
|
||||
}
|
||||
validationResults <- sectionValidationResult{
|
||||
repairs: repairs,
|
||||
sectionPos: sectionPos,
|
||||
enriched: enriched,
|
||||
approved: validated.approved,
|
||||
decisions: validated.decisions,
|
||||
rejected: validated.rejected,
|
||||
operations: validated.operations,
|
||||
filtered: validated.filtered,
|
||||
failure: failure,
|
||||
}
|
||||
}(nextSectionToProcess, sectionEnriched, sectionMeta)
|
||||
nextSectionToProcess++
|
||||
}
|
||||
return sectionValidationResult{
|
||||
interactions: capture,
|
||||
repairs: repairs,
|
||||
sectionPos: sectionPos,
|
||||
approved: validated.approved,
|
||||
decisions: validated.decisions,
|
||||
rejected: validated.rejected,
|
||||
operations: validated.operations,
|
||||
filtered: validated.filtered,
|
||||
failure: failure,
|
||||
}
|
||||
}
|
||||
|
||||
vwg.Wait()
|
||||
close(validationResults)
|
||||
|
||||
// A failed or canceled module can still have usable proposals from later
|
||||
// sections. Assign their indices in order after preserving the dispatched
|
||||
// prefix, without submitting new validation work.
|
||||
for pos := nextSectionToProcess; pos < len(input.Sections); pos++ {
|
||||
if result, ok := pending[pos]; ok {
|
||||
collectCandidates(result, result.section.Index)
|
||||
}
|
||||
}
|
||||
|
||||
for v := range validationResults {
|
||||
validationBySection[v.sectionPos] = v
|
||||
}
|
||||
|
||||
for i := range input.Sections {
|
||||
res, ok := validationBySection[i]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
unit := &out.WorkUnits[i]
|
||||
recordValidation := func(res sectionValidationResult) {
|
||||
unit := &out.WorkUnits[res.sectionPos]
|
||||
unit.Operations = append(unit.Operations, res.operations...)
|
||||
unit.Repairs = res.repairs
|
||||
mergeFilteredProposals(unit, res.filtered)
|
||||
@@ -617,6 +569,69 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
||||
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
||||
}
|
||||
|
||||
nextAdmission, active := 0, 0
|
||||
for {
|
||||
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
|
||||
offset := nextProposalIndex
|
||||
recordCandidates(nextSectionToRecord, work.candidates)
|
||||
if work.validation != nil {
|
||||
work.validation.assignProposalIndices(offset)
|
||||
recordValidation(*work.validation)
|
||||
}
|
||||
delete(pending, nextSectionToRecord)
|
||||
nextSectionToRecord++
|
||||
}
|
||||
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
|
||||
sectionPos := nextAdmission
|
||||
nextAdmission++
|
||||
pending[sectionPos] = §ionWork{}
|
||||
active++
|
||||
workers.Go(func() {
|
||||
proposed := proposeSection(runCtx, input, sectionPos)
|
||||
setFailure(proposed.failure)
|
||||
proposalResults <- proposed
|
||||
})
|
||||
}
|
||||
if active == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case result := <-proposalResults:
|
||||
active--
|
||||
unit := &out.WorkUnits[result.sectionPos]
|
||||
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
||||
if result.started {
|
||||
unit.State = outcomes.WorkInterrupted
|
||||
}
|
||||
unit.Failure = result.failure
|
||||
work := pending[result.sectionPos]
|
||||
if result.started && result.failure == nil {
|
||||
work.candidates = makeCandidates(result)
|
||||
if runCtx.Err() == nil {
|
||||
enriched := work.candidates
|
||||
active++
|
||||
workers.Go(func() {
|
||||
validated := validate(result, enriched)
|
||||
setFailure(validated.failure)
|
||||
validationResults <- validated
|
||||
})
|
||||
continue
|
||||
}
|
||||
}
|
||||
work.finished = true
|
||||
case validated := <-validationResults:
|
||||
active--
|
||||
work := pending[validated.sectionPos]
|
||||
work.validation = &validated
|
||||
work.finished = true
|
||||
}
|
||||
}
|
||||
workers.Wait()
|
||||
sort.SliceStable(out.Approved, func(i, j int) bool {
|
||||
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
|
||||
})
|
||||
|
||||
for _, unit := range out.WorkUnits {
|
||||
out.Operations = append(out.Operations, unit.Operations...)
|
||||
}
|
||||
|
||||
111
internal/framework/runner/section_evidence.go
Normal file
111
internal/framework/runner/section_evidence.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
// Section validation can finish before preceding proposal counts are known.
|
||||
type sectionInteractions struct {
|
||||
structuredoutput.InteractionSink
|
||||
mu sync.Mutex
|
||||
builders []structuredoutput.IndexedInteraction
|
||||
written []structuredoutput.InteractionArtifacts
|
||||
}
|
||||
|
||||
const pendingInteractionPrefix = "\x00interaction:"
|
||||
|
||||
func (s *sectionInteractions) WriteIndexedInteraction(build structuredoutput.IndexedInteraction) (structuredoutput.InteractionArtifacts, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
receipt := pendingInteractionPrefix + strconv.Itoa(len(s.builders))
|
||||
s.builders = append(s.builders, build)
|
||||
return structuredoutput.InteractionArtifacts{ResponsePayloadPath: receipt}, nil
|
||||
}
|
||||
|
||||
func (s *sectionInteractions) write(offset int) {
|
||||
for _, build := range s.builders {
|
||||
paths, _ := s.InteractionSink.WriteInteraction(build(offset, s.responsePath))
|
||||
s.written = append(s.written, paths)
|
||||
}
|
||||
s.builders = nil
|
||||
}
|
||||
|
||||
func (s *sectionInteractions) artifacts(paths structuredoutput.InteractionArtifacts) structuredoutput.InteractionArtifacts {
|
||||
value, pending := strings.CutPrefix(paths.ResponsePayloadPath, pendingInteractionPrefix)
|
||||
if !pending {
|
||||
return paths
|
||||
}
|
||||
index, err := strconv.Atoi(value)
|
||||
if err != nil || index < 0 || index >= len(s.written) {
|
||||
return structuredoutput.InteractionArtifacts{}
|
||||
}
|
||||
return s.written[index]
|
||||
}
|
||||
|
||||
func (s *sectionInteractions) responsePath(path string) string {
|
||||
return s.artifacts(structuredoutput.InteractionArtifacts{ResponsePayloadPath: path}).ResponsePayloadPath
|
||||
}
|
||||
|
||||
func (r *sectionValidationResult) assignProposalIndices(offset int) {
|
||||
capture := r.interactions != nil
|
||||
if capture {
|
||||
r.interactions.write(offset)
|
||||
}
|
||||
for i := range r.approved {
|
||||
r.approved[i].ProposalIndex += offset
|
||||
}
|
||||
for i := range r.rejected {
|
||||
r.rejected[i].ProposalIndex += offset
|
||||
}
|
||||
assignDecisions := func(decisions []ValidatorDecisionRecord) {
|
||||
for i := range decisions {
|
||||
decisions[i].ProposalIndex += offset
|
||||
if capture {
|
||||
decisions[i].DiagnosticArtifactPath = r.interactions.responsePath(decisions[i].DiagnosticArtifactPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
assignDecisions(r.decisions)
|
||||
for i := range r.operations {
|
||||
op := &r.operations[i]
|
||||
op.Reference.ProposalIndex = structuredoutput.OffsetProposalIndex(op.Reference.ProposalIndex, offset)
|
||||
op.ProposalIndices = slices.Clone(op.ProposalIndices)
|
||||
for j := range op.ProposalIndices {
|
||||
op.ProposalIndices[j] += offset
|
||||
}
|
||||
op.Filtered = slices.Clone(op.Filtered)
|
||||
for j := range op.Filtered {
|
||||
op.Filtered[j].ProposalIndex += offset
|
||||
}
|
||||
}
|
||||
for i := range r.filtered {
|
||||
r.filtered[i].ProposalIndex += offset
|
||||
}
|
||||
for i := range r.repairs {
|
||||
repair := &r.repairs[i]
|
||||
repair.ProposalIndex += offset
|
||||
if capture {
|
||||
repair.Original.ProposalIndex += offset
|
||||
repair.Final.ProposalIndex += offset
|
||||
}
|
||||
for j := range repair.Attempts {
|
||||
attempt := &repair.Attempts[j]
|
||||
if attempt.Revision != nil {
|
||||
attempt.Revision.ProposalIndex += offset
|
||||
}
|
||||
assignDecisions(attempt.Decisions)
|
||||
if capture {
|
||||
attempt.Artifacts = r.interactions.artifacts(attempt.Artifacts)
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.failure != nil && r.failure.Origin != nil {
|
||||
r.failure.Origin.ProposalIndex = structuredoutput.OffsetProposalIndex(r.failure.Origin.ProposalIndex, offset)
|
||||
}
|
||||
r.interactions = nil
|
||||
}
|
||||
275
internal/framework/runner/window_test.go
Normal file
275
internal/framework/runner/window_test.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type sharedBackendModule struct {
|
||||
*grammar.Module
|
||||
validator validators.Validator
|
||||
headEntered <-chan struct{}
|
||||
}
|
||||
|
||||
func (m sharedBackendModule) Validators() []validators.Validator {
|
||||
return []validators.Validator{m.validator}
|
||||
}
|
||||
|
||||
func (m sharedBackendModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if req.Section.Index > 0 {
|
||||
select {
|
||||
case <-m.headEntered:
|
||||
case <-ctx.Done():
|
||||
return contracts.ProposalResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return m.Module.Propose(ctx, req)
|
||||
}
|
||||
|
||||
func TestLaterSectionsValidateWhileEarlierProposalUsesSharedBackend(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
|
||||
defer cancel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(path, []byte("version: 3\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 2\n queue_capacity: 64\nprofiles:\n - id: base\n backend: shared\n model: fixture\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(path)}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := make(chan struct{})
|
||||
entered := make(chan struct{})
|
||||
validationStarted := make(chan struct{}, 1)
|
||||
var other, validating, active, peak atomic.Int32
|
||||
client := &http.Client{Transport: repairTransport(func(r *http.Request) (*http.Response, error) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n := active.Add(1)
|
||||
for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
|
||||
}
|
||||
defer active.Add(-1)
|
||||
output := `{"corrections":[{"id":1,"original_text":"word","corrected_text":"Word"}]}`
|
||||
switch {
|
||||
case bytes.Contains(body, []byte(`"validations"`)):
|
||||
validating.Add(1)
|
||||
select {
|
||||
case validationStarted <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
output = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||
case bytes.Contains(body, []byte("marker-000")):
|
||||
close(entered)
|
||||
select {
|
||||
case <-release:
|
||||
case <-r.Context().Done():
|
||||
return nil, r.Context().Err()
|
||||
}
|
||||
default:
|
||||
other.Add(1)
|
||||
}
|
||||
content, _ := json.Marshal(output)
|
||||
wire := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(wire))}, nil
|
||||
})}
|
||||
runtime, err := llm.NewRuntime(ctx, loaded, client)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
role, err := runtime.NewRoleClient("fixture", "base", config.LLM{OperationTimeout: 5 * time.Second}, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator, headEntered: entered}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Chunking.MaxSectionTokens = 1
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
transcript := &schema.Transcript{}
|
||||
for i := 0; i < 36; i++ {
|
||||
transcript.Segments = append(transcript.Segments, schema.Segment{ID: 100 + i, Text: fmt.Sprintf("word marker-%03d", i)})
|
||||
}
|
||||
type result struct {
|
||||
out RunOutput
|
||||
err error
|
||||
}
|
||||
finished := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := r.Run(ctx, RunInput{Config: &cfg, Transcript: transcript,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", Settings: config.Stage{Validation: cfg.Validation}, ProposalClient: role, ValidationClient: role}},
|
||||
})
|
||||
finished <- result{out, err}
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
select {
|
||||
case <-validationStarted:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("later validation did not reach the backend while the earlier proposal held a slot")
|
||||
}
|
||||
close(release)
|
||||
completed := <-finished
|
||||
if completed.err != nil {
|
||||
t.Fatal(completed.err)
|
||||
}
|
||||
if other.Load()+1 != int32(len(transcript.Segments)) || validating.Load() != int32(len(transcript.Segments)) || peak.Load() > 2 {
|
||||
t.Fatalf("unexpected native work: proposals=%d validations=%d peak=%d", 1+other.Load(), validating.Load(), peak.Load())
|
||||
}
|
||||
changes := completed.out.ModuleResults[0].AppliedChanges
|
||||
if len(changes) != len(transcript.Segments) {
|
||||
t.Fatalf("got %d applied changes", len(changes))
|
||||
}
|
||||
for i, change := range changes {
|
||||
if change.ProposalIndex != i || change.TargetSegmentID != transcript.Segments[i].ID {
|
||||
t.Fatalf("change %d: %+v", i, change)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||
for _, canceled := range []bool{false, true} {
|
||||
name := "completed"
|
||||
if canceled {
|
||||
name = "canceled"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
laterRevalidated := make(chan struct{})
|
||||
proposal := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
|
||||
section := *req.Context.SectionIndex
|
||||
if section == 0 {
|
||||
select {
|
||||
case <-laterRevalidated:
|
||||
case <-ctx.Done():
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
text := []string{"alpha", "bravo"}[section]
|
||||
dst.(*proposal_generation.StructuredCorrectionSet).Corrections = []proposal_generation.StructuredCorrectionProposal{{
|
||||
SegmentReference: 1, OriginalText: text, CorrectedText: strings.ToUpper(text),
|
||||
}}
|
||||
return contracts.StructuredCompletionResponse{Exchange: &structuredoutput.Exchange{}}, nil
|
||||
})
|
||||
validation := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, dst any) (contracts.StructuredCompletionResponse, error) {
|
||||
section := *req.Context.SectionIndex
|
||||
first := section == 0
|
||||
revision := req.Context.RepairAttempt != nil
|
||||
if first && canceled {
|
||||
cancel()
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
|
||||
CorrectionIndex: "1", Approved: first || revision, Reason: "checked",
|
||||
}}
|
||||
if revision {
|
||||
close(laterRevalidated)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
})
|
||||
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
module := sharedBackendModule{Module: &grammar.Module{}, validator: validator}
|
||||
// The module's transport gates are supplied by the scripted calls above.
|
||||
started := make(chan struct{})
|
||||
close(started)
|
||||
module.headEntered = started
|
||||
repair := &repairClient{t: t, outputs: []string{
|
||||
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
|
||||
}}
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Chunking.MaxSectionTokens = 1
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
||||
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{
|
||||
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
|
||||
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}},
|
||||
}},
|
||||
})
|
||||
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
later := out.WorkUnits[1]
|
||||
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
|
||||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 {
|
||||
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
|
||||
}
|
||||
attempt := later.Repairs[0].Attempts[0]
|
||||
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
|
||||
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
|
||||
t.Fatalf("lost ordered repair evidence: %+v", attempt)
|
||||
}
|
||||
var repaired struct {
|
||||
Decoded proposal_generation.RepairResponse
|
||||
Revision *proposals.EnrichedCorrectionProposal
|
||||
Decisions []ValidatorDecisionRecord
|
||||
}
|
||||
readInteraction(t, attempt.Artifacts.ResponsePayloadPath, &repaired)
|
||||
if repaired.Decoded.ProposalRef != 1 || repaired.Revision == nil || repaired.Revision.ProposalIndex != 1 ||
|
||||
len(repaired.Decisions) != 1 || repaired.Decisions[0].ProposalIndex != 1 ||
|
||||
repaired.Decisions[0].DiagnosticArtifactPath != attempt.Decisions[0].DiagnosticArtifactPath ||
|
||||
!strings.Contains(attempt.Artifacts.ResponsePayloadPath, "proposal-0001") {
|
||||
t.Fatalf("repair artifact uses different identities: %+v", repaired)
|
||||
}
|
||||
var checked struct {
|
||||
Decoded validators.LLMValidationResponse
|
||||
Reconciliation struct{ Decisions []validators.Decision }
|
||||
}
|
||||
readInteraction(t, attempt.Decisions[0].DiagnosticArtifactPath, &checked)
|
||||
if len(checked.Decoded.Validations) != 1 || checked.Decoded.Validations[0].CorrectionIndex != "1" ||
|
||||
len(checked.Reconciliation.Decisions) != 1 || checked.Reconciliation.Decisions[0].ProposalIndex != 1 {
|
||||
t.Fatalf("validation evidence uses different identities: %+v", checked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func readInteraction(t *testing.T, path string, result any) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -22,3 +22,24 @@ type InteractionArtifacts struct {
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
|
||||
// IndexedInteraction builds an artifact after the section's ordered proposal
|
||||
// indices and paths from earlier interactions are available.
|
||||
type IndexedInteraction func(offset int, resolvePath func(string) string) Interaction
|
||||
|
||||
func WriteIndexedInteraction(sink InteractionSink, build IndexedInteraction) (InteractionArtifacts, error) {
|
||||
if deferred, ok := sink.(interface {
|
||||
WriteIndexedInteraction(IndexedInteraction) (InteractionArtifacts, error)
|
||||
}); ok {
|
||||
return deferred.WriteIndexedInteraction(build)
|
||||
}
|
||||
return sink.WriteInteraction(build(0, func(path string) string { return path }))
|
||||
}
|
||||
|
||||
func OffsetProposalIndex(index *int, offset int) *int {
|
||||
if index == nil || offset == 0 {
|
||||
return index
|
||||
}
|
||||
next := *index + offset
|
||||
return &next
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -141,33 +142,44 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
if req.Interactions == nil {
|
||||
return structuredoutput.InteractionArtifacts{}
|
||||
}
|
||||
stageInstance := req.ModuleInstance
|
||||
if req.SectionIndex != nil {
|
||||
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
|
||||
}
|
||||
if req.RepairProposalIndex != nil && req.RepairAttempt != nil {
|
||||
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt)
|
||||
}
|
||||
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
|
||||
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
|
||||
if reconciled != nil {
|
||||
responsePayload["reconciliation"] = *reconciled
|
||||
}
|
||||
artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: stage, Context: coordinates, RequestMetadata: map[string]any{
|
||||
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
||||
"validator_name": v.name,
|
||||
"validator_type": v.validatorType,
|
||||
"batch_index": batch.BatchIndex,
|
||||
"proposal_indices": batchProposalIndices(batch.Items),
|
||||
"repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt,
|
||||
"prompt": completion.Prompt,
|
||||
"response_schema": responseSchema.DiagnosticsMap(),
|
||||
},
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: responsePayload,
|
||||
ErrorPayload: errPayload(callErr),
|
||||
artifacts, _ := structuredoutput.WriteIndexedInteraction(req.Interactions, func(offset int, _ func(string) string) structuredoutput.Interaction {
|
||||
coordinates := coordinates
|
||||
coordinates.ProposalIndex = structuredoutput.OffsetProposalIndex(req.RepairProposalIndex, offset)
|
||||
stageInstance := req.ModuleInstance
|
||||
if req.SectionIndex != nil {
|
||||
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
|
||||
}
|
||||
if coordinates.ProposalIndex != nil && req.RepairAttempt != nil {
|
||||
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *coordinates.ProposalIndex, *req.RepairAttempt)
|
||||
}
|
||||
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
|
||||
if reconciled != nil {
|
||||
view := *reconciled
|
||||
view.Decisions = slices.Clone(view.Decisions)
|
||||
for i := range view.Decisions {
|
||||
view.Decisions[i].ProposalIndex += offset
|
||||
view.Decisions[i].DiagnosticArtifactPath = ""
|
||||
}
|
||||
view.UnavailableIndices = slices.Clone(view.UnavailableIndices)
|
||||
for i := range view.UnavailableIndices {
|
||||
view.UnavailableIndices[i] += offset
|
||||
}
|
||||
responsePayload["reconciliation"] = view
|
||||
}
|
||||
indices := batchProposalIndices(batch.Items)
|
||||
for i := range indices {
|
||||
indices[i] += offset
|
||||
}
|
||||
return structuredoutput.Interaction{
|
||||
Name: stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex), Context: coordinates, RequestMetadata: map[string]any{
|
||||
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
||||
"validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex,
|
||||
"proposal_indices": indices, "repair_proposal_index": coordinates.ProposalIndex, "repair_attempt": req.RepairAttempt,
|
||||
"prompt": completion.Prompt, "response_schema": responseSchema.DiagnosticsMap(),
|
||||
},
|
||||
RequestPayload: completion.Prepared, ResponsePayload: responsePayload, ErrorPayload: errPayload(callErr),
|
||||
}
|
||||
})
|
||||
return artifacts
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user